
/**************************************************************************
 * This file is part of TraceTuner, the DNA sequencing quality value,
 * base calling and trace processing software.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received (LICENSE.txt) a copy of the GNU General Public
 * License along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 *************************************************************************/


/*******************************************************************
 * Copyright (c) 2003-2008 Gennady Denisov.  All rights reserved.

 * Filename: Btk_process_raw_data.c
 *
 * Description: The functions in this file are for processing the 
 *     raw data into so-called analyzed data.
 *
 *
 *******************************************************************/

#include <stdlib.h>
#include <stdio.h>
#include <math.h>
#include <limits.h>
#include <string.h>
#include <sys/param.h>
#include <float.h>
#include <time.h>

#include "Btk_qv.h"
#include "Btk_qv_data.h"
#include "util.h"
#include "Btk_match_data.h"
#include "Btk_qv_io.h"
#include "Btk_qv_funs.h"
#include "Btk_compute_match.h"
#include "Btk_process_raw_data.h"
#include "train.h"
#include "Btk_process_peaks.h"
#include "Btk_scf_data.h"
#include "nr.h"
#include "tracepoly.h"
#include "context_table.h"
#include "Btk_call_bases.h"

#define AD_MAX 8191     // largest value generated by A/D converter
#define APPLY_MOB_SHIFTS 1
#define AVERAGE_PEAK_HEIGHT 1000
#define CALCULATE_NEW_SPACING_CURVE 0
#define CLIP_DATA_BEG 1
#define CLIP_DATA_END 1
#define COMPLETE_MOB_SHIFTS 1
#define DATA_WINDOW_SIZE 200
#define DEBUG  0
#define DEBUG0 0
#define DEBUG_CURTIS 0
#define DEFAULT_NUM_WINDOWS 40
#define DEFAULT_SPACING 12
#define DPRINT(x) fprintf(stderr, #x " = %g\n", (float)(x)) // for debugging
#define ERROR -1
#define HIST_HEIGHTS_STEP 100
#define MAX_PEAK_HEIGHT 4000
#define MAX_SHIFT_MAX 2.0
#define MAX_SHIFT_INC 0.5
#define MIN_RELATIVE_CHANNEL_WEIGHT 0.12
#define MIN_WIN_SIZE    300
#define OUTPUT_HISTOGRAMS 0
#define OUTPUT_SPACING 0
#define POLY_MOB_SHIFT_APPROX 0
#define POLY_SPAC_MODEL_APPROX 0
#define RATIO_SHIFT_MAX_REL_SPC_ERR 1.5
#define REL_SPC_VAR_THRESH 0.25
#define SAMPLE_RATE 1
#define SHIFT_ERR_GROW_FACTOR 1.2
#define SHIFT_MAX_THRES 2.0
#define SHIFTING_TOLERANCE 0.5
#define SHOW_FILTERS_OUTPUT 1
#define SHOW_NORM_FACTOR 0
#define SHOW_NUM_DATA 0
#define SHOW_NUM_PEAKS 0
#define SHOW_POWER_SPECTRUM 1
#define SHOW_SIGNAL_EVOLUTION 1
#define SIGNAL_DROP_THRESHOLD 6
#define SQRT_ENVELOPE_ASYMPTOTE 0.5
#define SWAP(a,b) tempr=(a);(a)=(b);(b)=tempr

/* Crude estimate of peak spacing */
static int    num_windows = DEFAULT_NUM_WINDOWS;
static int    isweet;
static double crude_spacing_estimate = 8.0;
static float  mobs_model_coeff[POLYFIT_DEGREE + 1];
static float  spac_model_coeff[POLYFIT_DEGREE + 1];
static float  spac_mod_val[DEFAULT_NUM_WINDOWS];
static float  spac_mod_pos[DEFAULT_NUM_WINDOWS];
static float  norm_mod_val[NUM_COLORS][DEFAULT_NUM_WINDOWS];
static float  norm_mod_pos[DEFAULT_NUM_WINDOWS];

static void 
bubble(int *data, int num_data)

   /*********************************************************************
    * Simple buble sort for int data
    *
    * Inputs:   data    pointer to array of data
    *           num_data  size of data array
    * Outputs:  *data   sorted data
    * Return:   void
    * Comments:
    */

{
    long        i, j;
    int         temp=0;

    for (i=num_data-1; i>=1; i--)
        for (j=1; j<=i; j++)
            if (data[j-1] > data[j]) {  // swap
                temp = data[j];
                data[j] = data[j-1];
                data[j-1] = temp;
            }
}

/*********************************************************************
 * Function: median_filter_n5
 * Data which are smaller than (average - stddev/2) of 5 point window
 * are replaced with the average of the 3 largest data in the window.
 *
 * Inputs:	data	pointer to array of data
 * 		num_data	length of data aray
 * Outputs:	*data	filtered data
 * Return:	void
 * Comments:	CSG - copied from ABI Basecaller source code
 */
static void
median_filter_n5(int *data, int num_data)
{
    int	    j, *ptr;
    long    i, size;
    int	    subData[5], replacement;
    float   sum, sumsqrd, mean, std, threshold;

    ptr = data + 2;
    size = num_data - 4;

#if DEBUG > 1
fprintf(stderr, "median_filter_n5() ...\n");
#endif

    for (i=0; i<size; i++) {
        // Fill subData with 5 points in this window
        sum = 0;
        sumsqrd = 0;
        for (j=0; j<5; j++) {
            subData[j] = ptr[i+j-2];
            sum += subData[j];
            sumsqrd += subData[j]*subData[j];
        }
        // Compute stats on 5 points
        mean = sum/5;
        std = sqrt( fabs(sumsqrd/5 - mean*mean) );
        threshold = mean - 0.5 * std;

        // Sort subdata
        bubble(subData, 5);

        // Replace points lower than threshold with avg of 3 largest points 
        // in this window
        if (ptr[i] < threshold) {
            // Compute the replacement value
            replacement = (int)(subData[2] + subData[3] + subData[4]) / 3;
	/* This method would be prefered, but it causes problems when there 
	    are large positive spikes like primer peaks
                int left, right;
                left = ptr[i-1];
                j = 1;
                while ( (j<5)&&(ptr[i+j] <= threshold) ) j++;
                if (j<5) right = ptr[i+j];
                else right = left;
                replacement = left + (right-left)/(j+1); // left + slope * 1
	*/

            ptr[i] = replacement;
        }
    }
    return;
}


static int 
max_element(int *data, int  num_data)

   /*********************************************************************
    * Find largest value of array of data.
    *
    * Inputs:   data    pointer to array of data
    *           num_data  length of data array
    * Outputs:  none
    * Return:   maximum value of data
    * Comments:
    */

{
    int         max = data[0];
    long        i;

    for (i=1; i < num_data; i++)
        if ( data[i] > max ) max = data[i];

    return(max);
}


/*********************************************************************
 * Identify the location of the first base in the data.
 *
 * Inputs:	data	pointer to array of data
 * 		num_data	length of data array
 * Outputs:	none
 * Return:	index location of first base in data array
 * Comments:	
 */
static long
first_base_pos(int *data[NUM_COLORS], int num_data)
{
    const long	a = 6;	// interval for differencing
    const int	lim_big = 300,	// required number of points above threshold
		lim_small = 75; // reset counters if points < threshold 
				// exceed this

    int		i;
    long	j;
    int		diff[NUM_COLORS];
    int	       *diff_env, *ptr;
    long	sum = 0, sumofsqr = 0;	// for statistics
    float	noise_max, sig_mean, threshold;
    int		num_big, num_small; 		// counters

#if DEBUG
    fprintf(stderr, "first_base_pos ...\n");
    fprintf(stderr, "int first_base_pos: num_data=%d\n", (int)num_data);
#endif

    diff_env = (int *) malloc((size_t) num_data * sizeof(int));

    for ( j=0; j < num_data-a; ++j ) 	// CSG - could probably stop earlier
    {
        for ( i=0; i < NUM_COLORS; i++ ) {
	    diff[i] = abs(data[i][j+a] - data[i][j]);
        }
	diff_env[j] = max_element(diff, NUM_COLORS);
	// CSG: printf("%8ld %8d\n", j, diff_env[j]);
        // diff_env ("difference envelope"?) contains the max local
        //          "derivative" among all the for traces
    }

    // use early part of data as sample of noise
    for ( i=100; i < 400; i++) {
	sum += diff_env[i];
	sumofsqr += diff_env[i] * diff_env[i];
    }
    // noise max = average + 3*(std. dev.)
    noise_max = (float) sum / 300.0;
    noise_max += 3 * sqrt( (float) sumofsqr / 300.0 - noise_max*noise_max );

#if DEBUG > 2
DPRINT(noise_max);
#endif

    // examine signal around scan 3800
    sum = sumofsqr = 0;
    for ( i=QVMIN(3690, num_data)-300; i < QVMIN(3690, num_data); i++ ) {
	sum += diff_env[i];
	sumofsqr += diff_env[i] * diff_env[i];
    }
    // signal min = average - 3*(std. dev.)
    sig_mean = (float) sum / 300.0;
#if DEBUG > 2
DPRINT(sig_mean);
#endif

    // compute threshold collared between 32 and 250
    if ( sig_mean > noise_max ) {
        threshold = 0.5 * (sig_mean + noise_max);
    }
    else {
        threshold = 2 * noise_max;
    }
    if ( threshold > 250 ) {
        threshold = 250;
    }
    else if ( threshold < 32 ) {
        threshold = 32;
    }
#if DEBUG > 2
DPRINT(threshold);
#endif

    // scan diff_env to find start point
    // scan through until the threshold is exceeded
    ptr = diff_env; 
    while ( *++ptr <= threshold ) ; // putc('.', stderr);
    // putc('\n', stderr);
    num_big = num_small = 0;
    do {
	ptr += num_big + num_small;
	num_big = num_small = 0;
	if ( ptr - diff_env + lim_big + lim_small > num_data ) {
	    fprintf(stderr, "first_base_pos: reached end of data\n");
	    FREE(diff_env);
	    return(0);  // too close to end of data
	}
	for ( i=0; num_big <= lim_big && num_small <= lim_small; i++) 
	    if ( ptr[i] > threshold ) {
		++num_big;
		num_small = 0;
	    }
	    else ++num_small;
    } while ( num_small > lim_small );

    // CSG - Done, as long as we're working with terminator dye data.
    // CSG - Primer dye data requires finding end of primer peak.

    i = ptr - diff_env;
    FREE(diff_env);
    return(i);
}


/*********************************************************************
 * Find smallest member of array of data.
 *
 * Inputs:   data    array of data
 *           num_data  length of data array
 * Outputs:  none
 * Return:   minimum value of data
 * Comments:
 */
static int
min_element(int *data, int  num_data)
{
    int         min = data[0];
    long        i;

    for ( i=1; i < num_data; i++ ) {
        if ( data[i] < min ) {
            min = data[i];
        }
    }

    return(min);
}


static int
second_smallest(int *data, int  num_data)

   /*********************************************************************
    * Find the second smallest elements of array of data
    *
    * Inputs:   data    array of data
    *           num_data  length of data array
    * Outputs:  none
    * Return:   second smallest value of data
    * Comments:
    */

{
    int         min1 = SHRT_MAX,
                min2 = SHRT_MAX;
    long        i;

#if DEBUG > 3
fprintf(stderr, "second_smallest() ...\n");
#endif
    for ( i=0; i < num_data; i++ ) {
        if ( data[i] < min1 ) {
            min2 = min1;
            min1 = data[i];
        }
        if ( min1 < data[i] && data[i] < min2 ) {
            min2 = data[i];
        }
    }

    return(min2);
}


static void 
boxcar_filt(int *data, int  num_data, int num_win)

   /*********************************************************************
    * Low-pass filter by convolution with a square impulse function.
    *
    * Inputs:   data    array of data
    *           num_data  length of data array
    *           num_win   length of square impulse function
    * Outputs:  data    array of filtered data
    * Return:   void
    * Comments: Temporarily allocates memory for buffer for filtered data.
    */

{
    int         j;
    long        i;
    int         *temp;
    long        sum = 0;;

#if DEBUG > 3
fprintf(stderr, "boxcar filt() ...\n");
#endif

    if ( num_win > num_data/2 ) return;     // num_win too large

    // allocate memory for data buffers
    temp = (int *) calloc(num_data, sizeof(int));

    j = (num_win-1)/2;    // use j to track the number of data in the window
    for ( i=0; i < (num_win-1)/2; i++ ) {
        sum += data[i];
    }

    for ( i=1; i < num_data; i++ ) {
        // update sum of data in window
        if ( i > num_win/2) 
            sum -= data[i-num_win/2-1];
        else 
            ++j;
        if ( i < num_data - (num_win-1)/2 ) 
            sum += data[i+(num_win-1)/2];
        else 
            --j;
        temp[i] = sum/j;
    }

    memcpy(data, temp, num_data*sizeof(int));
    FREE(temp);
    return;
}

static void
fir_filter(int *data, int  num_data, int filt_id)
   /*********************************************************************
    * Apply a finite impulse response filter to array of data.
    * Inputs:   data    array of data
    *           num_data  length of data array
    *           filt_id integer ID code of filter to apply
    * Outputs:  data    filtered data
    * Return:   void
    * Comments: Copies data.
    */
{
    const int   f0coef[] = { 1, 2, 1}; // lowpass: FT = [cos(omega/2)]^4
    const int   f1coef[] = { 
            34,   30,   -22,   -95,  -107,   // ~ideal lowpass
             0,  180,   269,   107,  -269,    // ( ABI's / 3 )
          -592, -461,   338,  1612,  2795,  
                       3277,
          2795, 1612,   338,  -461,  -592,
          -269,  107,   269,   180,     0,
          -107,  -95,   -22,    30,     34 };
    const int   f2coef[] = { 
          -178, -544,  -926,  -873,     0, // bandpass (ABI's / 4)
          1578, 3141,  3796,  3141,  1578,
             0, -873,  -926,  -544,  -178 };
    const int   *f[] = { f0coef, f1coef, f2coef };
    const int   fsize[] = { 3, 31, 15 };
    const int   *fcent = f[filt_id] + (fsize[filt_id]-1)/2;

    int         i, j;
    int         *temp;
    long        csum, fsum = 0;

    // allocate data buffer
    temp = (int *) malloc(num_data*sizeof(int));
    memcpy(temp, data, num_data*sizeof(int));

    // find sum of filter coefficients
    // CSG - this won't work for perfect DC-stop filters
    for ( i=0; i < fsize[filt_id]; i++ ) {
        fsum += f[filt_id][i];
    }

    for ( i = fsize[filt_id]/2; i < num_data - (fsize[filt_id]-1)/2; i++ ) {
        csum = 0;
        for ( j = -(fsize[filt_id]-1)/2; j <= fsize[filt_id]/2; ++j ) {
            csum += (long) fcent[j] * temp[i-j];
        }
        data[i] = csum / fsum;
    }

    // Ignore the ends.  CSG - ideally, should do something here.

    FREE(temp);
    return;
}

static void 
fgauss(int *data, int  num_data)

   /*********************************************************************
    * Remove high frequencies with Gaussian filter.
    *
    * Inputs:   data    array of data
    *           num_data  length of data array
    * Outputs:  *data   filtered data
    * Return:   void
    * Comments: Temporarily allocates memory for buffer for filtered data.
    */

{
    const int   lp_filt_id = 0;

    fir_filter(data, num_data, lp_filt_id);

    return;
}


static void 
zero_baseline(int *data, long num_data)

   /*********************************************************************
    * Remove baseline drift from data.
    *
    * Inputs:	data	array of data
    *		num_data	length of data array
    * 		num_win	length of local window
    * Outputs:	data	modified data
    * Return:	voi_
    * Comments:	Temporarily allocates memory for data buffers.
    */

{
    const int	num_win = 50;	// size of local window for filtering
				// should be >> typical peak spacing

				// but << baseline drift scale
    
    long	i;
    int		ibuf;		// used modulo num_win;
    int		*bl, *buf;
    int		min;

    bl = (int *) malloc(num_data*sizeof(int));

#if DEBUG > 1
fprintf(stderr, "zero_baseline() ...\n");
#endif

    // estimate baseline
    bl = memcpy(bl, data, num_data*sizeof(int));
    boxcar_filt(bl, num_data, num_win);
    for ( i=0; i < num_data; i++) {
	if ( data[i] < bl[i] ) {
            bl[i] = data[i];
        }
    }
    boxcar_filt(bl, num_data, num_win);
    // for ( i=0; i < num_data; i++)
	// if ( data[i] < bl[i] ) bl[i] = data[i];

    // allocate and initialize window buffer;
    buf = (int *) malloc(num_win*sizeof(int));
    for ( i=0; i < (num_win-1)/2; i++ ) {
        buf[i] = data[i];
    }
    for ( i=(num_win-1)/2; i < num_win; i++ ) {
        buf[i] = SHRT_MAX;
    }

#if DEBUG > 2
    fprintf(stderr, "averaging 1st baseline estimate with local min "
			"and subtracting...\n");
#endif
    // average last baseline estimate with local minimum
    min = min_element(data, (num_win-1)/2);
    for ( i=0; i < num_data - (num_win-1)/2; i++ ) {
	ibuf = (i + (num_win-1)/2) % num_win;
	if ( buf[ibuf] == min ) min = second_smallest(buf, num_win);
				// about to replace smallest element in buf
	buf[ibuf] = data[i+(num_win-1)/2];
	if ( buf[ibuf] < min ) min = buf[ibuf];
	bl[i] = (bl[i] + min) / 2;
    }
    for ( i = num_data - (num_win-1)/2; i < num_data; i++ ) {
	ibuf = (i + (num_win-1)/2) % num_win;
	if ( buf[ibuf] == min ) min = second_smallest(buf, num_win);
				// about to replace smallest element in buf
	buf[ibuf] = SHRT_MAX;
	bl[i] = (bl[i] + min) / 2;
    }

    // smooth bl and subtract from data
    fgauss(bl, num_data);
    for ( i=0; i < num_data; i++ ) data[i] -= bl[i];

    FREE(bl);
    FREE(buf);
    return;
}


static void 
bandpass(int *data, int  num_data)

   /*********************************************************************
    * Apply bandpass FIR filter.
    *
    * Inputs:   data    array of data
    *           num_data  length of data array
    * Outputs:  data    filtered data
    * Return:   void
    * Comments: Nearly identical to ABI's bandpass filter.
    */

{
    const int   bp_filt_id = 2;

    fir_filter(data, num_data, bp_filt_id);
    return;
}


static int 
med_element(int *data, int  num_data)

   /*********************************************************************
    * Determine median of array of data.
    *
    * Inputs:	data	array of data
    * 		num_data	length of data array
    * Outputs:	none
    * Return:	median value
    * Comments:	Makes copy of data array and uses bubble sort.
    *		Bad for large num_data.
    */

{
    int		*copy;
    int		median;

    copy = (int *) malloc(num_data*sizeof(int));
    memcpy(copy, data, num_data*sizeof(int));
    bubble(copy, num_data);

    if ( num_data % 2 ) median = copy[(num_data-1)/2];
    else median = (copy[num_data/2] + copy[num_data/2 - 1]) / 2;

    FREE(copy);
    return(median);
}


/*********************************************************************
 * Clip data values below floor and above ceiling.
 *
 * Inputs:	data	array of data
 *		num_data	length of data array
 *		floor	minimum allowable data value
 * 		ceiling	maximum allowable data value
 * Outputs:	*data	clipped data
 * Return:	number of data values modified
 * Comments:	if floor >= ceiling, do nothing
 */
static long
clip(int *data, int  num_data, int floor, int ceiling)
{
    int	 *ptr = data;
    long  num_mod = 0;

    if ( floor >= ceiling ) return(0);

    for ( ptr=data; ptr - data < num_data; ptr++ )
    {
        if (*ptr > ceiling ) {
	   *ptr = ceiling;
	    num_mod++;
	}
	else if (*ptr < floor ) {
	   *ptr = floor;
	    num_mod++;
	}
    }

    return(num_mod);
}



static void 
median_filter(int *data, int  num_data, int n)

   /*********************************************************************
    * Replace each data point with median of data in window of size n
    * centered on that point.
    *
    * Inputs:	data	array of data
    * 		num_data	length of data array
    *		n	size of filter window (preferably odd and small)
    * Outputs:	*data	filtered data
    * Return:	void
    * Comments:	First n/2 and last (n-1)/2 elements of data are not 
    *		modified.  CSG - Could be made more efficient.
    */

{
    int		*ptr, *tmp, *window;

    window = tmp = (int *) malloc(num_data*sizeof(int));
    memcpy(tmp, data, num_data*sizeof(int));
    for ( ptr = data+n/2; ptr-data < num_data-(n-1)/2; ++ptr, ++window )
	*ptr = med_element(window, n);

    FREE(tmp);
    return;
}


static void 
lowpass(int *data, int  num_data)

   /*********************************************************************
    * Filter out high frequencies with nearly ideal (square) frequency
    * response.
    *
    * Inputs:	data	array of data
    * 		num_data	length of data array
    * Outputs:	*data	filtered data
    * Return:	void
    * Comments:	Nearly identical to ABI's lowpass filter.
    */

{
    const int	lp_filt_id = 1;

    fir_filter(data, num_data, lp_filt_id);

    return;
}




static void 
savgol255_filter(int *xdata, int  num_data, int filt_id)
{
    int i, j;
    int halfwin= 5;
    const float xcoeff[] = {
	    -0.0839160839160839, 0.02097902097902099, 0.10256410256410256,
	    0.16083916083916083, 0.19580419580419578, 0.20745920745920743,
	    0.19580419580419578, 0.16083916083916083, 0.10256410256410256, 
	    0.02097902097902099, -0.0839160839160839};
    int *ydata = CALLOC(int, num_data);

    filt_id = 255;
    for (i = 0; i < num_data; i++) {
        float sum = 0.;
        for (j=-halfwin; j<=halfwin; j++) {
	    int ind = i+j;
	    if ((ind < 0) || (ind > num_data-1)) 
		continue;
	    sum += xcoeff[j+halfwin] * (float)xdata[ind];
	}
        ydata[i] = (int)sum;
    }  

    for (i = 0; i < num_data; i++) {
        xdata[i] = ydata[i];
    }
    FREE(ydata);    
}

void savgol41616_filter(int *xdata, int  num_data, int filt_id)
{
    int i, j;
    int halfwin= 16;
    const float xcoeff[] = {
           0.03685503685503666,  0.002457002457002422,  -0.019021954505825445,
          -0.02972180391535223, -0.03163493152369673,   -0.02660613895319111,
          -0.01633264369638107, -0.0023640791160256386,  0.013897505554902722, 
  	   0.0311976452132181,   0.048429458885521165,   0.06463364972819918,
	   0.07899850502742597,  0.09085989619916195,    0.0997012787891541,
	   0.10515369247293593,  0.10699576105582766,    0.10515369247293593,  
           0.0997012787891541,   0.09085989619916195,    0.07899850502742597,
	   0.06463364972819918,  0.048429458885521165,   0.0311976452132181,
	   0.013897505554902722,-0.0023640791160256386, -0.01633264369638107,
	  -0.02660613895319111, -0.03163493152369673,   -0.02972180391535223,
	  -0.019021954505825445, 0.002457002457002422,   0.03685503685503666};
    int *ydata = CALLOC(int, num_data);

    filt_id = 255;
    for (i = 0; i < num_data; i++) {
        float sum = 0.;
        for (j=-halfwin; j<=halfwin; j++) {
            int ind = i+j;
            if ((ind < 0) || (ind > num_data-1))
                continue;
            sum += xcoeff[j+halfwin] * (float)xdata[ind];
        }
        ydata[i] = (int)sum;
    }

    for (i = 0; i < num_data; i++) {
        xdata[i] = ydata[i];
    }
    FREE(ydata);
}

/********************************************************************************
 * xgr_output_curve
 *
*********************************************************************************
 */
void
xgr_output_curve(FILE *fp, int *data, int shift, int beg, int end, int color)
{
    int j;

    fprintf(fp, "\ncolor = %d\n", color);
    for (j=beg; j<end; j++) {
#if 0
        if (j-shift < 0)
            fprintf(stderr, "j=%d shift=%d j-shift = %d < 0!!!\n", j, shift, j-shift);
#endif
        fprintf(fp, "%d %d\n", j+shift, data[j]);
    }
    fprintf(fp, "next\n");
}

/*******************************************************************************
 * Function: fil_er_noise        
 *******************************************************************************
 */
int
explore_filtering_noise(int num_datapoints, int *data, char base,
    Options options, BtkMessage *message)
{
    FILE *fp=NULL;
    int i;
    int *new_data;
    char filename[MAXPATHLEN];

    if (options.xgr) {
        sprintf(filename, "chromat_%c.xgr", base);
        fp = fopen(filename, "w");
        fprintf(fp, "Title = Chromatogram for base =%c\n", base);
        fprintf(fp, "title_x = Scan #\n");
        fprintf(fp, "title_y = Florescence Intensity\n");

        /* Original data */
        xgr_output_curve(fp, data, 0, 0, num_datapoints, 1);        /* white */
    }

    /* Data after fir_filter0 */
    new_data = CALLOC(int, num_datapoints);
    for (i=0; i<num_datapoints; i++) {
        new_data[i] = data[i];
    }
    fgauss(new_data, num_datapoints);
    
    if (options.xgr) 
        xgr_output_curve(fp, new_data, 0, 0, num_datapoints, 2);           /* red */

    /* Data after fir_filter1 */
    for (i=0; i<num_datapoints; i++) {
        new_data[i] = data[i];
    }
    lowpass(new_data, num_datapoints);
    if (options.xgr)
        xgr_output_curve(fp, new_data, 0, 0, num_datapoints, 7);           /* yellow */

    /* Data after fir_filter2 */
    for (i=0; i<num_datapoints; i++) {
        new_data[i] = data[i];
    }
    bandpass(new_data, num_datapoints);
    if (options.xgr)
        xgr_output_curve(fp, new_data, 0, 0, num_datapoints, 4);          /* green */

    /* Data after savgol255 filter */
    for (i=0; i<num_datapoints; i++) {
        new_data[i] = data[i];
    }
    savgol255_filter(  new_data, num_datapoints, 1);
    if (options.xgr)
        xgr_output_curve(fp, new_data, 0, 0, num_datapoints, 9);          /* cyan */

    /* Data after savgol41616 filter */
    for (i=0; i<num_datapoints; i++) {
        new_data[i] = data[i];
    }
    savgol41616_filter(new_data, num_datapoints, 1);
    if (options.xgr)
        xgr_output_curve(fp, new_data, 0, 0, num_datapoints, 3);          /* blue */

    /* Data after iir filter */
    for (i=0; i<num_datapoints; i++) {
        new_data[i] = data[i];
    }
    median_filter_n5(new_data, num_datapoints);
    savgol255_filter(  new_data, num_datapoints, 1);
    if (options.xgr)
        xgr_output_curve(fp, new_data, 0, 0, num_datapoints, 5);          /* violet */

    FREE(new_data);
    if (options.xgr)
        fclose(fp);

    return SUCCESS;
}

/*****************************************************************************
 * Function: create_DP_list
 * Purpose:  Create an array containing pointers to dominant peaks
 *****************************************************************************/
static int
create_DP_list(Data *data, int beg_pos, int end_pos, int *shift, 
    Peak ***dp, int *num_DPs, Options *options, BtkMessage *message)
{
    int    i; 
    int    max_shift=NINF;
    int    n_peak    = data->peak_list_len;
    int    prev_DP_pos = -1;
    Peak **peak_list = data->peak_list;
 
   *dp = CALLOC(Peak *, n_peak);
    MEM_ERROR(dp);

    for (i=0; i<NUM_COLORS; i++) {
        if (max_shift < abs(shift[i]))
            max_shift = abs(shift[i]);
    }

   *num_DPs = 0;
    for ( i=0; i < n_peak; i++ ) {
        int color = peak_list[i]->color_index;
        
        if ((peak_list[i]->ipos + shift[color] <  beg_pos + max_shift) || 
            (peak_list[i]->ipos + shift[color] >= end_pos - max_shift))
            continue;
 
        if (is_dp(peak_list[i], shift, data) &&
            (prev_DP_pos != peak_list[i]->ipos + shift[color]))
        {
          (*dp)[*num_DPs] = peak_list[i];
          (*num_DPs)++;
            prev_DP_pos = peak_list[i]->ipos + shift[color];
        }
    }

// *dp = REALLOC(*dp, Peak *, *num_DPs);
//  MEM_ERROR(dp);

    return SUCCESS;

 error:
    FREE(*dp);
    return ERROR;
}

/*****************************************************************************
 * Function: bubbleSortPeakPos
 * Purpose: Sort an array of peaks by pointers
 *****************************************************************************/
void bubbleSortPeakPos(Peak *peak[], int n_peak, int *shift,
    int win_beg, int win_end)

   /**********************************************************************
    * Sort an array of peak pointers according to increasing positions of
    * the peaks.
    *
    * Inputs:
    *   peak            array of pointers to peaks
    *   n_peak          length of array of peak pointers
    *
    * Outputs:  none
    * Return:   void
    * Comments: Uses bubble sort algorithm, which is good if the peaks are
    *   already close to sorted.  Useful after applying mobility shift
    *   corrections where neighboring peaks might be transposed.
    */

{
    Peak *temp;
    int   i, swap = 1;
    int color0, color1;

    while (swap) {
        swap = 0;
        for ( i=0; i < n_peak-1; i++ )
        {
            color0 = peak[i]->color_index;
            if ((peak[i]->ipos + shift[color0] < win_beg) || 
                (peak[i]->ipos + shift[color0] > win_end))
                continue;

            color1 = peak[i+1]->color_index;
            if ((peak[i+1]->ipos + shift[color1] < win_beg) || 
                (peak[i+1]->ipos + shift[color1] > win_end))
                continue;

            if (peak[i]->ipos + shift[color0] > peak[i+1]->ipos + shift[color1])
            {
                swap = 1;
                temp = peak[i];
                peak[i  ] = peak[i+1];
                peak[i+1] = temp;
            }
        }
    }
}


/*****************************************************************************
 * Function: output_histogram
 */
static void
output_histogram(char *histname, char *xname, char *yname, float *hist,
   int hist_len, int step, int color, int ind_win)
{
    int   j;
    float x0, x1;
    FILE *fp;

    fp = fopen(histname, "w");
    fprintf(fp, "Title = Histogram for window %d\n", ind_win);
    fprintf(fp, "title_x = %s\n", xname);
    fprintf(fp, "title_y = %s\n", yname);
    fprintf(fp, "\ncolor = %d\n", color);
    for (j=0; j<hist_len; j += step) {
        if (j==0)
            x0 = 0;
        else 
            x0 = (float)j - 0.5*(float)step;
        x1 = (float)j + 0.5*(float)step;
        fprintf(fp, "%f %f\n", x0, hist[j]);
        fprintf(fp, "%f %f\n", x1, hist[j]);
    }
    fprintf(fp, "next\n");
    fclose(fp);
}

/*****************************************************************************
 * Function: get_bin   
 *****************************************************************************
 */
int 
get_height_bin(int noise_thresh, int step, int height)
{
     int bin, delta1, delta2;

     if (height < noise_thresh) 
         return 0;
     height  -= noise_thresh;
     if (height < step)
         return 1;
     delta1 = height % step;
     delta2 = step - delta1;
     bin = (delta1 < delta2) ? 
           (height - delta1)/step + 1: 
           (height + delta2)/step + 1;

     return bin;
}

/*****************************************************************************
 * Function: estimate_spacing_variation_of_shifted_peaks
 * Purpose:  Return the estimated spacing at a given scan
 * Estimates the spacing of a list of peaks after applying shifts to ipos.
 *
 * Inputs:
 *      dp[]            Array of pointers to peaks
 *      num_DPs         Length of peaks[]
 *      shift[]         Array of shifts, indexed by color
 *      ignoreSmallest  Smallest fraction of spacings to ignore
 *                      (e.g., ignore_smallest = 0.03 means ignore the
 *                      smallest 3% of the spacings)
 *
 * Outputs:
 *      spacing         Estimate of central tendency of shifted peak spacings
 *      spacing_var     Uncertainty of spacing estimate; set to NULL to
 *                      skip this calculation.
 *
 * Return:              Error flag
 * Comments:
 */
static int
estimate_spacing_variation_of_shifted_peaks(Peak *dp[], int num_DPs,
    int win_beg, int win_end, int shift[NUM_COLORS], float ignoreSmallest,
    float *mean_spacing, float *spacing_var, float *std_dev, int ind_win,
    int *min_spacing, int   *max_spacing, int *min_pos, int *max_pos,
    int   *min_color0,  int *min_color1, Data * data, Options *options,
    BtkMessage *message)
{
    int    hist_spacings_len = INT_DBL(5.0 * crude_spacing_estimate);

    /* Tolerance for ratio of maximum weight to total weight */
    int      i, spacing, color;
    float    weight, tot_wgt;
    double   u, v, numer, numer_sq, denom; /* utility variables */
    float   *hist_spacings = CALLOC(float, hist_spacings_len);

#if 0
        fprintf(stderr, "hist_spacings_len=%d\n", hist_spacings_len);
#endif
    MEM_ERROR(hist_spacings);

    if (DEBUG0) {
        for ( i=0; i < num_DPs; i++ ) {
            color=dp[i]->color_index;
            fprintf(stderr, "In est_spa... dip[%d] ipos=%f iheight=%f shift=%d\n",
            i, dp[i]->ipos, dp[i]->iheight, shift[color]);
        }
    }

    /* Build a histogram of peak spacings */
   *min_spacing = INF;
   *max_spacing = NINF;

    tot_wgt = 0.;
    for (i=0; i < num_DPs; i++)
    {
        int i0, i1;
        int pos0 = -1, pos1 = -1;
        int color0, color1;
        double ave_width20, ave_width21;
        ColorData *cd0, *cd1;

        i0 = (i < num_DPs-1) ?  i    : (i-1);
        i1 = (i < num_DPs-1) ? (i+1) :  i   ;
        color1 = dp[i1]->color_index;
        color0 = dp[i0]->color_index;
        cd0 = &data->color_data[color0];
        cd1 = &data->color_data[color1];

        pos1= dp[i1]->ipos + shift[color1];
        pos0= dp[i0]->ipos + shift[color0];

        if ((pos0 < win_beg) || (pos1 < win_beg) ||
            (pos0 > win_end) || (pos1 > win_end))
            continue;

        weight = 1.;
        ave_width20=dp[i0]->ave_width2;
        ave_width21=dp[i1]->ave_width2;

        if (is_dye_blob(dp[i0]->ipos, dp[i0], data, 0) ||
            is_dye_blob(dp[i1]->ipos, dp[i1], data, 0))
            continue;

        if ((dp[i0]->width2 > ave_width20 * 1.5) ||
            (dp[i1]->width2 > ave_width21 * 1.5))
            continue;

        if (dp[i0]->is_truncated || dp[i1]->is_truncated)
            continue;

        weight *= dp[i1]->height * dp[i0]->height;
        if (color1 == color0)
             weight *= 5;
        spacing =  dp[i1]->ipos + shift[color1]
                 - dp[i0]->ipos - shift[color0];
        spacing = QVMIN(spacing, hist_spacings_len-1);
        spacing = QVMAX(spacing, 0);
        hist_spacings[spacing] += weight;
        tot_wgt += weight;
        if ((*min_spacing > spacing)
            &&
            (color0 != color1) /* ignore small "features" on peak */
                                    )
        {
            *min_spacing = spacing;
            *min_pos     = pos0;
            *max_pos     = pos1;
            *min_color0  = color0;
            *min_color1  = color1;
        }
        if (*max_spacing < spacing) {
            *max_spacing = spacing;
        }
    }

    if (options->xgr && OUTPUT_HISTOGRAMS && (ind_win >= 0)) {
        char histname[100];
        if (ind_win < 10)
            sprintf(histname, "hist_spacings_0%d.xgr", ind_win);
        else
            sprintf(histname, "hist_spacings_%d.xgr", ind_win);
        output_histogram(histname, "Spacing", "Weight",
            hist_spacings, hist_spacings_len, 1, 7, ind_win);
    }

    if (DEBUG0) {
        fprintf(stderr, "Initial tot_wgt=%lf\n" , tot_wgt);
    }

    /* Find mean spacing */
    numer = 0.;
    denom = 0.;
    numer_sq = 0.;
    for (spacing = 0; spacing < hist_spacings_len; spacing++) {
        numer += hist_spacings[spacing] * (double)spacing;
        denom += hist_spacings[spacing];
        numer_sq += hist_spacings[spacing] * (double)spacing * (double)spacing;
    }
   *mean_spacing = numer    / denom;
   *std_dev      = numer_sq / denom - *mean_spacing;

    if (DEBUG0) {
        fprintf(stderr, "tot_wgt=%f u=%f v=%f numer1=%f denom=%f\n", tot_wgt,
        u, v, numer, denom);
    }

    /* Find the spacing variation */
   *spacing_var = *max_spacing - *min_spacing;

    if (DEBUG0) {
        fprintf(stderr, "mean spacing = %f numer2=%f spacing var = %f\n",
           *mean_spacing, numer, *spacing_var);
    }
    FREE(hist_spacings);
    return SUCCESS;

 error:
    FREE(hist_spacings);
    return ERROR;
}


/*****************************************************************************
 * Function: mobility_shift_curve
 * Purpose:  Returns the shift  at specified scan according to polynomial
 *           model specified by mobs_model_coeff[]
 */
float
mobility_shift_curve(int scan, float *a, int poly_degree)
{
    int i;
    float shift = 0;
    float *powers = CALLOC(float, poly_degree);

    fpoly(scan, powers-1, poly_degree);   // compute powers of scan; put in array p
    for (i=0; i < poly_degree; i++)
         shift += a[i] * pow(scan, i);

    FREE(powers);
    return shift;
}


/*****************************************************************************
 * Function: mobility_shift_curve2
 * Purpose:  Returns the shift  at specified scan according to piecewise
 *           linear interplation between windows
 */
float
mobility_shift_curve2(int scan, float *x, float *y, int num_points)
{
    int i;
    float shift = 0;

    for (i=0; i < num_points-1; i++)
    {
        if (((float)scan >= x[i]-1.e-9) && ((float)scan < x[i+1]))
        {
            shift += y[i] + ((float)scan-x[i])*(y[i+1]-y[i])/(x[i+1]-x[i]);
        }
    }
    return shift;
}

/*****************************************************************************
 * Function: output_mobility_curve
 *****************************************************************************
 */
static void
output_mobility_curve(float *x, float *y, int num_points,
    float *model_coeff, int degree, int color)
{
    int    i, j;
    char   filename[MAXPATHLEN];
    float *powers;
    FILE  *fp;

    powers = CALLOC(float, degree);

    sprintf(filename, "Mob_shift_curve_%d.xgr", color);
    fp = fopen(filename, "w");
    fprintf(fp, "Title = Mobility shift curve for trace %d\n", color);
    fprintf(fp, "title_x = Window #\n");
    fprintf(fp, "title_y = Mobility Shift\n");

    /* Output computed shifts for each window */
    fprintf(fp, "\ncolor = %d\n", 7);       /* yellow */
    for (i=0; i<num_points; i++) {
        fprintf(fp, "%f %f\n", x[i], y[i]);
    }
    fprintf(fp, "next\n");

     /* Output polynomial interpolation of the shifts */
    fprintf(fp, "\ncolor = %d\n", 4);      /* green */
#if 0
    fprintf(stderr, "Model coeff =");
    for (j=0; j<degree-1; j++) {
        fprintf(stderr, "%f ", model_coeff[j]);
    }
#endif
    for (i=0; i<num_points; i++) {
        if (POLY_MOB_SHIFT_APPROX) {
            y[i] = 0.;
            for (j=0; j<degree; j++) {
                y[i] += model_coeff[j] * pow(x[i], j);
            }
            y[i] = mobility_shift_curve(x[i], model_coeff, degree);
        }
        else
            y[i] = mobility_shift_curve2((int)x[i], x, y, num_points);
        fprintf(fp, "%f %f\n", x[i], y[i]);
    }
    fprintf(fp, "next\n");

    fclose(fp);
    FREE(powers);
}


/*****************************************************************************
 * Function: output_analyzed_data
 */
void
output_analyzed_data(char *filename, char *title,
    int *data0,    int *data1,    int *data2, int *data3,
    int  data_beg, int  data_end, int *shift, int  ind_win, Data *data)
{
    int   i;
    int   max_shift=NINF;
    FILE *fp;

#if 0
    fprintf(stderr, "File=%s shift1=%d\n", filename, shift[1]);
#endif
    for (i=0; i<NUM_COLORS; i++) {
        if (max_shift < abs(shift[i]))
            max_shift = abs(shift[i]);
    }

    fp = fopen(filename, "w");
    if (ind_win >= 0)
        if (max_shift==0)
            fprintf(fp, "Title = %s %d\n", title, ind_win);
        else
            fprintf(fp,
                "Title = Shifted analyzed data for window %d\n", ind_win);
    else
        fprintf(fp, "Title = %s\n", title);
    fprintf(fp, "title_x = Scan #\n");
    fprintf(fp, "title_y = Florescence Intensity\n");

#if 0
    if ((data_beg+max_shift< 0) || (data_end-max_shift< 0))
            fprintf(stderr, "data_beg=%d shift=%d data_end= %d\n", data_beg, max_shift, data_end);
#endif
    xgr_output_curve(fp, data0, shift[0], 
        QVMAX(data_beg+max_shift, data->pos_data_beg),
        QVMIN(data_end-max_shift, data->pos_data_end), 4);           // green
    xgr_output_curve(fp, data1, shift[1], 
        QVMAX(data_beg+max_shift, data->pos_data_beg),
        QVMIN(data_end-max_shift, data->pos_data_end), 3);           // blue
    xgr_output_curve(fp, data2, shift[2], 
        QVMAX(data_beg+max_shift, data->pos_data_beg),
        QVMIN(data_end-max_shift, data->pos_data_end), 7);           // yellow
    xgr_output_curve(fp, data3, shift[3], 
        QVMAX(data_beg+max_shift, data->pos_data_beg),
        QVMIN(data_end-max_shift, data->pos_data_end), 2);           // red
    fclose(fp);
}

static void
get_abi_spacing_curve(float *x, float *y, int num_points,
    int win_size, Data *data)
{
    int i, j;

    for (i=0; i<num_points; i++) {
        int num_bases =  0, first_pos = -1, last_pos  = -1;
        int win_beg = x[i] - win_size/2;
        int win_end = x[i] + win_size/2;

        if (win_beg < 0)
            win_beg = 0;
        if (win_end > data->length-1)
            win_end = data->length-1;

        for (j=0; j<data->bases.length; j++) {
            int coord = data->bases.coordinate[j];

            if ((first_pos <  0) && (coord < win_beg))
                continue;
            if ((first_pos <  0) && (coord >= win_beg))
                first_pos = coord;
            if ((first_pos >= 0) && (last_pos < 0))
                num_bases++;
            if ((first_pos >= 0) && (coord > win_end)) {
                last_pos  = data->bases.coordinate[j-1];
                break;
            }
#if 0
            fprintf(stderr, 
                "In get_abi_spacing_curve: j=%d coord=%d win_beg=%d win_end=%d num_bases=%d\n",
                j, coord, win_beg, win_end, num_bases);
#endif
        }            
        y[i] = (float)(last_pos - first_pos) / (float)(num_bases - 1);
#if 0
            fprintf(stderr, 
                "    Window=%d ABI_spacing = %f last_pos=%d first_pos=%d num_bases=%d\n\n", 
                i, y[i], last_pos, first_pos, num_bases);
#endif

    }

    return;
}

/*****************************************************************************
 * Function: sliding_window_avarage
 *****************************************************************************
 */
static void
sliding_window5_average(float *y, int num_data)
{
    int i;
    float *z = CALLOC(float, num_data);

    z[0] = y[0];
    z[num_data-1] = y[num_data-1];
    if (z[num_data-1] <= 0)
        z[num_data-1] =  y[num_data-2];
    for (i=1; i<num_data-1; i++) {
        if ((i>1) && (i<num_data-2))
            z[i] = (y[i-2] + y[i-1] + y[i] + y[i+1] +  y[i+2])/5.;
        else
            z[i] = (y[i-1] + y[i] + y[i+1])/3.;
    }
    for (i=0; i<num_data; i++) {
        y[i] = z[i];
    }
    FREE(z);
}

/****************************************************************************e
 * Function: spacing_curve
 * Purpose:  Returns the spacing at specified scan according to polynomial
 *           model specified by spac_model_coeff[]
 *
 * Inputs:
 *   scan        integer data scan
 *
 * Outputs:      none
 * Return:       spacing estimate
 * Comments:
 */
double
spacing_curve(int scan)
{
    int   i;
    float powers[POLYFIT_DEGREE + 1];
    double r = -1;

    if (POLY_SPAC_MODEL_APPROX > 0) {
        fpoly(scan, powers-1, POLYFIT_DEGREE + 1);    /* offset for NR */
        for ( i=0; i < POLYFIT_DEGREE + 1; i++)
            r += spac_model_coeff[i] * powers[i];
    }
    else
    {
#if 0
        fprintf(stderr, 
            "In spacing_curve: scan=%d num_windows=%d spac_mod_pos0=%f spac_mod_pos_last=%f\n",
            scan, num_windows, spac_mod_pos[0], spac_mod_pos[num_windows - 1]);
#endif
        if (scan <= spac_mod_pos[0])
            r = spac_mod_val[0];
        else if (scan >= spac_mod_pos[num_windows - 1])
            r = spac_mod_val[num_windows - 1];
        else 
        {
            for (i=0; i< num_windows - 1; i++)
            {
                if (((float)scan >= spac_mod_pos[i  ]) &&
                   ((float)scan <  spac_mod_pos[i+1]))
                {
                   r = spac_mod_val[i] +
                      (spac_mod_val[i+1] - spac_mod_val[i]) *
                      ((float)scan       - spac_mod_pos[i]) /
                      (spac_mod_pos[i+1] - spac_mod_pos[i]);
                }
            }
        }
    } 
    return(r);
}

/*****************************************************************************
 * Function: output_spacing_curve
 *****************************************************************************
 */
static void
output_spacing_curve(float *x, float *y, int num_points, int degree, 
    int win_size, Data *data)
{
    int   i, j;
    char filename[MAXPATHLEN];
    float *powers, *z;
    FILE *fp;

    powers = CALLOC(float, degree);
    z = CALLOC(float, num_points);

    sprintf(filename, "Spacing_curve.xgr");
    fp = fopen(filename, "w");
    fprintf(fp, "Title = spacing curve\n");
    fprintf(fp, "title_x = Window #\n");
    fprintf(fp, "title_y = Spacing Curve\n");

    /* Output computed spacings for each window */
    fprintf(fp, "\ncolor = %d\n", 7);       /* yellow */
    for (i=0; i<num_points; i++) {
        fprintf(fp, "%d %f\n", i, y[i]);
    }
    fprintf(fp, "next\n");

     /* Output polynomial interpolation of spacings */
    fprintf(fp, "\ncolor = %d\n", 4);      /* green */
    for (i=0; i<num_points; i++) {
        z[i] = 0.;
        for (j=0; j<degree; j++) {
            z[i] += spac_model_coeff[j] * pow(x[i], j);
        }
        z[i] = spacing_curve(x[i]);
        if (z[i] >0) 
            fprintf(fp, "%d %f\n", i, z[i]);
    }
    fprintf(fp, "next\n");

    /* Output sliding window avarage approximation of spacing curve */
    fprintf(fp, "\ncolor = %d\n", 5);      /* violet */
    for (i=0; i<num_points; i++) {
        z[i] = spacing_curve(x[i]);
    }
    (void)sliding_window5_average(z, num_points);
    for (i=0; i<num_points; i++) {
        fprintf(fp, "%d %f\n", i, z[i]);
    }
    fprintf(fp, "next\n");

    /* Output ABI spacing curve */
    get_abi_spacing_curve(x, z, num_points,
        win_size, data);
    fprintf(fp, "\ncolor = %d\n", 9);      /* cyan */
    for (i=0; i<num_points; i++) {
        float v=-1;
        if (i==0)
            if (z[0]>0) 
                v = z[0];
            else 
                v = z[1];
        else if (i==num_points-1)
            if (z[num_points-1]>0)
                v = z[num_points-1];
            else 
                v = z[num_points-2];
        else {
            if ((z[i-1]>0) && (z[i]>0) && (z[i+1]>0)) 
                v = (z[i-1] + 2.*z[i] + z[i+1])/4.;
            else if ((z[i]>0) && (z[i+1]<0))
                v = z[i];
            else if ((z[i]>0) && (z[i-1]<0))
                v = z[i];
            else if ((z[i]<0) && (z[i-1]<0))
                v = z[i+1];
            else if ((z[i]<0) && (z[i+1]<0))
                v = z[i-1]; 
        }
        if (v>0)
            fprintf(fp, "%d %f\n", i, v);
    }
    fprintf(fp, "next\n");
    
    fclose(fp);
    FREE(powers);
    FREE(z);
}


/*****************************************************************************
 * Function: output_new_spacing_curve
 *****************************************************************************
 */
static void
output_new_spacing_curve(float *x, float *y, int num_points)
{
    int   i;
    char filename[MAXPATHLEN];
    FILE *fp;

    sprintf(filename, "Spacing_curve.xgr");
    fp = fopen(filename, "a");

    /* Output computed spacings for each window */
    fprintf(fp, "\ncolor = %d\n", 2);       /* red */
    for (i=0; i<num_points; i++) {
        fprintf(fp, "%d %f\n", i, y[i]);
    }
    fprintf(fp, "next\n");
    fclose(fp);
}

/****************************************************************************e
 * Function: normalization_curve
 * Purpose:  Returns the spacing at specified scan according to piecewise
 *           model specified by spac_model_coeff[]
 *
 * Inputs:
 *   scan        integer data scan
 *
 * Outputs:      none
 * Return:       spacing estimate
 * Comments:
 */
double
normalization_curve(int scan, int color)
{
    int   i;
    double r = -1;

    if (scan <= norm_mod_pos[0])
        r = norm_mod_val[color][0];
    else if (scan >= norm_mod_pos[num_windows - 1])
        r = norm_mod_val[color][num_windows - 1];
    else
    {
        for (i=0; i< num_windows - 1; i++)
        {
            if (((float)scan >= norm_mod_pos[i  ]) &&
               ((float)scan <  norm_mod_pos[i+1]))
            {
               r = norm_mod_val[color][i] +
                  (norm_mod_val[color][i+1] - norm_mod_val[color][i]) *
                  ((float)scan       - norm_mod_pos[i]) /
                  (norm_mod_pos[i+1] - norm_mod_pos[i]);
            }
        }
    }
    return(r);
}


/*****************************************************************************
 * Function: find_optimal_shift_in_a_window
 * Purpose:  Find uniform mobility shifts that minimize total penatly
 *           for non-uniform of peak spacing
 *
 * Inputs:   peak_list	List of pointers to peaks
 *	     num_peaks	Length of peak_list
 *	     shift		Array of shift references
 *	     shiftInc	Ratio of shift increment to average peak
 *			spacing; if <= 0, default value of 0.5 is
 *			used.
 *	     shiftMax	Ratio of maximum shift (relative to reference
 *			specified by shift) to average peak spacing;
 *			if <= 0, default value of 1.5 is used.
 *	     shift_flag	Array of flags; if shift_flag[j] == 0,
 *			channel i is not shifted from reference
 
 * Outputs:  shift		Array of resulting shifts
 *	     spacing	Estimate of "average" spacing of peaks
 *	     spac_var	Estimate of error of spacing estimate
 *	     shift_err	Estimate of uncertainty of returned shift
 * Return:  (void)
 * Comments: Complexity is proportional to (shift_max/shift_inc)^3.
 *		  Histogram technique assumes that ipos is integer.
 */
int  
find_optimal_shift_in_a_window(Data *data, int ind_win, int win_beg, int win_end, 
    Peak *dp[], int num_DPs, int shift[NUM_COLORS], float shiftInc, float shiftMax, 
    int shift_flag[NUM_COLORS], float *best_mean_spacing, 
    float *best_spacing_var, int *best_min_spa, int *best_max_spa,
    int *best_min_pos, int *best_max_pos, float *shift_err, 
    int *best_min_color0, int *best_min_color1, Options *options, BtkMessage *message)
{
    const double shiftIncDefault		= 0.5;
    const double shiftMaxDefault		= 1.5;
    const double minRelatChannelWgt     = MIN_RELATIVE_CHANNEL_WEIGHT;
                 /* minimum total color height as fraction of total height */
    const int	 hist_spacings_len = INT_DBL(2.5*crude_spacing_estimate);
		 /* length of hist_spacings; maximum spacing in histogram + 1 */
    const double minErr = 0.2;

    int	         i, color, i1, i2, i3, i4;
    int	         shift_inc, shift_max, min_pos, max_pos, min_spacing, max_spacing;
    int	         best_shift[NUM_COLORS] = {0,0,0,0};
    int          shift_flag2[NUM_COLORS];
    int          init_shift[NUM_COLORS] = {0,0,0,0};
    float        tot_wgt, mean_spacing; 
    float        spacing_var, min_rel_spacing_err, best_std_dev=INF, std_dev=INF;
    int          wgt[NUM_COLORS] = {0, 0, 0, 0};
    float       *hist_spacings = CALLOC(float, hist_spacings_len);	
                 /* histogram of weighted polychromatic spacings */

    MEM_ERROR(hist_spacings);

    if ( shiftInc <= 0 ) 
        shiftInc = shiftIncDefault;
    if ( shiftMax <= 0 ) 
        shiftMax = shiftMaxDefault;

    /* Set absolute parameters */
    shift_inc = qv_round(shiftInc * crude_spacing_estimate);
    if ( shift_inc < 1 ) 
        shift_inc = 1;
    shift_max = qv_round(shiftMax * crude_spacing_estimate);

    /* Check for a good signal in each channel */
    tot_wgt = 0;
  
    for (color=0; color < NUM_COLORS; color++) 
        init_shift[color] = shift[color];
 
    for ( i=0; i < num_DPs; i++ ) 
    {

        if ((dp[i]->ipos < win_beg) || (dp[i]->ipos > win_end))
            continue;

        tot_wgt += dp[i]->iheight;
        wgt[dp[i]->color_index] += INT_DBL(dp[i]->iheight);
    }

    /* Make sure the reference channel has peaks (=signal weight is high enough)
     * shift_flag2 is the same as shift_flag, except that
     * nonzero components with no channel peaks will be set to 0
     */
    for (color=0; color < NUM_COLORS; color++) {
        if (DBL_LT_DBL(wgt[color], minRelatChannelWgt * tot_wgt) &&
            shift_flag[color])
            shift_flag2[color] = 0;
        else
            shift_flag2[color] = shift_flag[color];
    }

#if 0
    shift_max =  32;
    fprintf(stderr, "shift_max=%d shift_flag2[i]= %d %d %d %d \n", 
        shift_max, shift_flag2[0], shift_flag2[1], shift_flag2[2],
        shift_flag2[3]);
#endif
    /* Start a big loop */
    while ((shift_max >= 1) && (shift_inc > 0))
    {
        int min_color0, min_color1;
#if 0
        fprintf (stderr, "      shift_max=%d shift_inc=%d\n", shift_max, shift_inc);
#endif

	/* Find the optimal shifts 
	 * (at least one of these loops should be trivial)
         */
        for (i1=0; i1 <= shift_flag2[0]*shift_max/shift_inc; i1++)
        {
            int i5 = (i1+1)/2;
            shift[0]=shift_inc*pow(-1, i1)*i5; 
            for (i2=0; i2 <= shift_flag2[1]*shift_max/shift_inc; i2++)
            {
                i5 = (i2+1)/2;
                shift[1]=shift_inc*pow(-1, i2)*i5;
	        for (i3=0; i3 <= shift_flag2[2]*shift_max/shift_inc; i3++)    
                {
                    i5 = (i3+1)/2;
                    shift[2]=shift_inc*pow(-1, i3)*i5;
	            for (i4=0; i4 <= shift_flag2[3]*shift_max/shift_inc; i4++)
	            {
                        i5 = (i4+1)/2;
                        shift[3]=shift_inc*pow(-1, i4)*i5;
                        int final_shift[NUM_COLORS];
                        for ( color=0; color < NUM_COLORS; color++ )
                            final_shift[color] = shift[color]+init_shift[color];

                        /* Create a list of all (shifted) peaks and a list of DPs */
                        bubbleSortPeakPos(dp, num_DPs, final_shift, win_beg, win_end);

#if 0
                       if (num_DPs == 0) {
                           fprintf(stderr, "num_DPs=%d win_beg=%d win_end=%d final_shift=%d %d %d %d\n\n",
                               num_DPs, win_beg, win_end, final_shift[0], final_shift[1], 
                               final_shift[2], final_shift[3]);
                       }
#endif
                        if (num_DPs <= 1) {
                            continue;
                        }

                        if (estimate_spacing_variation_of_shifted_peaks(dp, num_DPs,
                            win_beg, win_end, final_shift, 0., &mean_spacing, &spacing_var, &std_dev, 
                            ind_win, &min_spacing, &max_spacing, &min_pos, &max_pos, 
                            &min_color0, &min_color1, data, options, message) 
                            != SUCCESS)
                            goto error;  
#if 0
          fprintf(stderr, 
              "    window=%d shift=%3d %3d %3d %3d spac_var=%2d ", ind_win, 
              final_shift[0], final_shift[1], final_shift[2], final_shift[3],
               (int)spacing_var);
          fprintf(stderr,
              "best_spac_var=%d best_shift=%3d %3d %3d %3d\n",
               (int)(*best_spacing_var), best_shift[0], best_shift[1], 
                best_shift[2], best_shift[3]);
          fprintf(stderr,
              "        std_dev=%f best_std_dev=%f \n",  std_dev, best_std_dev);
#endif 
                        if ((mean_spacing > DBL_EPSILON) &&
                            (mean_spacing > crude_spacing_estimate-1.) &&
                            (mean_spacing < crude_spacing_estimate+1.) &&
                            ( (spacing_var  < *best_spacing_var) 
                              ||
                             ((spacing_var == *best_spacing_var) &&
                              (std_dev < best_std_dev))
                                                       ))
                        {
                           *best_mean_spacing = mean_spacing;
                           *best_spacing_var  = spacing_var;
                            best_std_dev      = std_dev;
                           *best_min_spa      = min_spacing;
                           *best_max_spa      = max_spacing;
                           *best_min_pos      = min_pos;
                           *best_max_pos      = max_pos;
                           *best_min_color0   = min_color0;
                           *best_min_color1   = min_color1;
                            for (color=0; color < NUM_COLORS; color++)
                                best_shift[color] = final_shift[color]; 

                            min_rel_spacing_err = *best_spacing_var /
                                                  *best_mean_spacing;
                            if (min_rel_spacing_err < minErr) {
                                break;
                            }
                        }
                    }
                }
            }
        }
        shift_max = shift_inc;
        shift_inc /= 2 ;
        for (color=0; color<NUM_COLORS; color++) 
            init_shift[color] = best_shift[color];
    } /* while */
#if 0
    if (ind_win == 7) {
        best_shift[0] = 0;
        best_shift[1] = 0;
        best_shift[2] = 4;
        best_shift[3] = 0;
    }
#endif
    for (color=0; color < NUM_COLORS; color++)
        shift[color] = best_shift[color];

    /* ad hoc error measure */
    if (shift_err != NULL) {
        float penalty = *best_spacing_var / *best_mean_spacing;
       *shift_err = *best_mean_spacing * penalty / (1 - penalty + DBL_EPSILON);
        if (*shift_err < minErr )
            *shift_err = minErr;
    }
  
    FREE(hist_spacings); 
    return SUCCESS; 

error:
    FREE(hist_spacings);

    return ERROR;
}

/*****************************************************************************
 * Check whether the mobility shift and associated spacing estimates
 * are trustworthy.
 *
 * Return:      1 if estimates are deemed trustworthy; 0 otherwise.
 * Comments:    Based on set of heuristic rules.
 *****************************************************************************
 */
int areMobEstTrustworthy(int shift[NUM_COLORS], float spacing)
{
    int i;
    for (i=0; i< NUM_COLORS; i++) {
        if (abs(shift[i]) > spacing * SHIFTING_TOLERANCE)
            return 0;
    }

    return 1;
}

/*****************************************************************************
 * Function: prune_called_peaks
 * Purpose:  Uncall peaks that have been shifted beyond ends of traces
 *****************************************************************************/
void prune_called_peaks( Data *data )
{
    int	   i, pos;
    int	   n_old = data->bases.length;
    int	   n_new = 0;
    int	   max_scan = data->length;
    char  *myBases = data->bases.bases;
    Peak **myCPL  = data->bases.called_peak_list;

    for ( i=0; i < n_old; i++ ) {
        pos = myCPL[i]->ipos;
        if ( pos > 0 || pos < max_scan ) {		/* keep peak */
            myBases[n_new] = myBases[i];
            myCPL[n_new++] = myCPL[i];
        }
        else {					/* uncall this peak */
            myCPL[i]->is_called = 0;
        }
    }

    data->bases.length = n_new;
    if ( n_new < n_old/2 ) {
        data->bases.bases = REALLOC( data->bases.bases, char, n_new );
        data->bases.called_peak_list =
	    REALLOC( data->bases.called_peak_list, Peak*, n_new );
    }
}

/*****************************************************************************
 * Function: make_DP_mobility_shifts
 * Purpose:  Construct a mobility shift curve by using spacings of dominant
 *           intrinsic peaks.
 * Inputs:   data	complete data structure
 *	     shift_flag	flags indicating which channels to shift
 *	     options	options structure that includes field
 *			Verbose
 *
 * Outputs:  data	data->peak_list[]->ipos are
 *			updated for mobility shift correction
 * Return:	  Error code
 *****************************************************************************
 */
static int 
make_DP_mobility_shifts(Data *data, int shift_flag[NUM_COLORS], 
    Options *options, BtkMessage *message)
{
    Peak	**peaks = data->peak_list; /* shortcut to peak_list */
    const int	  num_peaks = data->peak_list_len;
    const double  maxShiftMax        = MAX_SHIFT_MAX;
    const double  maxShiftInc        = MAX_SHIFT_INC;
    const double  shiftMaxThresh     = SHIFT_MAX_THRES;
    const double  shiftErrGrowFactor = SHIFT_ERR_GROW_FACTOR;
    const double  alpha = 1.1,	                 /* alpha >= 1 */
                  beta  = 0.5;	                 /* beta  >  0 */

    int	          i, i0, j=-1, color;
    int           win_size;
    int	         *igood=0, ngood;                /* index array and length */
    int           data_beg = data->pos_data_beg, 
                  data_end = data->pos_data_end;
    char         *progc = ".:";	   	         /* characters for progress output */
    int	          num_wins;		       	 /* number of windows */
    int	          shift_buf[NUM_COLORS] = {0, 0, 0, 0};
    int          *shift[NUM_COLORS] = { NULL };  /* initialize as NULL pointers */
    float        *spacing=0, *spac_var=0;
    double        min_var, max_var;
    float        *var=0, std_dev;	                         /* utility float */
    double        shift_inc, shift_max;
    double        s;
    float         powers[POLYFIT_DEGREE+1];
    float        *x=0, *y[NUM_COLORS]={0, 0, 0, 0}, *shift_err=0;
    float         a[NUM_COLORS][POLYFIT_DEGREE+1] = {{0}};
    Peak        **dp = NULL;
    int	          num_DPs = 0, min_pos, max_pos, min_spacing, max_spacing; 
    int           best_min_pos, best_max_pos, best_min_spa, best_max_spa;
    int           min_color0, min_color1, best_min_color0, best_min_color1;
    int           win_beg, win_end;
    FILE         *fp;

    num_wins = DEFAULT_NUM_WINDOWS;
    win_size = 2 * (data_end - data_beg) / num_wins;
    if (win_size < MIN_WIN_SIZE) {
        win_size = MIN_WIN_SIZE;
        num_wins = 2 * (data_end - data_beg) / win_size;
    }

    for ( color=0; color < NUM_COLORS; color++ ) {
        shift[color] = CALLOC(int, num_wins);
    }
    shift_err        = CALLOC(float, num_wins);
    spacing          = CALLOC(float, num_wins);
    spac_var         = CALLOC(float, num_wins);
    var = CALLOC(float, num_wins);
    MEM_ERROR(var);

    if (create_DP_list(data, data->pos_data_beg, data->pos_data_end, shift_buf,
        &dp, &num_DPs, options, message) != SUCCESS )
    {
        goto error;
    }

    if (MONITOR) {
        fprintf(stderr,
            "\nIn make_DP_mobility_shifts: win_size=%d num_wins=%d\n\n",
        win_size, num_wins);
    }

#if DEBUG_CURTIS
    if ( options->Verbose > 2 )
        fprintf(stderr, "Finding mobility shifts, %d windows\n", num_wins);
#endif

    /* Find region of low spacing variation, a.k.a. "sweet spot" */
    for ( i=0; i < num_wins; i++) 
    { 
        win_beg =  data_beg + win_size*i/2;
        win_end =  (win_beg + win_size  < data_end) ?
                   (win_beg + win_size) : data_end;

        estimate_spacing_variation_of_shifted_peaks(dp, num_DPs, win_beg, win_end,
            shift_buf, 0, spacing+i, var+i, &std_dev, -1, &min_spacing, &max_spacing, 
            &min_pos, &max_pos, &min_color0, &min_color1, data, options, 
            message);
#if 0
            fprintf(stderr, "win_ind=%d win_beg=%d win_end=%d num_DPs=%d \n",
                i, win_beg, win_end, num_DPs);
#endif

#if 0
            fprintf(stderr, "    Old spacing=%f min_spacing=%d max_spacing=%d \n",
                spacing[i], min_spacing, max_spacing);
#endif

        if (options->xgr && options->raw_data) {
            char filename[MAXPATHLEN];
            if (i<10)
                sprintf(filename, "analyzed_data_0%d.xgr", i);
            else 
                sprintf(filename, "analyzed_data_%d.xgr", i);
#if 0
            fprintf(stderr, "Before output anal data for win %d shift_buf=%d %d %d %d \n",
                i, shift_buf[0], shift_buf[1], shift_buf[2], shift_buf[3]);
#endif 
            output_analyzed_data(filename, "Analyzed data for window", 
                data->color_data[0].data, data->color_data[1].data,
                data->color_data[2].data, data->color_data[3].data,
                win_beg, win_end, shift_buf, i, data);
        }

        if (MONITOR) {
            fprintf(stderr, 
               "0th call: window=%d spacing=%f spacing_var=%f rel_error=%f\n",
                i, *(spacing+i), var[i], var[i]/spacing[i]);
            fprintf(stderr,
               "   min_spacing=%d max_spacing=%d min_pos=%d max_pos=%d\n", 
               min_spacing, max_spacing, min_pos, max_pos);
        }
        if ( spacing[i] != 0 )
            var[i] /= spacing[i];
        else
            var[i] *= 10;               /* ad hoc handling of spacing = 0 */
    }
   
    /* Find index i0 of the window with minimal spacing variation */ 
    i0 = 0;
    min_var = INF;        
    max_var = 0.;
    for ( i=0; i < num_wins; i++ ) {
        spac_var[i] = var[i];
        if (min_var * ONE_MINUS > spac_var[i]) {
            min_var = spac_var[i];
            i0 = i;
            crude_spacing_estimate = spacing[i0];
        }
        if (max_var < spac_var[i] * ONE_MINUS ) {
            max_var = spac_var[i];
            j = i;
        }
    }
    isweet = i0;

    if (MONITOR > 1) {
        if ( (fp = fopen("tt_rel_spac_var", "w")) != NULL ) {
            for ( i=0; i < num_wins; i++ )
                fprintf(fp, "%f\n", spac_var[i]);
            fclose(fp);
        }
    }

    if (MONITOR > 0) {
        fprintf(stderr, 
            "\nIndex of the window with minimal rel_error %f is %d\n",
            min_var, i0);
        fprintf(stderr,
            "Index of the window with maximal rel_error %f is %d\n\n",
            max_var, j);
    }

    if (DEBUG_CURTIS && options->Verbose > 2) {
        fprintf(stderr, "Start window for mob shifts = %d \n", i0);
    }
    FREE(var);

    /* Estimate mob shifts in windows.  
     * First estimate shifts in region of low spacing variation 
     */

    /* Comment by GD: why do we do this again? Didn't we do it already above? 
     *                Nothing seems to have changed !!!
     */ 
    if (options->shift)
    {
#if 0
        fprintf(stderr, "Shifting ... \n");
#endif
        win_beg = data_beg + win_size* i0   /2;
        win_end = data_beg + win_size*(i0+2)/2;
        
        bubbleSortPeakPos(dp, num_DPs, shift_buf, win_beg, win_end);

        estimate_spacing_variation_of_shifted_peaks(dp, num_DPs, 
           win_beg, win_end, shift_buf, 0,
           spacing+i0, spac_var+i0, &std_dev, i0, &min_spacing, &max_spacing,
           &min_pos, &max_pos, &min_color0, &min_color1, data, options, 
           message);

        if (spacing[i0] == 0) 
            shift_max = maxShiftMax;
        else {
            min_var = RATIO_SHIFT_MAX_REL_SPC_ERR * spac_var[i0]/spacing[i0];
            shift_max = MIN2(min_var, maxShiftMax);
        }

        shift_inc = MIN2(2*shift_max/3, maxShiftInc);

        if (find_optimal_shift_in_a_window(data, i0, win_beg, win_end, 
            dp, num_DPs, shift_buf, 
            shift_inc, shift_max, shift_flag, spacing+i0, spac_var+i0, 
            &best_min_spa, &best_max_spa, &best_min_pos, &best_max_pos, 
            shift_err+i0, &best_min_color0, &best_min_color1, options, message)
            != SUCCESS)
            return ERROR;

        for (color=0; color < NUM_COLORS; color++)
            shift[color][i0] = shift_buf[color];

        if (options->xgr && options->raw_data) {
            char filename[MAXPATHLEN];
            if (i0 < 10)
                sprintf(filename, "analyzed_data_shifted_0%d.xgr", i0);
            else
                sprintf(filename, "analyzed_data_shifted_%d.xgr", i0);
#if 0
            fprintf(stderr, "Before output shifted anal data for win i0 shift_buf=%d %d %d %d \n",
                shift_buf[0], shift_buf[1], shift_buf[2], shift_buf[3]);
#endif
            output_analyzed_data(filename, "Shifted analyzed data for window", 
                data->color_data[0].data, data->color_data[1].data,
                data->color_data[2].data, data->color_data[3].data,
                win_beg, win_end, shift_buf, i0, data);
        }

        if (MONITOR) {
            fprintf(stderr, 
                "1st call: window=%d spacing=%f spacing_var=%f rel_error=%f\n",
                i0, spacing[i0], spac_var[i0], spac_var[i0]/spacing[i0]);
            fprintf(stderr,
                   "   best_min_pos=%d best_max_pos=%d\n", best_min_pos, best_max_pos);
            fprintf(stderr,
                   "   best_min_color0=%d best_min_color1=%d\n",
                   best_min_color0, best_min_color1);
            fprintf(stderr,
                   "   best_min_spa=%d best_max_spa=%d\n", best_min_spa, best_max_spa);
            fprintf(stderr, "1st call: best shift = %d %d %d %d\n\n",
                shift_buf[0], shift_buf[1], shift_buf[2], shift_buf[3]);
        }

    }

    /* work back towards beginning */
    /* Comment by GD: need to make sure that i0 > 0 !!! */
    if (options->shift)
    {
        for (i=i0-1; i >= 0; i--) 
        {
#if 0
        fprintf(stderr, "Shifting ... \n");
#endif
            win_beg = data_beg + win_size* i   /2;
            win_end = data_beg + win_size*(i+2)/2;
            for (j=0; j<NUM_COLORS; j++)
                shift_buf[j] = 0;

            /* Need update the peak list because "shifted" list
             * has been created by find_optimal_shift_in_a_window in i-loop
             */
            bubbleSortPeakPos(dp, num_DPs, shift_buf, win_beg, win_end);

            estimate_spacing_variation_of_shifted_peaks(dp, num_DPs, 
                win_beg, win_end, shift_buf, 0, spacing+i, spac_var+i, 
                &std_dev, i, &min_spacing, &max_spacing,
                &min_pos, &max_pos, &min_color0, &min_color1, data, options, 
                message);

            if ( spacing[i] == 0 ) 
                shift_max = maxShiftMax;
            else {
                min_var = RATIO_SHIFT_MAX_REL_SPC_ERR * spac_var[i]/spacing[i];
                shift_max = MIN2(min_var, maxShiftMax);
            }
            shift_inc = MIN2(2*shift_max/3, maxShiftInc);

            if (DEBUG_CURTIS && options->Verbose > 2 )
                fprintf(stderr, "i = %d shift_max = %f\n",
	            i, shift_max);

            if ( shift_max * spacing[i] * F_ONE_MINUS > shiftMaxThresh ) 
            {
                progc = ".:";
                find_optimal_shift_in_a_window(data, i, win_beg, win_end, 
                     dp, num_DPs, shift_buf, shift_inc, shift_max,
	    	     shift_flag, spacing+i, spac_var+i, &best_min_spa, &best_max_spa,
                     &best_min_pos, &best_max_pos, shift_err+i, 
                     &best_min_color0, &best_min_color1, options, message);
            }
            else {
                progc = ",;";
                shift_err[i] = shiftErrGrowFactor * shift_err[i+1];
            }

            if (options->xgr && options->raw_data) {
                char filename[MAXPATHLEN];
                if (i<10)
                    sprintf(filename, "analyzed_data_shifted_0%d.xgr", i);
                else
                    sprintf(filename, "analyzed_data_shifted_%d.xgr", i);
#if 0
            fprintf(stderr, "Before output shifted anal data for win %d shift_buf=%d %d %d %d \n",
                i, shift_buf[0], shift_buf[1], shift_buf[2], shift_buf[3]);
#endif
                output_analyzed_data(filename, "Shifted analyzed data for window",
                    data->color_data[0].data, data->color_data[1].data,
                    data->color_data[2].data, data->color_data[3].data,
                    win_beg, win_end, shift_buf, i, data);
            }

            if (MONITOR) {
                fprintf(stderr, 
                "2nd call:  window=%d spacing=%f spacing_var=%f rel_error=%f\n",
                    i, spacing[i], spac_var[i], spac_var[i]/spacing[i]);
                fprintf(stderr,
                   "   best_min_pos=%d best_max_pos=%d\n", best_min_pos, best_max_pos);
                fprintf(stderr,
                   "   best_min_spa=%d best_max_spa=%d\n", best_min_spa, best_max_spa);
                fprintf(stderr,
                   "   best_min_color0=%d best_min_color1=%d\n", 
                   best_min_color0, best_min_color1);
                fprintf(stderr, 
                "2nd call: best shift = %d %d %d %d\n\n",
                  shift_buf[0], shift_buf[1], shift_buf[2], shift_buf[3]);
            }


            for ( color=0; color < NUM_COLORS; color++ )
                shift[color][i] = shift_buf[color];

            if (DEBUG_CURTIS && options->Verbose > 2 ) {
                putc((i0-i) % 10 == 0 ? progc[1] : progc[0], stderr);
                fflush(stderr);
            }
        }
    }

#if COMPLETE_MOB_SHIFTS
	/* work forward towards end */

    if (DEBUG_CURTIS && options->Verbose > 2 ) 
        putc('o', stderr); fflush(stderr);
  
    for (color=0; color < NUM_COLORS; color++)
        shift_buf[color] = shift[color][i0];

    /* Comment by GD: need to make sure that i0 < num_wins-1 !!! */
    if (options->shift)
    {
#if 0
        fprintf(stderr, "Shifting ... \n");
#endif
        for (i=i0+1; i < num_wins; i++) {

            win_beg = data_beg + win_size* i   /2;
            win_end = data_beg + win_size*(i+2)/2;
            for (j=0; j<NUM_COLORS; j++)
                shift_buf[j] = 0;

            /* Need update the peak list because "shifted" list
             * has been created by find_optimal_shift_in_a_window in i-loop
             */
            bubbleSortPeakPos(dp, num_DPs, shift_buf, win_beg, win_end);

            estimate_spacing_variation_of_shifted_peaks(dp, num_DPs, 
                win_beg, win_end, shift_buf, 
                0, spacing+i, spac_var+i, &std_dev, i, &min_spacing, &max_spacing,
                &min_pos, &max_pos, &min_color0, &min_color1, data, options, 
                message);

            if ( spacing[i] == 0 ) 
                shift_max = maxShiftMax;
            else {
                min_var = RATIO_SHIFT_MAX_REL_SPC_ERR * spac_var[i]/spacing[i];
                shift_max = MIN2(min_var, maxShiftMax);
            }
            shift_inc = MIN2(2*shift_max/3, maxShiftInc);

            if (DEBUG_CURTIS && options->Verbose > 2 )
                fprintf(stderr, "i = %d    shift_max = %f\n",
    	          i, shift_max);

            if ( shift_max * spacing[i] * F_ONE_MINUS > shiftMaxThresh ) {
                progc = ".:";
                find_optimal_shift_in_a_window(data, i, win_beg, win_end, 
                    dp, num_DPs, shift_buf, shift_inc, shift_max,
	            shift_flag, spacing+i, spac_var+i, &best_min_spa, &best_max_spa,
                    &best_min_pos, &best_max_pos, shift_err+i, 
                    &best_min_color0, &best_min_color1, options, message);
            }
            else {
                progc = ",;";
                shift_err[i] = shiftErrGrowFactor * shift_err[i-1];
            }

            if (options->xgr && options->raw_data) {
                char filename[MAXPATHLEN];
                if (i<10)
                    sprintf(filename, "analyzed_data_shifted_0%d.xgr", i);
                else
                    sprintf(filename, "analyzed_data_shifted_%d.xgr", i);
#if 0
            fprintf(stderr, "Before output shifted anal data for win %d shift_buf=%d %d %d %d \n",
                i, shift_buf[0], shift_buf[1], shift_buf[2], shift_buf[3]);
#endif
                output_analyzed_data(filename,"Shifted analyzed data for window",
                    data->color_data[0].data, data->color_data[1].data,
                    data->color_data[2].data, data->color_data[3].data,
                    win_beg, win_end, shift_buf, i, data);
            }

            if (MONITOR) {
                fprintf(stderr,
                "3rd call: window=%d spacing=%f spacing_var=%f rel_error=%f\n",
                i, spacing[i], spac_var[i], spac_var[i]/spacing[i]);
                fprintf(stderr,
                   "   best_min_pos=%d best_max_pos=%d\n", best_min_pos, best_max_pos);
                fprintf(stderr,
                   "   best_min_color0=%d best_min_color1=%d\n",
                   best_min_color0, best_min_color1);
                fprintf(stderr,
                   "   best_min_spa=%d best_max_spa=%d\n", best_min_spa, best_max_spa);
                fprintf(stderr, 
                "3rd call: best shift = %d %d %d %d\n\n",
                    shift_buf[0], shift_buf[1], shift_buf[2], shift_buf[3]);
            }

            for (color=0; color < NUM_COLORS; color++)
                shift[color][i] = shift_buf[color];

            if (DEBUG_CURTIS && options->Verbose > 2) {
                putc((i0-i) % 10 == 0 ? progc[1] : progc[0], stderr);
                fflush(stderr);
            }
        }
    }

#else
    for (color=0; color < NUM_COLORS; color++) 
        shift_buf[color] = 0;

    for (i=i0+1; i < num_wins; i++) {
        win_beg = data_beg + win_size* i   /2;
        win_end = data_beg + win_size*(i+2)/2;
        for (j=0; j<NUM_COLORS; j++)
            shift_buf[j] = 0;
 
        bubbleSortPeakPos(dp, num_DPs, shift_buf, win_beg, win_end);

        estimate_spacing_variation_of_shifted_peaks(dp, num_DPs, 
            win_beg, win_end, shift_buf, 0,
	    spacing+i, spac_var+i, &std_dev, i, &min_spacing, &max_spacing,
            &min_pos, &max_pos, &min_color0, &min_color1, data, options, 
            message);
        shift_err[i] = spacing[i]/2;
    }
#endif

    if (DEBUG_CURTIS && (options->Verbose > 2)) 
        fprintf(stderr, " Done\n");

    /* Find trusted estimates
     * GD: not all of the shifts in indivu=idual windows will be used in
     * calculation of shifting curve!!!
     */
    ngood = num_wins;
    igood = CALLOC(int, num_wins);

    for ( i=0; i < num_wins; i++ ) {
        for ( color=0; color < NUM_COLORS; color++ )
            shift_buf[color] = shift[color][i];

        if (areMobEstTrustworthy(shift_buf, spacing[i]))
            igood[i] = 1;
        else 
            igood[i] = 0;
    }

    if (ngood < 2 * POLYFIT_DEGREE + 1) {	/* not enough trustworthy estimates */
        if (options->Verbose > 1)
            fprintf(stderr, "Warning: Poor mobility shift estimates.\n"
	    "    No shift corrections applied.\n"
	    "    Using constant default spacing of %f\n", crude_spacing_estimate);
        spac_model_coeff[0] = crude_spacing_estimate;
        for ( i=1; i < POLYFIT_DEGREE + 1; i++ ) 
            spac_model_coeff[i] = 0;
    }
    else {
	/* Fit mob. shifts with polynomial models */
        x = CALLOC(float, ngood+1);	/* float buffers for fitting */
        for (i=0; i< NUM_COLORS; i++)
            y[i] = CALLOC(float, ngood+1);
        var = CALLOC(float, ngood+1);
        MEM_ERROR(var);

        for ( i=0; i < ngood; i++ ) {
            x[i]   = data_beg + win_size* i/2 + win_size/2;
            var[i] = shift_err[igood[i]];
        }
        x[ngood]   = alpha * (win_size*(num_wins-2)/2+win_size/2);  
        var[ngood] = beta  * spacing[ngood-1];	/* at end to control fit */

        for (color=0; color < NUM_COLORS; color++) 
        {
            if (shift_flag[color]==0) 
                continue;

            for (i=0; i<ngood; i++)
                if (igood[i])
	            y[color][i] = (float) shift[color][i];
                else 
                    y[color][i] = 0;

#if POLY_MOB_SHIFT_APPROX
            /* Approximate mobility shift curve with polynomial */
            polyfit(x, y[color], var, ngood+1, mobs_model_coeff, 
                POLYFIT_DEGREE + 1);
            for (i=0; i < POLYFIT_DEGREE + 1; i++) 
                a[color][i] = mobs_model_coeff[i];
#endif
            if (options->xgr) {
                output_mobility_curve(x, y[color], ngood+1, mobs_model_coeff, 
                    POLYFIT_DEGREE + 1, color);
            }

            /* GD: this is where the polynomial coefficients for mobility 
             *     shifts are stored 
             */
#if DEBUG_CURTIS
            if ( options->Verbose > 2 )
	        printf("fitted polynomial coeff's for color %d: "
	           "%10.3g %10.3g %10.3g\n",
	           color, mobs_model_coeff[0], 
                   mobs_model_coeff[1], mobs_model_coeff[2]);
#endif
        }

        /* Apply mobility shifts to data */
        if (options->raw_data && APPLY_MOB_SHIFTS) 
        {
            int length = data->color_data[0].length;
            int *new_trace = CALLOC(int, length);

            /* Compress / stretch traces according to mobility shift curve */
            for (color=0; color < NUM_COLORS; color++)  
            {
               if (shift_flag[color]==0) 
                   continue;
               
               for (j=0; j<length; j++) {   
                   new_trace[j] = -1;
               }

               for (j=0; j<length; j++) {
                   int shift;
                   if (POLY_MOB_SHIFT_APPROX)
                       shift = qv_round(mobility_shift_curve(j, a[color], 
                           POLYFIT_DEGREE+1));
                   else 
                       shift = qv_round(mobility_shift_curve2(j, x, y[color], 
                           ngood));

                   if ((shift+j < 0) || 
                       (shift+j > data->color_data[color].length-1))
                       continue;
                   if (new_trace[shift+j] < 0)
                       new_trace[shift+j] =  data->color_data[color].data[j]; 
                   else
                      new_trace[shift+j] = (data->color_data[color].data[j]
                                          + new_trace[shift+j]) / 2.;
               }

               /* Now fill the gaps in the new trace using linear interpolation 
                * or set to zero if the beginning or end of trace
                */
               {
                   int lpos=-1, rpos;
                   for (j=0; j<length; j++) 
                   {
                       if (new_trace[j] >= 0)
                           lpos = j;
                       else 
                       {
                           rpos = j;
                           while ((rpos <= length-1) && (new_trace[rpos] < 0))
                               rpos++;

                           if ((lpos >= 0) && (rpos <= length-1)) {
                               new_trace[j] = new_trace[lpos] + 
                                   (double)(j-lpos)*(double)(new_trace[rpos]-new_trace[lpos])
                                  /(double)(rpos-lpos);
                           }
                           else 
                               new_trace[j] = 0;
                       }
                   }
               }
               memcpy(data->color_data[color].data, new_trace, 
                   length*sizeof(int));
            }
            FREE(new_trace);
        }
        else 
        {
   	    /* Update ipos, beg, and end peak fields */
            for ( i=0; i < num_peaks; i++ ) {
                color = peaks[i]->color_index;
                s = 0;
                fpoly(peaks[i]->ipos+shift_flag[color], powers-1, 
                    POLYFIT_DEGREE+1);
                for ( j=0; j < POLYFIT_DEGREE+1; j++ ) 
                    s += a[color][j] * powers[j];
                j = qv_round(s);
                peaks[i]->ipos += j;
                peaks[i]->beg += j;
                peaks[i]->end += j;
            }
        }

 	/* Calculate the spoacing curve 
         * num_wins is local variable, and num_windows = global
         */
        num_windows = num_wins; 
        spac_mod_pos[i0] = data_beg + win_size* i0/2 + win_size/2;
        spac_mod_val[i0] = spacing[i0];
        for ( i=i0+1; i < num_windows; i++ )
        {
            spac_mod_pos[i] = data_beg + win_size* i/2 + win_size/2;
            spac_mod_val[i] = spacing[i];
            if (spac_mod_val[i] < spac_mod_val[i-1])
                spac_mod_val[i] = spac_mod_val[i-1];

            if ((spacing[i] <= 0) || isnan(spacing[i])) {
                spac_mod_val[i] = spac_mod_val[i-1];
            }
            var[i] = spac_var[i];
        }
        for ( i=i0-1; i >=0; i--)
        {
            spac_mod_pos[i] = data_beg + win_size* i/2 + win_size/2; 
            spac_mod_val[i] = spacing[i];
            if ((spacing[i] <= 0) || isnan(spacing[i])) {
                spac_mod_val[i] = spac_mod_val[i+1];
            } 
            var[i] = spac_var[i];
        }
#if 0
        fprintf(stderr, "i0=%d data_beg=%d data_end=%d num_windows=%d\n", 
            i0, data_beg, data_end, num_windows);
        fprintf(stderr, "spac_mod_pos=\n");
        for ( i=0; i < num_windows; i++ )
            fprintf(stderr, "%f ", spac_mod_pos[i]);
        fprintf(stderr, "\n");
        fprintf(stderr, "spacing=\n");
        for ( i=0; i < num_windows; i++ )
            fprintf(stderr, "%f ", spacing[i]);
        fprintf(stderr, "spac_mod_val=\n");
        fprintf(stderr, "\n");
        for ( i=0; i < num_windows; i++ )
            fprintf(stderr, "%f ", spac_mod_val[i]);
        fprintf(stderr, "\n");
#endif

        sliding_window5_average(spac_mod_val, num_windows);

//      polyfit(spac_mod_pos, spac_mod_val, var, num_windows, spac_model_coeff, 
//          POLYFIT_DEGREE + 1);

        if (options->Verbose > 2) {
            fprintf(stderr, "Spacing model coeff:\n");
            for ( i=0; i < POLYFIT_DEGREE; i++ ) {
                fprintf(stderr, "c[%d]=%f ", i, spac_model_coeff[i]);
            }
            fprintf(stderr, "\n");
        }

        if (options->xgr) {
            output_spacing_curve(spac_mod_pos, spac_mod_val, num_windows, 
                POLYFIT_DEGREE + 1, win_size, data);
        }

        FREE(var);
        FREE(x);
        for (color=0; color < NUM_COLORS; color++)
            FREE(y[color]);
    }

    if ( MONITOR > 1 ) {		/* write out results */
        if ( (fp = fopen("tt_make_optimal_DP_mobility_shifts", "w")) != NULL ) {
            for ( i=0; i < num_wins; i++ ) {
	        fprintf(fp, "%7d", win_size*(i-1)/2+win_size/2);
	        for ( color=0; color < NUM_COLORS; color++ )
	            fprintf(fp, "%4d", shift[color][i]);
	            fprintf(fp, " %8.3g\n", shift_err[i]);
            }
            fclose(fp);
        }

        if ( (fp = fopen("tt_dp_mob_shift_curves", "w")) != NULL ) 
        {
            for ( i=0; i < num_wins; i++ ) {
	        fprintf(fp, "%7d", win_size*(i-1)/2+win_size/2);
	        for ( color=0; color < NUM_COLORS; color++ ) {
	            s = 0;
	            fpoly(win_size*(i-1)/2+win_size/2, powers-1, 
                        POLYFIT_DEGREE+1);
	            for ( j=0; j < POLYFIT_DEGREE+1; j++ )
	                s += a[color][j] * powers[j];
	            fprintf(fp, " %6.2f", s);
	        }
	        putc('\n', fp);
            }
            fclose(fp);
        }

        if ( (fp = fopen("tt_dp_spacing", "w")) != NULL ) {
            for ( i=0; i < num_wins; i++ )
	        fprintf(fp, "%7d %6.2f %6.2f %6.2f\n", 
                    win_size*(i-1)/2+win_size/2, spacing[i],
		    spac_var[i], spacing_curve(INT_FLT(win_size*(i-1)/2+win_size/2)));
            fclose(fp);
        }
    }

		/* release memory */
    for ( color=0; color < NUM_COLORS; color++ )
        FREE(shift[color]);
    FREE(shift_err);
    FREE(spacing);
    FREE(spac_var);
    FREE(igood);
    FREE(dp);
    return SUCCESS;

 error: 
    for ( color=0; color < NUM_COLORS; color++ )
        FREE(shift[color]);
    FREE(shift_err);
    FREE(spacing);
    FREE(spac_var);
    FREE(var);
    FREE(x);
    for (color=0; color < NUM_COLORS; color++)
        FREE(y[color]);
    FREE(igood);
    FREE(dp);
    return ERROR;
}

/*******************************************************************************
 * Function: get_peak_spacing
 *
 *******************************************************************************
 */
int
get_peak_spacing(Data *data, Options *options, BtkMessage *message)
{

    int     i, num_peaks = data->peak_list_len;
    int     shift_flag[] = {1,0,1,1};
    clock_t time;
    FILE    *fp;
    double  seconds;

    if (MONITOR > 1) {
        if ( (fp = fopen("tt_peaks", "w")) != NULL ) {
            for ( i=0; i < num_peaks; i++ )
                fprintf(fp, "%5f %8.2f\n", data->peak_list[i]->ipos,
                        data->peak_list[i]->wiheight);
            fclose(fp);
        }
    }
    
    if (options->shift == 0) {
        for ( i=0; i < NUM_COLORS; i++ )
            shift_flag[i] = 0;
    }
    else
        if (options->Verbose > 1)
            fprintf(stderr, "correcting mobility shifts ... ");

    time = clock();
    if (data->peak_list_len > 1)
        make_DP_mobility_shifts(data, shift_flag, options, message);

    time = clock() - time;
    seconds = (double) time / CLOCKS_PER_SEC;
    if ((options->Verbose > 2) && options->shift )
    fprintf(stderr, "Mobility shift time = %6.3f\n", seconds);
    if (options->Verbose > 2)
        fprintf(stderr, " done\n");

    return SUCCESS;
}

/*******************************************************************************
 * Function: last_base_pos  
 * Purpose: determine the actual end of trace, where signal drops significantly
 * Algorithm: - loop through a set of windows in the trace data
 *            - in each window, determine max value of signal
 *            - if max signal in the current window drops significantly compared
 *              to max signal in the previous window, mark the beginning of the
 *              current window as the actual end of trace
 *******************************************************************************
 */
static int
last_base_pos(int *num_datapoints, int *data_end, int **chromatogram)
{
    int i, j, k, win_size = DATA_WINDOW_SIZE;
    int max_signal = 0, old_max_signal;
    int signal_drop_factor = SIGNAL_DROP_THRESHOLD;
    int num_wins = *num_datapoints/win_size;

    for (i=0; i<num_wins; i++) {
        int k_beg =  i   *win_size;
        int k_end = (i+1)*win_size;
        old_max_signal = max_signal;
        max_signal = 0;
        for (j=0; j< NUM_COLORS; j++) {
            for (k=k_beg; k<k_end; k++) {
                if (max_signal < chromatogram[j][k]) {
                    max_signal = chromatogram[j][k];
                }
            }
        }

        if ((i > QVMAX(1, num_wins/2)) &&
            (max_signal * signal_drop_factor < old_max_signal)) 
        {
           *data_end = k_beg;
            break;
        }
    }
    return SUCCESS;
}

/*****************************************************************************
 * Function: output_spacing_curve
 *****************************************************************************
 */
static void
output_normalization_curves()
{
    int   i;
    FILE *fp;

    fp = fopen("Normalization_curves.xgr", "w");
    fprintf(fp, "Title = Normalization curves\n");
    fprintf(fp, "title_x = Window #\n");
    fprintf(fp, "title_y = Normalization Curve\n");

    /* Output computed normalization factor for 0th color */
    fprintf(fp, "\ncolor = %d\n", 4);       /* green  */
    for (i=0; i<num_windows; i++) {
        fprintf(fp, "%d %f\n", i, norm_mod_val[0][i]);
    }
    fprintf(fp, "next\n");

    /* Output computed normalization factor for 1st  color */
    fprintf(fp, "\ncolor = %d\n", 9);       /* cyan   */
    for (i=0; i<num_windows; i++) {
        fprintf(fp, "%d %f\n", i, norm_mod_val[1][i]);
    }
    fprintf(fp, "next\n");

    /* Output computed normalization factor for 2nd  color */
    fprintf(fp, "\ncolor = %d\n", 7);       /* yellow */
    for (i=0; i<num_windows; i++) {
        fprintf(fp, "%d %f\n", i, norm_mod_val[2][i]);
    }
    fprintf(fp, "next\n");

    /* Output computed normalization factor for 3rd  color */
    fprintf(fp, "\ncolor = %d\n", 2);       /* red    */
    for (i=0; i<num_windows; i++) {
        fprintf(fp, "%d %f\n", i, norm_mod_val[3][i]);
    }
    fprintf(fp, "next\n");
    fclose(fp);
}


/*****************************************************************************
 * Function: output_new_spacing_curve
 *****************************************************************************
 */
static void
output_log_norm_curve(float *ave_int[NUM_COLORS], float *sum_ints[NUM_COLORS], 
    int *num_peaks[NUM_COLORS], int num_points)
{
    int    i;
    FILE  *fp;

    fp = fopen("Log_normalization_curves.xgr", "w"); 
    fprintf(fp, "Title = Log norm. curves\n");
    fprintf(fp, "title_x = Sum_ints\n");
    fprintf(fp, "title_y = Log ave intensity\n");

    /* Output computed results for the 0th color */
    fprintf(fp, "\ncolor = %d\n", 4);       /* green  */
    for (i=0; i<num_points ; i++) {
        fprintf(fp, "%f %f\n", sum_ints[0][i], log10(ave_int[0][i]));
    }
    fprintf(fp, "next\n");

    /* Output computed results for the 1st  color */
    fprintf(fp, "\ncolor = %d\n", 9);       /* cyan   */
    for (i=0; i<num_points ; i++) {
        fprintf(fp, "%f %f\n", sum_ints[1][i], log10(ave_int[1][i]));
    }

    fprintf(fp, "next\n");

    /* Output computed results for the 2nd color */
    fprintf(fp, "\ncolor = %d\n", 7);       /* yellow */
    for (i=0; i<num_points ; i++) {
        fprintf(fp, "%f %f\n", sum_ints[2][i], log10(ave_int[2][i]));
    }
    fprintf(fp, "next\n");

    /* Output computed results for the 3rd  color */
    fprintf(fp, "\ncolor = %d\n", 2);       /* red    */
    for (i=0; i<num_points ; i++) {
        fprintf(fp, "%f %f\n", sum_ints[3][i], log10(ave_int[3][i]));
    }
    fprintf(fp, "next\n");

    fclose(fp);
}

/*******************************************************************************
 * Function: total_number_of_peaks
 *******************************************************************************
 */
static int
normalize_signals(Data *data, Options *options, BtkMessage *message)
{
    int    i, j, m, num_wins, win_size, win_beg, win_end, num_factors;
    int    shift[NUM_COLORS]       = {0, 0, 0, 0};
    float  norm_factor[NUM_COLORS] = {0., 0., 0., 0.};
    float  average_peak_height;
    float *sum_ints[NUM_COLORS]  = {NULL, NULL, NULL, NULL}; 
    float *ave_int[NUM_COLORS]   = {NULL, NULL, NULL, NULL};
    int   *num_peaks[NUM_COLORS] = {NULL, NULL, NULL, NULL};

    num_wins = DEFAULT_NUM_WINDOWS;
    win_size = 2 * (data->pos_data_end - data->pos_data_beg) / num_wins;
    if (win_size < MIN_WIN_SIZE) {
        win_size = MIN_WIN_SIZE;
        num_wins = 2 * (data->pos_data_end - data->pos_data_beg) / win_size;
    }
    num_windows = num_wins;
    for (i=0; i< NUM_COLORS; i++) {
        sum_ints[i]  = CALLOC(float, num_wins);
        ave_int[i]   = CALLOC(float, num_wins);
        num_peaks[i] = CALLOC(int,   num_wins);
    }

    /* Calculate the coefficients of normalization model */
    for (m=0; m<num_wins; m++)
    {
        win_beg =  data->pos_data_beg + win_size*m/2;
        win_end =  (win_beg + win_size  < data->pos_data_end) ?
                   (win_beg + win_size) : data->pos_data_end;
        norm_mod_pos[m] = (win_beg + win_end) / 2;

        for (i=0; i< NUM_COLORS; i++) 
        {
            num_peaks[i][m] = 0;
            ave_int[i][m]   = 0.;
            for (j=0; j<data->color_data[i].peak_list_len; j++) {
                int k = data->color_data[i].peak_list[j].data_peak_ind;

                if ((data->color_data[i].peak_list[j].ipos <  win_beg) ||
                    (data->color_data[i].peak_list[j].ipos >= win_end))
                    continue;
#if 1
                if (!is_dp(data->peak_list[k], shift, data))
                    continue;
#endif
                ave_int[i][m] += data->color_data[i].peak_list[j].iheight;
                num_peaks[i][m]++;
            }
#if 0
//          if (ave_int[i][m] < 0) 
                fprintf(stderr, "ave_int[%d][%d]=%f\n", i, m, ave_int[i][m]);
#endif
            if (m == 0) 
                sum_ints[i][m] = ave_int[i][m];
            else
                sum_ints[i][m] = sum_ints[i][m-1] + ave_int[i][m];
#if 0
                fprintf(stderr, "ave_int[%d][%d]=%f sum_ints[%d][%d]=%f sum_ints[%d][%d]=%f\n", 
                    i, m, ave_int[i][m], i, m, sum_ints[i][m], i, m-1, sum_ints[i][m-1]);
#endif
            ave_int[i][m]  /= (float)num_peaks[i][m];
            /* Don't normalize a given trace if there are no peaks in the window */
            norm_factor[i] = (num_peaks>0) ?  ave_int[i][m] : -1;
#if SHOW_NORM_FACTOR
            fprintf(stderr, "win # = %d num_peaks=%d norm_factor[%d]=%f\n", 
                m, num_peaks, i, (float)norm_factor[i]);
#endif
        }
        average_peak_height = 0;
        num_factors = 0;
        for (i=0; i< NUM_COLORS; i++) {
            if (norm_factor[i] > 0) {
                average_peak_height += norm_factor[i];
                num_factors++;
            } 
        }
        average_peak_height /= (num_factors>0) ? (float)num_factors : 1;

        for (i=0; i< NUM_COLORS; i++) {
            norm_mod_val[i][m] = (norm_factor[i] > 0) ? 
                (average_peak_height / norm_factor[i]) : 1.;
        }
    }

    output_log_norm_curve(ave_int, sum_ints, num_peaks, num_wins);  

    /* Output normalization curves */
    for (i=0; i< NUM_COLORS; i++) {
        sliding_window5_average(norm_mod_val[i], num_windows);
    }

    if (options->xgr)
        output_normalization_curves();

    /* Apply normalization model to data */
    for (i=0; i< NUM_COLORS; i++) {
        for (j=0; j<data->color_data[i].length; j++) {
            data->color_data[i].data[j] *= normalization_curve(j, i);
        }

        /* Update peak heights upon normalization of data */
        for (j=0; j<data->color_data[i].peak_list_len; j++) {
            int pos = data->color_data[i].peak_list[j].pos;
            int height = data->color_data[i].data[pos];
            data->color_data[i].peak_list[j].height = height;
            data->color_data[i].peak_list[j].iheight= height;
        }
    }

    for (i=0; i< NUM_COLORS; i++) {
        FREE(sum_ints[i]);
        FREE(ave_int[i]);
        FREE(num_peaks[i]);
    }
    return SUCCESS;
}

/*********************************************************************
 * Function: output_chromatogram
 */
void
output_chromatogram(char *filename, char *title, int *chromatogram0,
    int *chromatogram1, int *chromatogram2, int *chromatogram3,
    int num_datapoints, Data *data)
{
    int shift[NUM_COLORS] = {0, 0, 0, 0};
    output_analyzed_data(filename, title, 
        chromatogram0, chromatogram1,
        chromatogram2, chromatogram3,
        0, num_datapoints, shift, -1, data);
}

/*********************************************************************
 * Function: get_baseline
 * Purpose:  return the min value of signal in the selected window
 *********************************************************************
 */
static int  
get_baseline(int *chromatogram, int beg, int end)
{
    int       i, min_value = INF;

    /* Collect baseline data in windows */
    for ( i=beg; i < end; i++)
    {
        if (min_value > chromatogram[i])
            min_value = chromatogram[i];
    }

    return min_value;
}

/*********************************************************************
 * Function: prebaseline
 * Purpose:  return the max value of baselined signal in the 
 * selected window
 *********************************************************************
 */
static int
get_normfact(int *chromatogram, int baseline, int beg, int end)
{
    int       i, max_value = NINF;

    /* Collect baseline data in windows */
    for ( i=beg; i < end; i++)
    {
        if (max_value < chromatogram[i]-baseline)
            max_value = chromatogram[i]-baseline;
    }

    return max_value;
}

/*********************************************************************
 * Function: is_highest_signal
 *********************************************************************
 */
static int
is_highest_signal(int win, int color, int pos, int **chromatogram)
{
    int i, is_highest = 1;
    int sig = chromatogram[color][pos];

    for (i=0; i<NUM_COLORS; i++) {
        if (i == color)
            continue;        
   
        if (sig <= chromatogram[i][pos]) 
            is_highest = 0; 
    }
   
    return is_highest; 
}

/*********************************************************************
 * Function: get_determinant3 
 *********************************************************************
 */
static float
get_determinant3(float **minormat3)
{
    return minormat3[0][0]*(minormat3[1][1]*minormat3[2][2] -
                            minormat3[2][1]*minormat3[1][2])
         - minormat3[0][1]*(minormat3[1][0]*minormat3[2][2] -
                            minormat3[2][0]*minormat3[1][2])
         + minormat3[0][2]*(minormat3[1][0]*minormat3[2][1] -
                            minormat3[2][0]*minormat3[1][1]);    
}

/*********************************************************************
 * Function: get_minor_matrix3   
 *********************************************************************
 */
static int
get_minor_matrix3(int iexcl, int jexcl, float **matrix4, float 
    **minormat3)
{
    int i, j, k, m;
    
    for (i=0; i<4; i++) {
        if (i == iexcl)
            continue;
        k = (i<iexcl) ? i : (i-1);
        for (j=0; j<4; j++) {
            if (j == jexcl)
                continue;
            m = (j<jexcl) ? j : (j-1);
            minormat3[k][m] = matrix4[i][j];          
#if 0
            fprintf(stderr, 
                "In get_minor_matrix3: i=%d j=%d iex=%d jex=%d k=%d m=%d\n", 
                i, j, iexcl, jexcl, k, m);
#endif
        }
    }
    return SUCCESS;
}

/*********************************************************************
 * Function: get_determinant4
 *********************************************************************
 */
static float
get_determinant4(float **matrix4)
{
    int i;
    float *minormat3[3], det4=0.;
    for (i=0; i<3; i++) {
        minormat3[i] = CALLOC(float, 3);
    }

    for (i=0; i<4; i++) {
        get_minor_matrix3(0, i, matrix4, minormat3);
            det4 += pow(-1, 0 + i) * get_determinant3(minormat3);

    }

    for (i=0; i<3; i++) {
        FREE(minormat3[i]);
    }
    return det4;
}


/******************************************************************************
 * Function: get_minor_matrix4
 * Purpose:  compute a matrix of minors complementary to a given element of 
 *           mcmatrix
 ******************************************************************************
 */
static int
get_minor_matrix4(float **mcmatrix, float **mcminmat)
{
    int i, j;
    float *minormat3[3];

    for (j=0; j<3; j++) {
        minormat3[j] = CALLOC(float, 3);    
    }
   
    for (i=0; i<NUM_COLORS; i++) {
        for (j=0; j<NUM_COLORS; j++) {
            get_minor_matrix3(i, j, mcmatrix, minormat3);
            mcminmat[i][j] = get_determinant3(minormat3);
        }
    }

    for (j=0; j<3; j++) {
        FREE(minormat3[j]);
    }
    return SUCCESS;
}

/******************************************************************************
 * Function: prebaseline      
 * Purpose:  prebaseline dye signals
 *
 ******************************************************************************
 */
int
prebaseline(int num_datapoints, int **chromatogram, Options *options,
    BtkMessage *message)
{
    int win_size;
    int i, j, pos, win_beg, win_end, num_wins;
    int baseline;

    /* Here we use non-overlapping windows,
     * because baseline will be subtracted and then added
     */
    num_wins = DEFAULT_NUM_WINDOWS;
    win_size = num_datapoints / num_wins;
    if (win_size < MIN_WIN_SIZE) {
        win_size = MIN_WIN_SIZE;
        num_wins = num_datapoints / win_size - 1;
    }

    for (i=0; i < num_wins; i++)
    {
        win_beg = win_size* i;
        win_end = (win_size*(i+1) < num_datapoints) ?
                   win_size*(i+1) : num_datapoints;
        if ((win_beg < 0) || (win_end > num_datapoints))
            fprintf(stderr,
            "ERROR: Windows boundaries (%d, %d) outside data range (0, %d)\n",
            win_beg, win_end, num_datapoints-1);

        for (j=0; j<NUM_COLORS; j++) {
            baseline = get_baseline(chromatogram[j], win_beg, win_end);
            for (pos = win_beg; pos < win_end; pos++)
            {
                chromatogram[j][pos] -= baseline;
            }
        }
    }
    return SUCCESS;
}

/******************************************************************************
* Function: make_multicomponent_iteration
* Purpose:  decouple dye signals
 *
 * 
 *  Y  =  A X  + B
 *
 ******************************************************************************
 */
int
make_multicomponent_iteration(int **chromatogram, int num_datapoints, Options *options,
    BtkMessage *message)
{
    int       win_size;
    int       i, j, k, m, pos, win_beg, win_end, num_wins;
    float     det4;
    float    *mcmatrix[NUM_COLORS],
             *mcstddev[NUM_COLORS],
             *mcminmat[NUM_COLORS];
    int      *normfact[NUM_COLORS];
    int      *num_good_wins[NUM_COLORS],
             *new_chromatogram[4];

    /* Here we use non-overlapping windows */ 
    num_wins = DEFAULT_NUM_WINDOWS;
    win_size = num_datapoints / num_wins;
    if (win_size < MIN_WIN_SIZE) {
        win_size = MIN_WIN_SIZE;
        num_wins = num_datapoints / win_size - 1;
    }

    for (j=0; j<NUM_COLORS; j++) {
        normfact[j] = CALLOC(int, num_wins);
        num_good_wins[j] = CALLOC(int,   NUM_COLORS);
        new_chromatogram[j] = CALLOC(int,   num_datapoints);
        mcmatrix[j]      = CALLOC(float, NUM_COLORS);
        MEM_ERROR(mcmatrix[j]);
        mcstddev[j]      = CALLOC(float, NUM_COLORS);
        MEM_ERROR(mcstddev[j]);
        mcminmat[j]      = CALLOC(float, NUM_COLORS);
        MEM_ERROR(mcminmat[j]);
        for (k=0; k<NUM_COLORS; k++) {
            mcmatrix[j][k] = 0.;
            mcstddev[j][k] = 0.;
            mcminmat[j][k] = 0.;
            num_good_wins[j][k] = 0;
        }
    }

    /* Calculate the elements of multicomponenting
     * matrix for each window 
     * mcmatrix[j][k] describes the effect of k-th component
     *                on the intensity of j-th component
     */
    for (i=0; i < num_wins; i++)
    {
        win_beg =  win_size* i;
        win_end = (win_size*(i+1) < num_datapoints) ? 
                   win_size*(i+1) : num_datapoints;
        if ((win_beg < 0) || (win_end > num_datapoints))
            fprintf(stderr, 
            "ERROR: Windows boundaries (%d, %d) outside data range (0, %d)\n",
            win_beg, win_end, num_datapoints-1);
        for (j=0; j<NUM_COLORS; j++) 
        {
            normfact[j][i] = get_normfact(chromatogram[j], 0,
                    win_beg, win_end);
        }
        for (j=0; j<NUM_COLORS; j++)
        {
            for (k=0; k<NUM_COLORS; k++) 
            {
                float min_sig_ratio = INF;
                int best_pos;

                if (j == k) { 
                    mcmatrix[j][k] = 1.;
                    mcstddev[j][k] = 0;
                    continue;
                }

                for (pos = win_beg; pos < win_end; pos++) 
                {
                    float numer, denom;

                    if (!is_highest_signal(i, k, pos, chromatogram))
                        continue;

                    if (chromatogram[k][pos] <= 0)
                        continue;

                    numer = (float)chromatogram[j][pos];
//                          (float)normfact[j][i]; 
                    denom = (float)chromatogram[k][pos];
//                          (float)normfact[k][i];
                    if (min_sig_ratio > numer/denom) {
                        min_sig_ratio = numer/denom;
                        best_pos = pos;
                    }
                }
                if (min_sig_ratio < 1) {
                    if (min_sig_ratio < 0)
                        fprintf(stderr, "Error: min_sig_ratio < 0!!!\n");
                    mcmatrix[j][k] += min_sig_ratio;
                    mcstddev[j][k] += min_sig_ratio * min_sig_ratio;
                    num_good_wins[j][k]++;
#if 0
                    fprintf(stderr, "Best_pos[%d,%d] = %d\n", j, k, best_pos);
#endif
                }
#if 0
                fprintf(stderr,
                "     window=%d j,k=%d %d pos=%d mcmatrix[j][k]=%f\n",
                i, j, k, (win_beg+win_end)/2, min_sig_ratio);
#endif
            }
        } 
    }

    /* Compute mcmatrix averaged over all windows */
    for (j=0; j<NUM_COLORS; j++)
    {
        for (k=0; k<NUM_COLORS; k++)
        {
            if (j != k) {
                mcmatrix[j][k] /= (float)num_good_wins[j][k];
                mcstddev[j][k]  = mcstddev[j][k]/(float)num_good_wins[j][k] -
                                  mcmatrix[j][k] * mcmatrix[j][k];
            }
#if 0
            fprintf(stderr,
            "For j=%d k=%d  MC_matrix=%f MC_stddev=%f\n",
            j, k, mcmatrix[j][k], mcstddev[j][k]);
#endif
        }
    }

#if 0
    /* Normalize columns of the averaged matrix so that their sum equals unity */
    for (k=0; k<NUM_COLORS; k++) {      // k is column number
        float sum = 0;
        for (j=0; j<NUM_COLORS; j++)    // j is line number
            sum += mcmatrix[j][k];
        for (j=0; j<NUM_COLORS; j++) {
            mcmatrix[j][k] /= sum;
#if 0
            fprintf(stderr,
            "For j=%d k=%d  Normalized MC_matrix=%f\n",
            j, k, mcmatrix[j][k]);
#endif
        }
    }
#endif

    det4 = get_determinant4(mcmatrix);

    /* Compute matrix of minors complementary to elements of mcmatrix */
    get_minor_matrix4(mcmatrix, mcminmat);
#if 0
    fprintf(stderr, "MC_matrix_determinant = %f\n", det4);
    fprintf(stderr, "MC_MINORS_matrix=\n");
    for (j=0; j<NUM_COLORS; j++)
    {
        for (k=0; k<NUM_COLORS; k++)
        {
            fprintf(stderr, "%f ", mcminmat[j][k]);
        }
        fprintf(stderr, "\n");
    }
#endif

    /* Apply the results to chromatogram to prebaseline and multicomponent */
    for (m=0; m < num_wins; m++)
    {
        win_beg = win_size* m;
        win_end = win_size*(m+1);
        for (j=0; j<NUM_COLORS; j++)
        {
            for (i=win_beg; i< win_end; i++) 
            {
                for (k=0; k<NUM_COLORS; k++) {
                    new_chromatogram[j][i] += pow(-1, j+k)
                        * chromatogram[k][i] 
                        * mcminmat[k][j];
                }
            }
        }
    }

    for (i=0; i<num_datapoints; i++) 
    {
        for (j=0; j<NUM_COLORS; j++)
        {
            chromatogram[j][i] = new_chromatogram[j][i];     
            if (chromatogram[j][i] < 0) 
                chromatogram[j][i] = 0;               
        }
    }

    for (j=0; j<NUM_COLORS; j++) {
        FREE(normfact[j]);
        FREE(num_good_wins[j]);
        FREE(mcmatrix[j]);
        FREE(mcstddev[j]);
        FREE(mcminmat[j]);
        FREE(new_chromatogram[j]);
    }
    return SUCCESS;
error:
    for (j=0; j<NUM_COLORS; j++) {
        FREE(normfact[j]);
        FREE(num_good_wins[j]);
        FREE(mcmatrix[j]);
        FREE(mcstddev[j]);
        FREE(mcminmat[j]);
    }
    return ERROR;
}

/*********************************************************************
 * Function: prefilter_and_prebaseline
 *********************************************************************
 */
static void 
prefilter_and_prebaseline(int *num_datapoints, int **chromatogram, int sample_rate, 
    Options *options, BtkMessage *message)
{
    int i;
    long new_num_datapoints = (long)(*num_datapoints);
    long min_num_datapoints = (long)(*num_datapoints);
    for (i=0; i<NUM_COLORS; i++)
    {
//      median_filter_n5(chromatogram[i], (long)(*num_datapoints));  
//      new_num_datapoints = smooth_and_decimate(chromatogram[i], *num_datapoints, 
//          sample_rate);
        
        if (min_num_datapoints > new_num_datapoints)
            min_num_datapoints = new_num_datapoints;

#if 0
        // Only data from ABI 3700 require sheath flow correction.
        if (!strcmp(options->lut_type, "3700pop5") ||
            !strcmp(options->lut_type, "3700pop6"))
        {
            sheathFlowCorrection(chromatogram[i], min_num_datapoints);
        }
#endif
    }
   *num_datapoints = min_num_datapoints;
    prebaseline(*num_datapoints, chromatogram, options, message);
    return;
}

static void
multicomponent_data(int num_datapoints, int **chromatogram, 
    Options *options, BtkMessage *message, Data *data)
{
    int i;

    for (i=0; i<NUM_MULTICOMP_ITER; i++) {
        (void)make_multicomponent_iteration(chromatogram, num_datapoints, 
            options, message);
        prebaseline(num_datapoints, chromatogram, options, message);
        if (options->xgr) {
            char xgrfilename1[MAXPATHLEN];
            char xgrfilename2[MAXPATHLEN];
            sprintf(xgrfilename1, "2_%d_Prefilt_mult_data.xgr", i+1);
            sprintf(xgrfilename2,
                "Prefiltered and multicomponented-%d raw data", i+1);
            output_chromatogram(xgrfilename1, xgrfilename2,
                chromatogram[0], chromatogram[1], chromatogram[2],
                chromatogram[3], num_datapoints, data);
        }
    }
    return;
}

/*********************************************************************
 * Function: determine_beg_of_data   
 *********************************************************************
 */
static int 
determine_beg_of_data(int *data_beg, int start_tol,
    int *num_datapoints, int **chromatogram, Data *data) 
{
    int i;

    /* Get the first base pos. It the procedure fails, 
     * assign data_beg to -1 */
   if ((*data_beg = first_base_pos(chromatogram, (long)(*num_datapoints)))
       == ERROR)
       return ERROR;

    if (DEBUG)
        fprintf(stderr, "first base found at %d\n", *data_beg);

    if (CLIP_DATA_BEG && (*data_beg > start_tol)) {
       *num_datapoints -= (*data_beg - DEFAULT_SPACING/2 >= 0) ? 
            (*data_beg - DEFAULT_SPACING/2) : 0;
        for ( i=0; i < NUM_COLORS; i++)
        {
            memmove(chromatogram[i], chromatogram[i]+ *data_beg,
                (*num_datapoints)*sizeof(int));
        }
    }

    for (i=0; i < NUM_COLORS; i++) {
        clip(chromatogram[i], (long)(*num_datapoints), 0, AD_MAX);
        median_filter(chromatogram[i], (long)(*num_datapoints), 3);
        // baseline for primer dye data
        lowpass(chromatogram[i], (long)(*num_datapoints));
        data->color_data[i].length = *num_datapoints;
    }

    if (*data_beg < 0)
        *data_beg = 0;
    data->pos_data_beg = *data_beg; /* make sure it is>=0 */

    return SUCCESS;
}

/*********************************************************************
 * Function: baseline_data    
 *********************************************************************
 */
static void
baseline_data(int *num_datapoints, int **chromatogram, Data *data)
{
    int i;
    for ( i=0; i < NUM_COLORS; i++) {
        if (DEBUG > 2)
            fprintf(stderr, "color %d...\n", i);

//      fprintf(stderr, "baselining... ");
        zero_baseline(chromatogram[i], (long)(*num_datapoints));
//      fprintf(stderr, " done\n");
        clip(chromatogram[i], (long)(*num_datapoints), 0, SHRT_MAX);
    }

    for (i=0; i<NUM_COLORS; i++) {
        int j;
        data->color_data[i].length = *num_datapoints;
        for (j=0; j<data->color_data[i].length; j++)
            data->color_data[i].data[j] = chromatogram[i][j];
    }
    return;
}

/*******************************************************************************
 * Function: determine_end_of_data
 *******************************************************************************
 */
static int 
determine_end_of_data(int *num_datapoints, int **chromatogram,
    int *data_end, Data *data, Options options)   
{
    int i, init_num_datapoints = *num_datapoints;

#if 0
        fprintf(stderr, "data_end=%d *num_datapoints=%d \n",
            *data_end, *num_datapoints);
#endif
    if (options.Verbose > 2)
        fprintf(stderr, "determining last base pos ... ");
   *data_end = *num_datapoints;
    if (last_base_pos(num_datapoints, data_end, chromatogram)
        == ERROR)
    {
        return ERROR;
    }
    data->pos_data_end = *data_end;
    if (options.Verbose > 2)
        fprintf(stderr, " done...\n");

    if (CLIP_DATA_END) {
#if 0
        fprintf(stderr, "data_end=%d *num_datapoints=%d \n",
            *data_end, *num_datapoints);
#endif
       *num_datapoints = *data_end;
        for (i=0; i<NUM_COLORS; i++) {
            data->color_data[i].length = *num_datapoints;
            int j;
            for (j=0; j<*num_datapoints; j++)
                chromatogram[i][j] = data->color_data[i].data[j];
//          memcpy(data->color_data[i].data, chromatogram[i],
//              data->color_data[i].length*sizeof(int));
            if (options.Verbose > 2) {
                fprintf(stderr, "Number of datapoints reduced from %d to %d\n",
                    init_num_datapoints, *num_datapoints);
            }
        }
    }

    for (i=0; i<NUM_COLORS; i++)
        data->color_data[i].length = *num_datapoints;

    return SUCCESS;
}

/*******************************************************************************
 * Function: process_peaks 
 *******************************************************************************
 */
static int
process_peaks(int **chromatogram, Data *data, Options *options,
    BtkMessage *message)
{
    int i;
    int shift[NUM_COLORS] = {0, 0, 0, 0};

    /* Detect, expand and resolve peaks */
    if (options->Verbose > 2)
        fprintf(stderr, "detecting, expanding and resolving peaks ... ");
    if (data_detect_peaks(data, options, message) != SUCCESS) {
        return ERROR;
    }

    if (data_expand_peaks(data, options, message) != SUCCESS) {
        return ERROR;
    }
    if (data_resolve_peaks(data, options, message) != SUCCESS) {
        return ERROR;
    }

    if (options->Verbose > 2)
        fprintf(stderr, " done...\n");

    /* Create a single peak list */
    if (options->Verbose > 2)
        fprintf(stderr, "creating single peak list ... ");
    if (bc_data_create_single_ordered_peak_list(data, shift, message) != SUCCESS)
    {
        sprintf(message->text, "Error creating single peak list\n");
        return ERROR;
    }
    if (options->Verbose > 2)
        fprintf(stderr, " done...\n");

    /* Initialize wiheights to iheights.
     * These are used when estimating peak sapcing
     */
    for (i=0; i<data->peak_list_len; i++) {
        data->peak_list[i]->wiheight = data->peak_list[i]->height;
    }
    return SUCCESS;
}

/*******************************************************************************
 * Function: data_normalize   
 *******************************************************************************
 */
static int 
data_normalize(int **chromatogram, Data *data, Options *options, 
    BtkMessage *message)
{
    int i;

    if (options->Verbose > 2)
        fprintf(stderr, "normalizing signals ... ");
    if (normalize_signals(data, options, message) != SUCCESS) {
        return ERROR;
    }
    if (options->Verbose > 2)
        fprintf(stderr, " done...\n");

    for (i=0; i<NUM_COLORS; i++) {
        int j;
        for (j=0; j<data->color_data[i].length; j++)
            chromatogram[i][j] = data->color_data[i].data[j];
//      memcpy(chromatogram[i], data->color_data[i].data,
//         *num_datapoints*sizeof(int));
    if (SHOW_NUM_PEAKS)
        fprintf(stderr, "num_peaks[color=%d] = %d\n",
            i, data->color_data[i].peak_list_len);
    }
    return SUCCESS;
}

/*********************************************************************
 * Function: respace_chromatograms
 * win_size = number of sacns in the window minus one (= spacing)
 * chromatogram, scan, win_size - refer to "old"
 * new_chromatogram, new_scan, new_win_size - refer to new
 *********************************************************************
 */
static int
respace_chromatograms(int **chromatogram, int *num_data,
    int alloc_chromat_len, Data *data, Options *options,
    BtkMessage *message)
{
    int     i, j, k, init_num_data = *num_data;
    int     scan=0, new_scan=0, new_last_scan=0;
    int    *new_chromatogram[NUM_COLORS];
    double  spacing, new_spacing, pos, last_pos, max_pos = 0.;

#if 0
        for (i=0; i<NUM_COLORS; i++) {
            int len=data->color_data[i].peak_list_len;
            fprintf(stderr, "Last color %d old peak[%d] pos=%d\n",  i,
                len-1, data->color_data[i].peak_list[len-1].pos);
        }
#endif

    for (i=0; i<NUM_COLORS; i++) {
        new_chromatogram[i] = CALLOC(int, alloc_chromat_len);
        for (j=0; j<alloc_chromat_len; j++)
            new_chromatogram[i][j] = -1;
    }

#if 0
    fprintf(stderr, "alloc_chromat_len =%d init_num_data=%d\n",
        alloc_chromat_len, init_num_data);
#endif
#if 0
            fprintf(stderr, "Orig. pos_data_beg=%d pos_data_end=%d\n",
                data->pos_data_beg, data->pos_data_end);
#endif

    /* Initialize */
    spacing  = spacing_curve(scan);
    new_spacing = DEFAULT_SPACING;
    last_pos = 0.;
   *num_data = 0;
    for (i=0; i<NUM_COLORS; i++)
        new_chromatogram[i][0] = chromatogram[i][0];


    /* Loop through all windows */
    while ((scan + spacing < init_num_data-1) &&
           (*num_data + new_spacing < alloc_chromat_len))
    {

        spacing  = spacing_curve(scan);


//      if (scan + spacing > init_num_data) {
//          spacing = init_num_data-1 - scan;
//          new_spacing = spacing;
//      }
#if 0
        new_spacing = spacing;
#endif

#if 0
        fprintf(stderr, "scan = %d spacing=%f new_spacing=%f last_pos=%f\n",
            scan, spacing_curve(scan), new_spacing, last_pos);
#endif
        for (j=new_scan; j<new_scan+new_spacing; j++)
        {
            /* last_pos is the image of the last new scan populated at previous
             * step
             */
            pos = last_pos + (double)(j-new_scan) * spacing/new_spacing;
            for (k=scan;
                 k <= QVMIN(2*(scan+spacing), init_num_data-2);
                 k++)
            {
                if ((pos >= k) && (pos < k+1))
                {
                    if (k == data->pos_data_beg)
                        data->pos_data_beg = - j;

                    if (k == data->pos_data_end)
                        data->pos_data_end = - j;

                    if (max_pos < pos)
                        max_pos = pos;

                    if (new_last_scan < j)
                        new_last_scan = j;

                    for (i=0; i<NUM_COLORS; i++)
                    {
                        int val =
                            new_chromatogram[i][j] = ROUND(
                                (double) chromatogram[i][k  ] +
                                (double)(chromatogram[i][k+1] -
                                         chromatogram[i][k  ])*
                                        (pos - (double)k));
                        if (new_chromatogram[i][j] < 0)
                            new_chromatogram[i][j] = val;
                        else
                            new_chromatogram[i][j] =
                           (new_chromatogram[i][j] + val)/2;
#if 0
                        if (i == 0)
                        fprintf(stderr,
                        "i=%d j=%4d new_chr=%4d k=%4d chr=%4d pos=%4.3f max_pos=%4.3f l_pos=%4.3f scan=%4d spac=%4.6f new_spac=%4.6f\n",
                            i, j, new_chromatogram[i][j], k, chromatogram[i][k], pos, max_pos, last_pos, scan, spacing, new_spacing);
#endif
#if 0
                        if (i == 0)
                        fprintf(stderr,
                        "i=%d j=%4d k=%4d pos=%4.3f max_pos=%4.3f scan=%4d spac=%4.6f new_scan=%d new_last_scan=%d\n",
                            i, j, k, pos, max_pos, scan, spacing, new_scan, new_last_scan);
#endif
                    }
                }
            }
        }
        last_pos = max_pos;
        max_pos  = 0;
        scan     = (int)last_pos;
        new_scan = new_last_scan;
       *num_data = new_scan;
        if (*num_data > alloc_chromat_len)
            *num_data = alloc_chromat_len;
#if 0
    fprintf(stderr, "    next step\n");
#endif
    }
   *num_data = new_last_scan;

#if 0
    fprintf(stderr, "final num_data=%d new_spacing=%f\n", *num_data, new_spacing);
#endif
    if (data->pos_data_beg < 0)
        data->pos_data_beg *= -1;
    if (CLIP_DATA_BEG)
        data->pos_data_beg = 0;
    if (data->pos_data_end < 0)
        data->pos_data_end *= -1;
    if (CLIP_DATA_END)
        data->pos_data_end = *num_data;

#if 0
    fprintf(stderr, "new_pos_data_beg=%d new_pos_data_end=%d\n", data->pos_data_beg, data->pos_data_end);
#endif

    if (alloc_chromat_len < *num_data) {
        fprintf(stderr,
       "Allocated memory not sufficient to store respaced chromatogram.\n");
       exit_message(options, -1);
    }
    for (i=0; i<NUM_COLORS; i++)
    {
        data->color_data[i].length = *num_data;
#if 0
        fprintf(stderr, "alloc_chromat_len=%d num_data=%d\n",
            alloc_chromat_len, *num_data);
#endif
        data->color_data[i].data = REALLOC(data->color_data[i].data, int,
           *num_data);
        for (j=0; j < *num_data; j++)
        {
            chromatogram[i][j] = new_chromatogram[i][j];
            data->color_data[i].data[j] = new_chromatogram[i][j];
        }
    }

    for (i=0; i<NUM_COLORS; i++)
        FREE(new_chromatogram[i]);


    /* Calculate new spacing curve after respacing */
#if 1
    if (CALCULATE_NEW_SPACING_CURVE)
    {
        int    num_DPs = 0, min_pos, max_pos, min_spacing, max_spacing;
        int    shift_buf[NUM_COLORS] = {0, 0, 0, 0};
        int    min_color0, min_color1;
        int    i0, win_size, num_wins, win_beg, win_end;
        Peak **dp = NULL;
        float *mean_spacing=0;
        float *var=0, std_dev;

#if 1
        fprintf(stderr, "I DO IT !!!\n");
#endif
        /* Detect peaks after respacing */
        if (data_detect_peaks(data, options, message) != SUCCESS) {
            return ERROR;
        }

#if 0
        for (i=0; i<NUM_COLORS; i++) {
            int len=data->color_data[i].peak_list_len;
            fprintf(stderr, "Last color %d new peak[%d] pos=%d\n",  i,
                len-1, data->color_data[i].peak_list[len-1].pos);
        }
#endif
        if (data_expand_peaks(data, options, message) != SUCCESS) {
            return ERROR;
        }

        if (data_resolve_peaks(data, options, message) != SUCCESS) {
            return ERROR;
        }

        num_wins = DEFAULT_NUM_WINDOWS;
        win_size = 2 * (data->pos_data_end - data->pos_data_beg) / num_wins;
        if (win_size < MIN_WIN_SIZE) {
            win_size = MIN_WIN_SIZE;
            num_wins = 2 * (data->pos_data_end - data->pos_data_beg) / win_size;
        }

        mean_spacing    = CALLOC(float, num_wins);
        var        = CALLOC(float, num_wins);

#if 0
            fprintf(stderr, "pos_data_beg=%d pos_data_end=%d\n",
                data->pos_data_beg, data->pos_data_end);
#endif

        for ( i=0; i < num_wins; i++)
        {
            win_beg =  data->pos_data_beg + win_size*i/2;
            win_end =  (win_beg + win_size  < data->pos_data_end) ?
                       (win_beg + win_size) : data->pos_data_end;

            bc_data_create_single_ordered_peak_list(data, shift_buf,
                            message);

            if (create_DP_list(data, win_beg, win_end, shift_buf, &dp, &num_DPs,
                options, message) != SUCCESS ) {
                goto error;
            }

#if 0
            fprintf(stderr, "win_ind=%d win_beg=%d win_end=%d num_DPs=%d \n",
                i, win_beg, win_end, num_DPs);
#endif
            estimate_spacing_variation_of_shifted_peaks(dp, num_DPs, 
                win_beg, win_end, shift_buf,
                0, mean_spacing+i, var+i, &std_dev, -1, &min_spacing, &max_spacing,
                &min_pos, &max_pos, &min_color0, &min_color1, data, options,
                message);

#if 0
            fprintf(stderr, "    New spacing=%f min_spacing=%d max_spacing=%d \n",
                spacing[i], min_spacing, max_spacing);
#endif
            FREE(dp);
        }

        num_windows = num_wins;
#if 0
        fprintf(stderr, "num_windows=%d spacing=\n", num_windows);
#endif
        i0 = isweet;
        spac_mod_val[i0] = mean_spacing[i0];
        for ( i=i0+1; i < num_windows; i++ )
        {
            spac_mod_val[i] = mean_spacing[i];
            if ((mean_spacing[i] <= 0) || isnan(mean_spacing[i])) {
                spac_mod_val[i] = spac_mod_val[i-1];
            }
        }
        for ( i=i0-1; i >=0; i--)
        {
            spac_mod_val[i] = mean_spacing[i];
            if ((mean_spacing[i] <= 0) || isnan(mean_spacing[i])) {
                spac_mod_val[i] = spac_mod_val[i+1];
            }
        }

        sliding_window5_average(spac_mod_val, num_windows);

        if (options->xgr) {
            output_new_spacing_curve(spac_mod_pos, spac_mod_val, num_windows);   
        }

        FREE(mean_spacing);
        FREE(var);
        return SUCCESS;
error:
        FREE(mean_spacing);
        FREE(var);
        return ERROR;
    }
#endif
    return SUCCESS;
}


/*********************************************************************
 * Function: Btk_process_raw_data
 *********************************************************************
 * This is the main routine that navigates the multiple steps of
 * data processing before any peaks are identified.
 *
 * Major steps of processing:
 *    STEP1: multicomponent (= prebaseline, decouple dye signals)
 *    STEP2: filter out noise (= smoothen)
 *    STEP3: determine first base position; optionally clip 
 *           the beginning of chromatogram
 *    STEP4: baseline
 *    STEP5: determine the end of good data; optionally prune the end 
 *           of chromatogram
 *    STEP6: normsalize
 *    STEP7: correct mobility shifts
 *
 * Inputs:   data    array of four pointers to trace data
 *           num_data  array of lengths of data arrays
 * Outputs:  *data   processed trace data
 *           *num_data length of processed data
 * Return:   error status
 * Comments:
 */
int
Btk_process_raw_data(int *num_datapoints, int **chromatogram, 
    char *color2base, Data *data, Options options, BtkMessage *message)
{
    int        i, sample_rate = 2, alloc_chromat_len = *num_datapoints * 3;
    int        init_num_datapoints = *num_datapoints;
    int        data_beg=0, data_end=*num_datapoints;
    const long start_tol = 500;  // if *data_beg > start_tol, throw away lead data

    if (options.xgr) 
        output_chromatogram("0_Raw_data.xgr", "Raw data", 
        chromatogram[0], chromatogram[1], chromatogram[2], chromatogram[3],
       *num_datapoints, data);

    /* STEP1: filter out noise */
    prefilter_and_prebaseline(num_datapoints, chromatogram, sample_rate,
        &options, message);

    if (options.xgr)
        output_chromatogram("1_Prefilt_prebaselined_data.xgr",
        "Prefiltered and prebaselined data", 
        chromatogram[0], chromatogram[1], chromatogram[2], chromatogram[3],
        *num_datapoints, data);

    /* STEP2: (optionally) multicomponent */
    if (options.multicomp) {
        multicomponent_data(*num_datapoints, chromatogram,
            &options, message, data);
    }

    /* STEP3: determine first base position and optionally clip leading data*/
    determine_beg_of_data(&data_beg, start_tol, num_datapoints, 
        chromatogram, data);

    if (options.xgr)
        output_chromatogram("3_Filt_mult_lead-pruned_data.xgr",
            "Prefiltered, multicomponented and leader-pruned data", 
            chromatogram[0], chromatogram[1], chromatogram[2], chromatogram[3],
           *num_datapoints, data);

    /* STEP4: baseline */
    baseline_data(num_datapoints, chromatogram, data);
        
    if (options.xgr) 
        output_chromatogram("4_Filt_mult_lead-pruned_basel_data.xgr",
        "Prefiltered, multicomponented, leader-pruned and baselined data", 
        chromatogram[0], chromatogram[1], chromatogram[2], chromatogram[3],
       *num_datapoints, data);

    if (options.Verbose  > 1)
        fprintf(stderr, "Number of datapoints reduced from %d to %d\n",
            init_num_datapoints, *num_datapoints);
   
    /* STEP5: determine the end of good data; optionally prune the end */
    determine_end_of_data(num_datapoints, chromatogram,
       &data_end, data, options);

    /* STEP6: process peaks and normalize data */
    if (process_peaks(chromatogram, data, &options, message) != SUCCESS)
        return ERROR;

#if 1
    if (data_normalize(chromatogram, data, &options, message) != SUCCESS)
        return ERROR;
#endif

    if (options.xgr)
         output_chromatogram("5_Analyzed_data.xgr", "Analyzed_data",
            chromatogram[0], chromatogram[1], chromatogram[2], chromatogram[3],
           *num_datapoints, data);

    data->pos_data_beg = (CLIP_DATA_BEG ? 0 : data_beg);
    data->pos_data_end = (CLIP_DATA_END ? data->color_data[0].length-1 : 
        data_end);
 
    /* STEP7: correct mobility shifts */
    get_peak_spacing(data, &options, message);
    for (i=0; i<NUM_COLORS; i++) {
        int j;
        for (j=0; j < *num_datapoints; j++)
            chromatogram[i][j] = data->color_data[i].data[j];
    }
  
    if (options.xgr) 
         output_chromatogram("6_Analyzed_data_shifted.xgr",
            "Analyzed data_shifted", 
             chromatogram[0], chromatogram[1], chromatogram[2], chromatogram[3],
            *num_datapoints, data);

    if (respace_chromatograms(chromatogram, num_datapoints, alloc_chromat_len,
        data, &options, message) != SUCCESS)
        return ERROR;

    if (options.xgr)
         output_chromatogram("7_Analyzed_data_respaced.xgr",
            "Analyzed data_shifted_and_respaced", 
            chromatogram[0], chromatogram[1], chromatogram[2], chromatogram[3],
             *num_datapoints, data);

    if (DEBUG)
    fprintf(stderr, "Btk_process_raw_data finished.\n");

    return SUCCESS;
}


