/* $Id: casu_stats.c,v 1.2 2015/08/07 13:06:54 jim Exp $
 *
 * This file is part of the CASU Pipeline utilities
 * Copyright (C) 2015 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

/*
 * $Author: jim $
 * $Date: 2015/08/07 13:06:54 $
 * $Revision: 1.2 $
 * $Name:  $
 */

/* Includes */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <math.h>
#include <string.h>
#include <cpl.h>
#include <cxtypes.h>

#include "casu_stats.h"
#include "catalogue/casu_utils.h"

/* Static subroutine prototypes */

static float kselect(float *a, long n, long k);
static double dkselect(double *a, long n, long k);
static float histexam(int *histo, int nhist, int level);

/**
   \defgroup casu_stats casu_stats
   \ingroup casu_routines

   \brief
   These are support routines used for doing statistics on data arrays

   \author
   Jim Lewis, CASU, CASU
*/

/**@{*/

/*---------------------------------------------------------------------------*/
/**
    \par Name:
        casu_med
    \par Purpose:
        Find the median of a float array with a possible bad pixel mask
    \par Description:
        If a bad pixel mask is not included a straight forward median is
        performed on the input data array. If a bad pixel mask is included
        then the data good data are copied into a temporary array and then
        the median is performed. This is a routine for floating point data.
    \par Language:
        C
    \param data
        Input data
    \param bpm 
        Input bad pixel mask or NULL
    \param npts
        Number of pixels in the data
    \retval median 
        When all goes well  
    \retval CX_MAXFLOAT 
        if all are flagged as bad
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

extern float casu_med(float *data, unsigned char *bpm, long npts) {
    long i,j,is_even,ilevel;
    float *buf,value;

    /* Is there any point being here? */

    if (npts == 0)
        return(CX_MAXFLOAT);

    /* If there is not BPM, then just do a straight forward median */

    buf = cpl_malloc(npts*sizeof(*buf));
    if (bpm == NULL) {
        is_even = !(npts & 1);
        memmove((char *)buf,(char *)data,npts*sizeof(float));
        if (is_even) {
            ilevel = npts/2 - 1;
            value = kselect(buf,npts,ilevel);
            ilevel = npts/2;
            value = 0.5*(value + kselect(buf,npts,ilevel));
        } else {
            ilevel = npts/2;
            value = kselect(buf,npts,ilevel);
        }

    /* Otherwise get rid of the dodgy values and then do the median */

    } else {
        j = 0;
        for (i = 0; i < npts; i++) {
            if (bpm[i] == 0)
                buf[j++] = data[i];
        }
        if (j == 0) {
            cpl_free(buf);
            value = CX_MAXFLOAT;
            return(value);
        }
        is_even = !(j & 1);
        if (is_even) {
            ilevel = j/2 - 1;
            value = kselect(buf,j,ilevel);
            ilevel = j/2;
            value = 0.5*(value + kselect(buf,j,ilevel));
        } else {
            ilevel = j/2;
            value = kselect(buf,j,ilevel);
        }
    }
    cpl_free(buf);
    return(value);
}

/*---------------------------------------------------------------------------*/
/**
    \par Name:
        casu_dmed
    \par Purpose:
        Find the median of a double array with a possible bad pixel mask
    \par Description:
        If a bad pixel mask is not included a straight forward median is
        performed on the input data array. If a bad pixel mask is included
        then the data good data are copied into a temporary array and then
        the median is performed. This is a routine for double precision data.
    \par Language:
        C
    \param data
        Input data
    \param bpm 
        Input bad pixel mask or NULL
    \param npts
        Number of pixels in the data
    \retval median 
        When all goes well  
    \retval CX_MAXDOUBLE
        if all are flagged as bad
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

extern double casu_dmed(double *data, unsigned char *bpm, long npts) {
    long i,j,is_even,ilevel;
    double *buf,value;

    /* If there is not BPM, then just do a straight forward median */

    buf = cpl_malloc(npts*sizeof(*buf));
    if (bpm == NULL) {
        is_even = !(npts & 1);
        memmove((char *)buf,(char *)data,npts*sizeof(double));
        if (is_even) {
            ilevel = npts/2 - 1;
            value = dkselect(buf,npts,ilevel);
            ilevel = npts/2;
            value = 0.5*(value + dkselect(buf,npts,ilevel));
        } else {
            ilevel = npts/2;
            value = dkselect(buf,npts,ilevel);
        }

    /* Otherwise get rid of the dodgy values and then do the median */

    } else {
        j = 0;
        for (i = 0; i < npts; i++) {
            if (bpm[i] == 0)
                buf[j++] = data[i];
        }
        if (j == 0) {
            cpl_free(buf);
            value = CX_MAXDOUBLE;
            return(value);
        }
        is_even = !(j & 1);
        if (is_even) {
            ilevel = j/2 - 1;
            value = dkselect(buf,j,ilevel);
            ilevel = j/2;
            value = 0.5*(value + dkselect(buf,j,ilevel));
        } else {
            ilevel = j/2;
            value = dkselect(buf,j,ilevel);
        }
    }
    cpl_free(buf);
    return(value);
}

/*---------------------------------------------------------------------------*/
/**
    \par Name:
        casu_qmedsig
    \par Purpose:
        Find a "quick" median and sigma of a floating point array with 
        a bad pixel mask, with lower and upper cuts and with clipping.
    \par Description:
        Data outside the data window and data flagged in the bad pixel mask
        are removed from further consideration. The remaining data are 
        histogrammed in 1 adu bins. A number of iterations are used to find
        the median and sigma by finding the meidan in the histogram and its
        quartiles, then using these in conjunction with user defined clipping
        thresholds to define new estimates.
    \par Language:
        C
    \param data
        Input data
    \param bpm 
        Input bad pixel mask 
    \param npts
        Number of pixels in the data
    \param thresh
        Clipping threshold
    \param niter
        Number of clipping iterations
    \param lowv
        Lower value for data window
    \param highv
        Upper value for data window
    \param median
        The output median
    \param sigma
        The output dispersion
    \return Nothing
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

extern void casu_qmedsig(float *data, unsigned char *bpm, long npts,
                         float thresh, int niter, float lowv, float highv,
                         float *median, float *sigma) {
    int *histo,nbins,nhist,ilev,iclip,nhist2,halflev,quartlev;
    int irej,jst,j;
    long i;
    float mlev,qlev;
    unsigned char *b;

    /* Right, first thing is to histogram the data.  Cut values below
       and above the 'ceiling' values */

    if (bpm == NULL) 
        b = cpl_calloc(npts,sizeof(unsigned char));
    else
        b = bpm;
    nbins = casu_nint(highv - lowv + 1.0);
    histo = cpl_calloc(nbins,sizeof(*histo));
    nhist = 0;
    for (i = 0; i < npts; i++) {
        if (b[i] || data[i] < lowv || data[i] > highv)
            continue;
        ilev = casu_nint(data[i] - lowv);
        ilev = max(0,min(nbins-1,ilev));
        histo[ilev] += 1;
        nhist += 1;
    }
    if (bpm == NULL)
        freespace(b);
    if (nhist == 0) {
        *median = CX_MAXFLOAT;
        *sigma = CX_MAXFLOAT;
        freespace(histo);
        return;
    }

    /* Right, find the median value and the first quartile. */

    iclip = nbins - 1;
    nhist2 = nhist;
    for (i = 0; i <= niter; i++) {
        halflev = (nhist2 + 1)/2;
        quartlev = (nhist2 + 3)/4;
        mlev = histexam(histo,nbins,halflev);
        *median = mlev + lowv;
        qlev = histexam(histo,nbins,quartlev);
        *sigma = (mlev - qlev)*1.48;
        if (i == niter)
            break;
        irej = 0;
        jst = casu_nint(mlev + thresh*(*sigma));
        for (j = jst; j <= iclip; j++)
            irej += histo[j];
        if (irej == 0)
            break;
        iclip = jst - 1;
        nhist2 -= irej;
    }
    cpl_free(histo);
}

/*---------------------------------------------------------------------------*/
/**
    \par Name:
        casu_medmad
    \par Purpose:
        Find the median and median absolute deviation in a data array
    \par Description:
        The median of an array is found, taking into account any flagged
        values. An array of absolute deviations is created and the median
        is found of that array.
    \par Language:
        C
    \param data
        Input data
    \param bpm 
        Input bad pixel mask or NULL
    \param np
        Number of pixels in the data
    \param med
        The output median
    \param mad
        The output median absolute deviation
    \return Nothing
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

extern void casu_medmad(float *data, unsigned char *bpm, long np, float *med,
                        float *mad) {
    long i;
    float *work;

    /* First find the median value */

    *med = casu_med(data,bpm,np);

    /* Now work out the MAD. Start by getting a bit of workspace and filling
       it with absolute residuals */
    
    work = cpl_malloc(np*sizeof(*work));
    for (i = 0; i < np; i++)
        work[i] = (float)fabs((double)(data[i] - *med));

    /* Now get the median value of the absolute deviations */

    *mad = casu_med(work,bpm,np);

    /* Tidy and exit */

    cpl_free(work);
}

/*---------------------------------------------------------------------------*/
/**
    \par Name:
        casu_medmadcut
    \par Purpose:
        Find the median and median absolute deviation in a data array with
        a high and/or low cut
    \par Description:
        The median of an array is found, taking into account any flagged
        values and any values outside a low and high data value cut. An array 
        of absolute deviations is created and the median is found of that 
        array.
    \par Language:
        C
    \param data
        Input data
    \param bpm 
        Input bad pixel mask or NULL
    \param np
        Number of pixels in the data
    \param lcut
        Lower value for data window
    \param hcut
        Upper value for data window
    \param med
        The output median
    \param mad
        The output median absolute deviation
    \return Nothing
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

extern void casu_medmadcut(float *data, unsigned char *bpm, long np, float lcut,
                           float hcut, float *med, float *mad) {
    long i;
    float *work;
    unsigned char *bad;

    /* Get a workspace for the pseudo-pabd pixel mask... */

    bad = cpl_calloc(np,sizeof(*bad));
    if (bpm != NULL) {
        for (i = 0; i < np; i++) 
            if (bpm[i] != 0 || data[i] < lcut || data[i] > hcut)
                bad[i] = 1;
    } else {
        for (i = 0; i < np; i++) 
            if (data[i] < lcut || data[i] > hcut)
                bad[i] = 1;
    }

    /* First find the median value */

    *med = casu_med(data,bad,np);
    if (*med == CX_MAXFLOAT) {
        *mad = 0.0;
        cpl_free(bad);
        return;
    }

    /* Now work out the MAD. Start by getting a bit of workspace and filling
       it with absolute residuals */
    
    work = cpl_malloc(np*sizeof(*work));
    for (i = 0; i < np; i++)
        work[i] = (float)fabs((double)(data[i] - *med));

    /* Now get the median value of the absolute deviations */

    *mad = casu_med(work,bad,np);

    /* Tidy and exit */

    cpl_free(work);
    cpl_free(bad);
}

/*---------------------------------------------------------------------------*/
/**
    \par Name:
        casu_mean
    \par Purpose:
        Find the mean of a float array with a possible bad pixel mask
    \par Description:
        If a bad pixel mask is not included a straight forward mean is
        performed on the input data array. If a bad pixel mask is included
        then the data good data are copied into a temporary array and then
        the mean is performed. This is a routine for floating point data.
    \par Language:
        C
    \param data
        Input data
    \param bpm 
        Input bad pixel mask or NULL
    \param npts
        Number of pixels in the data
    \retval mean 
        When all goes well  
    \retval CX_MAXFLOAT 
        if all are flagged as bad
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

extern float casu_mean(float *data, unsigned char *bpm, long npts) {
    long i,n;
    float sum,value;

    /* Separate sections depending on whether there is a BPM or not */

    sum = 0.0;
    if (bpm == NULL) {
        n = npts;
        for (i = 0; i < npts; i++) 
            sum += data[i];
    } else {
        n = 0;
        for (i = 0; i < npts; i++) {
            if (bpm[i] == 0) {
                sum += data[i];
                n++;
            }
        }
    }
    if (n > 0)
        value = sum/(float)n;
    else
        value = CX_MAXFLOAT;
    return(value);
}

/*---------------------------------------------------------------------------*/
/**
    \par Name:
        casu_dmean
    \par Purpose:
        Find the mean of a double array with a possible bad pixel mask
    \par Description:
        If a bad pixel mask is not included a straight forward mean is
        performed on the input data array. If a bad pixel mask is included
        then the data good data are copied into a temporary array and then
        the mean is performed. This is a routine for double precision data.
    \par Language:
        C
    \param data
        Input data
    \param bpm 
        Input bad pixel mask or NULL
    \param npts
        Number of pixels in the data
    \retval mean 
        When all goes well  
    \retval CX_MAXDOUBLE
        if all are flagged as bad
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

extern double casu_dmean(double *data, unsigned char *bpm, long npts) {
    long i,n;
    double sum,value;

    /* Separate sections depending on whether there is a BPM or not */

    sum = 0.0;
    if (bpm == NULL) {
        n = npts;
        for (i = 0; i < npts; i++) 
            sum += data[i];
    } else {
        n = 0;
        for (i = 0; i < npts; i++) {
            if (bpm[i] == 0) {
                sum += data[i];
                n++;
            }
        }
    }
    if (n > 0)
        value = sum/(float)n;
    else
        value = CX_MAXDOUBLE;
    return(value);
}

/*---------------------------------------------------------------------------*/
/**
    \par Name:
        casu_meansig
    \par Purpose:
        Find the mean and sigma of a floating point array with a possible
        bad pixel mask
    \par Description:
        A straight-forward mean and standard deviation calculation is done.
        If a bad pixel mask is included, then the flagged pixels are removed
        from the calculations.
    \par Language:
        C
    \param data
        Input data
    \param bpm 
        Input bad pixel mask or NULL
    \param npts
        Number of pixels in the data
    \param mean 
        The output mean
    \param sig
        The output dispersion
    \retval CASU_OK
        If all went well
    \retval CASU_WARN
        If the variance is negative. Sigma is passed back as CX_MAXFLOAT.
        Also if all data points are flagged. In that case both the mean
        and sigma are set to CX_MAXFLOAT.
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

extern int casu_meansig(float *data, unsigned char *bpm, long npts, 
                        float *mean, float *sig) {
    long i,n;
    double sum,sum2,d;
    const char *fctid = "casu_meansig";

    /* Separate sections depending on whether there is a BPM or not */

    sum = 0.0;
    sum2 = 0.0;
    if (bpm == NULL) {
        n = npts;
        for (i = 0; i < npts; i++) {
            d = (double)(data[i]);
            sum += d;
            sum2 += d*d;
        }
    } else {
        n = 0;
        for (i = 0; i < npts; i++) {
            if (bpm[i] == 0) {
                d = (double)(data[i]);
                sum += d;
                sum2 += d*d;
                n++;
            }
        }
    }

    /* Check whether we can do the mean and sigma calculations */

    switch (n) {
    case 0:
        *mean = CX_MAXFLOAT;
        *sig = CX_MAXFLOAT;
        cpl_msg_warning(fctid,"All values flagged as bad");
        return(CASU_WARN);
    case 1:
        *mean = (float)sum;
        *sig = 0.0;
        return(CASU_OK);
    default:
        sum /= (double)n;
        *mean = (float)sum;
        sum2 = sum2/(double)n - sum*sum;
        *sig = (float)sqrt(max(1.0e-12,sum2));
        return(CASU_OK);
    }
}

/*---------------------------------------------------------------------------*/
/**
    \par Name:
        casu_medsig
    \par Purpose:
        Find the median and sigma about that median.
    \par Description:
        The median of an array is found, taking into account any flagged
        values. The sigma is then worked out about the median.
    \par Language:
        C
    \param data
        Input data
    \param bpm 
        Input bad pixel mask or NULL
    \param np
        Number of pixels in the data
    \param med
        The output median
    \param sig
        The output sigma
    \return Nothing
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

extern void casu_medsig(float *data, unsigned char *bpm, long np, float *med,
                        float *sig) {
    long i,n;
    float sum,resid;

    /* First find the median value */

    *med = casu_med(data,bpm,np);
    if (*med == CX_MAXFLOAT) {
        *sig = 0.0;
        return;
    }

    /* If no bpm is present the just use them all */

    if (bpm == NULL) {
        sum = 0.0;
        for (i = 0; i < np; i++) {
            resid = data[i] - *med;
            sum += resid*resid;
        }
        *sig = sqrt(sum/(float)np);

    /* Otherwise test the bpm */

    } else {
        sum = 0.0;
        n = 0;
        for (i = 0; i < np; i++) {
            if (bpm[i] == 0) {
                n++;
                resid = data[i] - *med;
                sum += resid*resid;
            }
        }
        if (n > 0) 
            *sig = sqrt(sum/(float)n);
        else
            *sig = 0.0;
    }
}

/*---------------------------------------------------------------------------*/
/**
    \par Name:
        casu_sumbpm
    \par Purpose:
        Get the sum from a bad pixel mask
    \par Description:
        Sum all the values in a bad pixel mask. This is useful for telling how
        many pixels are bad.
    \par Language:
        C
    \param bpm
        Input bad pixel mask
    \param npts
        Number of pixels in the bpm
    \param sumb
        The output sum
    \retval CASU_OK
        Always
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

extern int casu_sumbpm(unsigned char *bpm, long npts, int *sumb) {
    long j;

    *sumb = 0;
    for (j = 0; j < npts; j++)
        *sumb += bpm[j];
    return(CASU_OK);
}

/**@}*/

static float histexam(int *histo, int nhist, int level) {
    int ilev,ii;
    float value;

    ii = 0;
    ilev = -1;
    while (ii < level && ilev < nhist-1)
        ii += histo[++ilev];
    value = (float)ilev - (float)(ii - level)/(float)histo[ilev] + 0.5;
    return(value);
}

/*
 * given an array a of n elements, return the element that would be at
 * position k, (0 <= k < n), if the array were sorted.  from Algorithms
 * and Data Structures in C++ by Leendert Ammeraal, pg. 82.  O(n).
 *
 * NB: partially reorders data in array
 */

/* Stolen from C. Sabbey */

static float kselect(float *a, long n, long k) {
    while (n > 1) {
        long i = 0, j = n - 1;
        float x = a[j/2], w;

        do {
            while (a[i] < x) i++;
            while (a[j] > x) j--;
            if (i < j) {
                w = a[i]; a[i] = a[j]; a[j] = w;
            } else {
                if (i == j) i++;
                break;
            }
        } while (++i <= --j);

        if (k < i)
            n = i;
        else {
            a += i; n -= i; k -= i;
        }
    }

    return a[0];
}

static double dkselect(double *a, long n, long k) {
    while (n > 1) {
        long i = 0, j = n - 1;
        double x = a[j/2], w;

        do {
            while (a[i] < x) i++;
            while (a[j] > x) j--;
            if (i < j) {
                w = a[i]; a[i] = a[j]; a[j] = w;
            } else {
                if (i == j) i++;
                break;
            }
        } while (++i <= --j);

        if (k < i)
            n = i;
        else {
            a += i; n -= i; k -= i;
        }
    }

    return a[0];
}

/*

$Log: casu_stats.c,v $
Revision 1.2  2015/08/07 13:06:54  jim
Fixed copyright to ESO

Revision 1.1.1.1  2015/06/12 10:44:32  jim
Initial import

Revision 1.4  2015/01/29 11:54:33  jim
Added medmadcut routine. Some modified comments

Revision 1.3  2014/04/26 18:38:53  jim
changed integer to long in some routines

Revision 1.2  2013/11/21 09:38:14  jim
detabbed

Revision 1.1.1.1  2013-08-27 12:07:48  jim
Imported


*/

