/* $Id: hawki_twilight_flat_combine.c,v 1.15 2015/10/24 09:21:20 jim Exp $
 *
 * This file is part of the HAWKI Pipeline
 * Copyright (C) 2015 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

/*
 * $Author: jim $
 * $Date: 2015/10/24 09:21:20 $
 * $Revision: 1.15 $
 * $Name:  $
 */

/* Includes */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <stdio.h>
#include <cpl.h>
#include <math.h>
#include <string.h>

#include "hawki_utils.h"
#include "hawki_pfits.h"
#include "hawki_dfs.h"
#include "casu_utils.h"
#include "casu_stats.h"
#include "casu_fits.h"
#include "casu_mask.h"
#include "casu_wcsutils.h"
#include "casu_mods.h"

/* Define values for bit mask that flags dummy results */

#define MEANTWI     1
#define CONFMAP     2
#define RATIMG      4
#define STATS_TAB   8
#define BPM        16

/* Structure definitions */

typedef struct {

    /* Input */

    float       lthr;
    float       hthr;
    int         combtype;
    int         scaletype;
    int         xrej;
    float       thresh;
    int         ncells;
    int         prettynames;

    /* Output */

    float       flatrms;
    float       flatratio_med;
    float       flatratio_rms;
    float       minv;
    float       maxv;
    float       avev;
    float       medv;
    float       medsig;
    float       fluxratio;
    float       photnoise;
    float       snratio;
    int         nbad;
    float       badfrac;

    /* Bitmasks to monitor the products */

    int we_expect;
    int we_get;
} configstruct;


typedef struct {
    casu_fits        **good;
    int              ngood;
    cpl_size         *labels;
    cpl_frameset     *twilightlist;
    cpl_frame        *master_dark;
    cpl_frame        *master_twilight_flat;
    casu_mask        *master_mask;

    cpl_image        *outimage;
    cpl_image        *outconf;
    cpl_image        *outbpm;
    casu_fits        **twilights;
    int              ntwilights;
    cpl_propertylist *drs;
    cpl_propertylist *drs2;
    unsigned char    *rejmask;
    unsigned char    *rejplus;
    cpl_image        *ratioimg;
    cpl_table        *ratioimstats;
    casu_fits        *mdark;
} memstruct;


/* Function prototypes */

static int hawki_twilight_flat_combine_create(cpl_plugin *) ;
static int hawki_twilight_flat_combine_exec(cpl_plugin *) ;
static int hawki_twilight_flat_combine_destroy(cpl_plugin *) ;
static int hawki_twilight_flat_combine(cpl_parameterlist *, cpl_frameset *) ;
static int hawki_twilight_flat_combine_save(cpl_frameset *framelist, 
                                            cpl_parameterlist *parlist,
                                            int isfirst, memstruct *ps,
                                            configstruct *cs,
                                            cpl_frame **product_frame_mean_twi,
                                            cpl_frame **product_frame_conf,
                                            cpl_frame **product_frame_ratioimg,
                                            cpl_frame **product_frame_ratioimg_stats,
                                            cpl_frame **product_frame_bpm);
static void hawki_twilight_flat_combine_dummy_products(memstruct *ps,
                                                       configstruct *cs);
static void hawki_twilight_flat_combine_normal(int jext, memstruct *ps,
                                               configstruct *cs);
static int hawki_twilight_flat_combine_lastbit(int jext, 
                                               cpl_frameset *framelist,
                                               cpl_parameterlist *parlist,
                                               int isfirst, memstruct *ps,
                                               configstruct *cs,
                                               cpl_frame **product_frame_mean_twi,
                                               cpl_frame **product_frame_conf,
                                               cpl_frame **product_frame_ratioimg,
                                               cpl_frame **product_frame_ratioimg_stats,
                                               cpl_frame **product_frame_bpm);
static int hawki_twilight_flat_cull(cpl_frameset **list, configstruct *cs);
static int hawki_compare_fnames(const cpl_frame *f1, const cpl_frame *f2);
static void hawki_twilight_flat_combine_init(memstruct *ps);
static void hawki_twilight_flat_combine_tidy(memstruct *ps, int level);


static char hawki_twilight_flat_combine_description[] =
"hawki_twilight_flat_combine -- HAWKI twilight flat combine recipe.\n\n"
"Combine a list of twilight flat frames into a mean frame. Optionally\n"
"compare the output frame to a master twilight flat frame\n\n"
"The program accepts the following files in the SOF:\n\n"
"    Tag                   Description\n"
"    -----------------------------------------------------------------------\n"
"    %-21s A list of raw twilight flat images\n"
"    %-21s A master dark frame\n"
"    %-21s Optional reference twilight flat frame\n"
"    %-21s Optional master bad pixel map or\n"
"    %-21s Optional master confidence map\n"
"If no reference twilight flat is made available, then no comparison will be\n"
"done. This means there will be no output ratio image. If either a master"
"bad pixel mask or a master confidence map is specified, this will be used"
"to help mask out bad pixels\n"
"\n";

/**@{*/

/**
    \ingroup recipelist
    \defgroup hawki_twilight_flat_combine hawki_twilight_flat_combine
    \brief Combine a series of twilight flat exposures into a single mean 
           frame

    \par Name: 
        hawki_twilight_flat_combine
    \par Purpose: 
        Combine a series of twilight flat exposures into a single mean frame
    \par Description: 
        A list of twilight flats is corrected with an appropriate master dark
        frame. The dark corrected images are combined with rejection to form 
        a mean twilight flat. If a reference twilight flat is supplied, then 
        a ratio image is formed between it and the combined result from the 
        current frame list. This ratio image can be useful for looking at the 
        evolution of the structure of the flat field. Then the ratio image is 
        broken into lots of little cells. The median value of the ratio image 
        as well as the RMS in each cell is written to a ratio image statistics 
        table.
    \par Language:
        C
    \par Parameters:
        - \b lthr (float): The low threshold below which an image is to be
             considered underexposed.
        - \b hthr (float): The high threshold above which an image is to be
             considered overexposed.
        - \b combtype (string): Determines the type of combination that is 
             done to form the output map. Can take the following values:
            - median: The output pixels are medians of the input pixels
            - mean: The output pixels are means of the input pixels
        - \b scaletype (int): Determines how the input data are scaled or 
             offset before they are combined. Can take the following values:
            - none: No scaling of offsetting
            - additive: All input frames are biassed additively to bring their
              backgrounds to a common mean.
            - multiplicative: All input frames are scaled multiplicatively to 
              bring their backgrounds to a common mean.
            - exptime: All input frames are scaled to a uniform exposure time 
              and then additively corrected to bring their backgrounds to a 
              common mean.
        - \b xrej (int): If set, then an extra rejection cycle will be run. 
        - \b thresh (float): The rejection threshold in numbers of sigmas.
        - \b ncells (int): If a ratio image statistics table is being 
             done, then this is the number of cells in which to divide each
             readout channel. The value should be a power of 2, up to 64.
        - \b prettynames (bool): If set then a descriptive file name will
          be used for the output products. Otherwise the standard ESO 
          'predictable' name will be used.
    \par Input File Types:
        The following list gives the file types that can appear in the SOF
        file. The word in bold is the DO category value.
        - \b FLAT_TWILIGHT (required): A list of raw twilight flat images for 
             combining
        - \b MASTER_DARK (required): A master dark frame. This is needed 
             for the dark correction of the input twilight flat images.
        - \b REFERENCE_TWILIGHT_FLAT (optional): A library twilight flat frame
             from a previous run. If this is given, then a ratio image will be 
             formed and some basic statistics run on it.
        - \b MASTER_BPM or \b MASTER_CONF (optional): If either is given, then
             it will be used to mask out bad pixels during and statistical 
             analysis that is done. 
    \par Output Products:
        The following list gives the output data products that are generated
        by this recipe. The word in bold gives the DPR CATG keyword value for
        each product:
        - The output mean/median twilight frame, formed by either a mean or 
          median combination of the dark corrected input frames with rejection 
          (\b MASTER_TWILIGHT_FLAT).
        - The output ratio image, formed by dividing the input reference
          twilight frame into the current mean/median twilight frame. This is 
          only done if a reference twilight flat frame is specified from the 
          outset. (\b RATIOIMG_TWILIGHT_FLAT)
        - The output ratio image statistics table. 
          (\b RATIOIMG_STATS_TWLIGHT_FLAT).
        - The output master confidence map (\b MASTER_CONF).
    \par Output QC Parameters:
        - \b FLATRMS
          The RMS value of the mean twilight frame.
        - \b FLATRATIO_MED
          The median of the twilight flat ratio image
        - \b FLATRATIO_RMS
          The RMS of the twilight flat ratio image
        - \b FLAT MEDMIN
          The ensemble minimum of all the input images
        - \b FLAT MEDMAX
          The ensemble maximum of all the input images
        - \b FLAT MEDMEAN
          The ensemble average of all the input images
        - \b FLAT MEDMED
          The ensemble median of all the input images
        - \b FLAT MEDSTDEV
          The ensemble RMS of all the input images
        - \b FLUX RATIO
          The ratio of the ensemble max to the ensemble min
        - \b FLATRNG
          The ensemble range of the all input images
        - \b TWIPHOT
          The photon noise estimated from the middle two of the input 
          images (in ADU).
        - \b TWISNRATIO
          The estimated s/n ratio from the middle two of the input images
    \par Notes
        None.
    \par Fatal Error Conditions:
        - NULL input frameset
        - Input frameset headers incorrect meaning that RAW and CALIB frame
          can't be distinguished
        - No twilight flat frames in the input frameset
        - No master dark frame in input frameset
        - Unable to save data products
    \par Non-Fatal Error Conditions:
        - No reference twilight flat. No ratio image formed.
        - No master bad pixel or confidence map. All pixels considered to be 
          good.
    \par Conditions Leading To Dummy Products:
        - Twilight frame image extensions wouldn't load.
        - The detector for the current image extension is flagged dead
        - All images are either below the under-exposure threshold or above
          the over exposure threshold.
        - Can't load the master dark image extension or master dark image
          is a dummy
        - Combination routine failed
        - Master twilight frame image extension won't load or is flagged as a 
          dummy
    \par Author:
        Jim Lewis, CASU
    \par Code Reference: 
        hawki_twilight_flat_combine.c
*/

/* Function code */

/*---------------------------------------------------------------------------*/
/**
  @brief    Build the list of available plugins, for this module.
  @param    list    the plugin list
  @return   0 if everything is ok

  This function is exported.
 */
/*---------------------------------------------------------------------------*/

int cpl_plugin_get_info(cpl_pluginlist *list) {
    cpl_recipe  *recipe = cpl_calloc(1,sizeof(*recipe));
    cpl_plugin  *plugin = &recipe->interface;
    char alldesc[SZ_ALLDESC];
    (void)snprintf(alldesc,SZ_ALLDESC,hawki_twilight_flat_combine_description,
                   HAWKI_TWI_RAW,HAWKI_CAL_DARK,HAWKI_REF_TWILIGHT_FLAT,
                   HAWKI_CAL_BPM,HAWKI_CAL_CONF);

    cpl_plugin_init(plugin,
                    CPL_PLUGIN_API,
                    HAWKI_BINARY_VERSION,
                    CPL_PLUGIN_TYPE_RECIPE,
                    "hawki_twilight_flat_combine",
                    "HAWKI twilight combination recipe",
                    alldesc,
                    "Jim Lewis",
                    "jrl@ast.cam.ac.uk",
                    hawki_get_license(),
                    hawki_twilight_flat_combine_create,
                    hawki_twilight_flat_combine_exec,
                    hawki_twilight_flat_combine_destroy);

    cpl_pluginlist_append(list,plugin);

    return(0);
}

/*---------------------------------------------------------------------------*/
/**
  @brief    Setup the recipe options
  @param    plugin  the plugin
  @return   0 if everything is ok

  Create the recipe instance and make it available to the application using the
  interface.
 */
/*---------------------------------------------------------------------------*/

static int hawki_twilight_flat_combine_create(cpl_plugin *plugin) {
    cpl_recipe      *recipe;
    cpl_parameter   *p;

    /* Get the recipe out of the plugin */

    if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
        recipe = (cpl_recipe *)plugin;
    else 
        return(-1);

    /* Create the parameters list in the cpl_recipe object */

    recipe->parameters = cpl_parameterlist_new();

    /* Lower threshold for rejecting underexposed images */

    p = cpl_parameter_new_value("hawki.hawki_twilight_flat_combine.lthr",
                                CPL_TYPE_DOUBLE,
                                "Low rejection threshold for underexpsed images",
                                "hawki.hawki_twilight_flat_combine",
                                4000.0);
    cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"lthr");
    cpl_parameterlist_append(recipe->parameters,p);

    /* Upper threshold for rejecting overexposed images */

    p = cpl_parameter_new_value("hawki.hawki_twilight_flat_combine.hthr",
                                CPL_TYPE_DOUBLE,
                                "High rejection threshold for overexposed images",
                                "hawki.hawki_twilight_flat_combine",
                                30000.0);
    cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"hthr");
    cpl_parameterlist_append(recipe->parameters,p);

    /* Fill in the parameters. First the combination type */

    p = cpl_parameter_new_enum("hawki.hawki_twilight_flat_combine.combtype",
                               CPL_TYPE_STRING,
                               "Combination algorithm",
                               "hawki.hawki_twilight_flat_combine",
                               "median",2,"median","mean");
    cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"combtype");
    cpl_parameterlist_append(recipe->parameters,p);

    /* The requested scaling */

    p = cpl_parameter_new_enum("hawki.hawki_twilight_flat_combine.scaletype",
                               CPL_TYPE_STRING,
                               "Scaling algorithm",
                               "hawki.hawki_twilight_flat_combine",
                               "multiplicative",4,"none","additive",
                               "multiplicative","exptime");
    cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"scaletype");
    cpl_parameterlist_append(recipe->parameters,p);
    
    /* Extra rejection cycle */

    p = cpl_parameter_new_value("hawki.hawki_twilight_flat_combine.xrej",
                                CPL_TYPE_BOOL,
                                "True if using extra rejection cycle",
                                "hawki.hawki_twilight_flat_combine",
                                TRUE);
    cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"xrej");
    cpl_parameterlist_append(recipe->parameters,p);

    /* Rejection threshold */

    p = cpl_parameter_new_range("hawki.hawki_twilight_flat_combine.thresh",
                                CPL_TYPE_DOUBLE,
                                "Rejection threshold in sigma above background",
                                "hawki.hawki_twilight_flat_combine",5.0,
                                1.0e-6,1.0e10);
    cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"thresh");
    cpl_parameterlist_append(recipe->parameters,p);

    /* How many cells to divide each data channel */

    p = cpl_parameter_new_enum("hawki.hawki_twilight_flat_combine.ncells",
                               CPL_TYPE_INT,
                               "Number of cells for data channel stats",
                               "hawki.hawki_twilight_flat_combine",8,7,1,2,4,
                               8,16,32,64);
    cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"ncells");
    cpl_parameterlist_append(recipe->parameters,p);     

    /* Flag to use pretty output product file names */

    p = cpl_parameter_new_value("hawki.hawki_twilight_flat_combine.prettynames",
                                CPL_TYPE_BOOL,
                                "Use pretty output file names?",
                                "hawki.hawki_twilight_flat_combine",FALSE);
    cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"prettynames");
    cpl_parameterlist_append(recipe->parameters,p);     

    /* Get out of here */

    return(0);
}
    
    
/*---------------------------------------------------------------------------*/
/**
  @brief    Execute the plugin instance given by the interface
  @param    plugin  the plugin
  @return   0 if everything is ok
 */
/*---------------------------------------------------------------------------*/

static int hawki_twilight_flat_combine_exec(cpl_plugin *plugin) {
    cpl_recipe  *recipe;

    /* Get the recipe out of the plugin */

    if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
        recipe = (cpl_recipe *)plugin;
    else 
        return(-1);

    return(hawki_twilight_flat_combine(recipe->parameters,recipe->frames));
}
                                
/*---------------------------------------------------------------------------*/
/**
  @brief    Destroy what has been created by the 'create' function
  @param    plugin  the plugin
  @return   0 if everything is ok
 */
/*---------------------------------------------------------------------------*/

static int hawki_twilight_flat_combine_destroy(cpl_plugin *plugin) {
    cpl_recipe *recipe ;

    /* Get the recipe out of the plugin */

    if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
        recipe = (cpl_recipe *)plugin;
    else 
        return(-1);

    cpl_parameterlist_delete(recipe->parameters);
    return(0);
}

/*---------------------------------------------------------------------------*/
/**
  @brief    The recipe data reduction part is implemented here 
  @param    parlist     the parameters list
  @param    framelist   the frames list
  @return   0 if everything is ok
 */
/*---------------------------------------------------------------------------*/

static int hawki_twilight_flat_combine(cpl_parameterlist *parlist, 
                                       cpl_frameset *framelist) {
    const char *fctid="hawki_twilight_flat_combine";
    int j,retval,status,live,nx,ny,ndit,npts,isfirst;
    cpl_size nlab;
    long i;
    cpl_parameter *p;
    casu_fits *ff;
    cpl_propertylist *pp;
    cpl_image *im1,*im2,*newim,*diffim;
    cpl_frame *product_frame_mean_twi,*product_frame_conf;
    cpl_frame *product_frame_ratioimg,*product_frame_ratioimg_stats;
    cpl_frame *product_frame_bpm;
    cpl_vector *res;
    double val1,val2,scl;
    float *data,med,mad;
    unsigned char *bpm;
    memstruct ps;
    configstruct cs;

    /* Check validity of input frameset */

    if (framelist == NULL || cpl_frameset_get_size(framelist) <= 0) {
        cpl_msg_error(fctid,"Input framelist NULL or has no input data");
        return(-1);
    }

    /* Check the files in the frameset */

    if (casu_frameset_fexists(framelist) != CASU_OK) {
        cpl_msg_error(fctid,"Input frameset is missing files. Check SOF");
        return(-1);
    }

    /* Initialise some things */

    hawki_twilight_flat_combine_init(&ps);
    cs.we_expect = MEANTWI + CONFMAP + BPM;

    /* Get the parameters */

    p = cpl_parameterlist_find(parlist,"hawki.hawki_twilight_flat_combine.lthr");
    cs.lthr = (float)cpl_parameter_get_double(p);
    p = cpl_parameterlist_find(parlist,
                               "hawki.hawki_twilight_flat_combine.hthr");
    cs.hthr = (float)cpl_parameter_get_double(p);
    p = cpl_parameterlist_find(parlist,
                               "hawki.hawki_twilight_flat_combine.combtype");
    cs.combtype = (strcmp(cpl_parameter_get_string(p),"median") ? 2 : 1);
    p = cpl_parameterlist_find(parlist,
                               "hawki.hawki_twilight_flat_combine.scaletype");
    if (! strcmp(cpl_parameter_get_string(p),"none")) 
        cs.scaletype = 0;
    else if (! strcmp(cpl_parameter_get_string(p),"additive")) 
        cs.scaletype = 1;
    else if (! strcmp(cpl_parameter_get_string(p),"multiplicative")) 
        cs.scaletype = 2;
    else if (! strcmp(cpl_parameter_get_string(p),"exptime")) 
        cs.scaletype = 3;
    p = cpl_parameterlist_find(parlist,
                               "hawki.hawki_twilight_flat_combine.xrej");
    cs.xrej = cpl_parameter_get_bool(p);
    p = cpl_parameterlist_find(parlist,
                               "hawki.hawki_twilight_flat_combine.thresh");
    cs.thresh = (float)cpl_parameter_get_double(p);
    p = cpl_parameterlist_find(parlist,
                               "hawki.hawki_twilight_flat_combine.ncells");
    cs.ncells = cpl_parameter_get_int(p);
    p = cpl_parameterlist_find(parlist,
                               "hawki.hawki_twilight_flat_combine.prettynames");
    cs.prettynames = (cpl_parameter_get_bool(p) ? 1 : 0);

    /* Sort out raw from calib frames */

    if (hawki_dfs_set_groups(framelist) != CASU_OK) {
        cpl_msg_error(fctid,"Cannot identify RAW and CALIB frames");
        hawki_twilight_flat_combine_tidy(&ps,0);
        return(-1);
    }

    /* Get the twilight frames */

    if ((ps.labels = cpl_frameset_labelise(framelist,casu_compare_tags,
                                           &nlab)) == NULL) {
        cpl_msg_error(fctid,"Cannot labelise the input frames");
        hawki_twilight_flat_combine_tidy(&ps,0);
        return(-1);
    }
    if ((ps.twilightlist = casu_frameset_subgroup(framelist,ps.labels,nlab,
                                                  HAWKI_TWI_RAW)) == NULL) {
        cpl_msg_error(fctid,"Cannot find twilight frames in input frameset");
        hawki_twilight_flat_combine_tidy(&ps,0);
        return(-1);
    }
    ps.ntwilights = cpl_frameset_get_size(ps.twilightlist);

    /* Check to see if there is a master dark frame */

    if ((ps.master_dark = casu_frameset_subgroup_1(framelist,ps.labels,nlab,
                                                   HAWKI_CAL_DARK)) == NULL) {
        cpl_msg_error(fctid,"No master dark found");
        hawki_twilight_flat_combine_tidy(&ps,0);
        return(-1);
    }
        
    /* Check to see if there is a reference twilight flat frame */

    if ((ps.master_twilight_flat = casu_frameset_subgroup_1(framelist,
            ps.labels,nlab,HAWKI_REF_TWILIGHT_FLAT)) == NULL)
        cpl_msg_info(fctid,"No master twilight flat found -- no ratio image will be formed");
    else
        cs.we_expect |= RATIMG;
        
    /* Check to see if there is a master bad pixel map. If there isn't one 
       then look for a confidence map */

    ps.master_mask = casu_mask_define(framelist,ps.labels,nlab,HAWKI_CAL_CONF,
                                      HAWKI_CAL_BPM);
    if (cs.we_expect & RATIMG) 
        cs.we_expect |= STATS_TAB;

    /* Redefine the twilight list to include only those that are between the
       low and high threshold levels */

    if (hawki_twilight_flat_cull(&(ps.twilightlist),&cs) != 0) {
        hawki_twilight_flat_combine_tidy(&ps,0);
        return(-1);
    }
    ps.ntwilights = cpl_frameset_get_size(ps.twilightlist);

    /* Get the number of DITs */

    pp = cpl_propertylist_load(cpl_frame_get_filename(cpl_frameset_get_position(ps.twilightlist,0)),0);
    if (hawki_pfits_get_ndit(pp,&ndit) != CASU_OK) {
        cpl_msg_error(fctid,"No value for NDIT available");
        freepropertylist(pp);
        hawki_twilight_flat_combine_tidy(&ps,0);
        return(-1);
    }
    cpl_propertylist_delete(pp);

    /* Get some space for the good frames */

    ps.good = cpl_malloc(ps.ntwilights*sizeof(casu_fits *));

    /* Now loop for all the extensions... */

    for (j = 1; j <= HAWKI_NEXTN; j++) {
        status = CASU_OK;
        cs.we_get = 0;
        isfirst = (j == 1);

        /* Load the images and the master dark. */

        ps.twilights = casu_fits_load_list(ps.twilightlist,CPL_TYPE_FLOAT,j);
        if (ps.twilights == NULL) {
            cpl_msg_info(fctid,
                         "Extension %" CPL_SIZE_FORMAT " twilights wouldn't load",
                         (cpl_size)j);
            retval = hawki_twilight_flat_combine_lastbit(j,framelist,parlist,
                                                         isfirst, &ps, &cs,
                                                         &product_frame_mean_twi,
                                                         &product_frame_conf,
                                                         &product_frame_ratioimg,
                                                         &product_frame_ratioimg_stats,
                                                         &product_frame_bpm);
                                                         
            if (retval != 0)
                return(-1);
            continue;
        }

        /* Are any of these twilight flats any good? */
        
        ps.ngood = 0;
        for (i = 0; i < ps.ntwilights; i++) {
            ff = ps.twilights[i];
            hawki_pfits_get_detlive(casu_fits_get_ehu(ff),&live);
            if (! live) {
                cpl_msg_info(fctid,"Detector flagged dead %s",
                             casu_fits_get_fullname(ff));
                casu_fits_set_error(ff,CASU_FATAL);
            } else {
                ps.good[ps.ngood] = ff;
                ps.ngood += 1;
            }
        }

        /* If there are no good images, then signal that we need to create 
           dummy products and move on */

        if (ps.ngood == 0) {
            cpl_msg_info(fctid,"All images flagged bad for this extension");
            retval = hawki_twilight_flat_combine_lastbit(j,framelist,parlist,
                                                         isfirst, &ps, &cs,
                                                         &product_frame_mean_twi,
                                                         &product_frame_conf,
                                                         &product_frame_ratioimg,
                                                         &product_frame_ratioimg_stats,
                                                         &product_frame_bpm);
            if (retval != 0)
                return(-1);
            continue;
        }

        /* Load up the mask */

        nx = (int)cpl_image_get_size_x(casu_fits_get_image(ps.good[0]));
        ny = (int)cpl_image_get_size_y(casu_fits_get_image(ps.good[0]));
        if (casu_mask_load(ps.master_mask,j,nx,ny) == CASU_FATAL) {
            cpl_msg_info(fctid,
                         "Unable to load mask image %s[%" CPL_SIZE_FORMAT "]",
                         casu_mask_get_filename(ps.master_mask),
                         (cpl_size)j);
            cpl_msg_info(fctid,"Forcing all pixels to be good from now on");
            casu_mask_force(ps.master_mask,nx,ny);
        }

        /* Get some stats on the input flats */

        res = cpl_vector_new(ps.ngood);
        for (i = 0; i < ps.ngood; i++) {
            im1 = casu_fits_get_image(ps.good[i]); 
            val1 = cpl_image_get_median_window(im1,768,768,1791,1791);
            cpl_vector_set(res,i,val1);
        }
        cs.minv = (float)cpl_vector_get_min(res);
        cs.maxv = (float)cpl_vector_get_max(res);
        cs.avev = (float)cpl_vector_get_mean(res);
        cs.medv = (float)cpl_vector_get_median(res);
        cs.medsig = (float)cpl_vector_get_stdev(res);
        if (cs.minv != 0.0)
            cs.fluxratio = cs.maxv/cs.minv;
        else
            cs.fluxratio = 0.0;
        cpl_vector_delete(res);

        /* Take the middle two exposures and create a scaled difference image
           estimate */

        if (ps.ngood > 1) {
            i = ps.ngood/2 - 1;
            im1 = casu_fits_get_image(ps.good[i]);
            im2 = casu_fits_get_image(ps.good[i+1]);
            val1 = cpl_image_get_median_window(im1,500,500,1000,1000);
            val2 = cpl_image_get_median_window(im2,500,500,1000,1000);
            val1 /= (double)ndit;
            val2 /= (double)ndit;
            scl = val1/val2;
            newim = cpl_image_multiply_scalar_create(im2,scl);
            diffim = cpl_image_subtract_create(im1,newim);
            cpl_image_delete(newim);
            data = cpl_image_get_data_float(diffim);
            bpm = casu_mask_get_data(ps.master_mask);
            npts = nx*ny;
            casu_medmad(data,bpm,npts,&med,&mad);
            mad *= 1.48/CPL_MATH_SQRT2;
            cs.photnoise = mad;
            cs.snratio = val1*sqrt((double)(ps.ngood))/mad;
            cpl_image_delete(diffim);
        } else {
            cs.photnoise = 0.0;
            cs.snratio = 0.0;
        }

        /* Right, we want to dark correct, so we need to load the mean 
           dark and make sure it isn't a dummy */

        ps.mdark = casu_fits_load(ps.master_dark,CPL_TYPE_FLOAT,j);
        if (ps.mdark == NULL) {
            cpl_msg_info(fctid,
                         "Can't load master dark for extension %" CPL_SIZE_FORMAT,
                         (cpl_size)j);
            retval = hawki_twilight_flat_combine_lastbit(j,framelist,parlist,
                                                         isfirst, &ps, &cs,
                                                         &product_frame_mean_twi,
                                                         &product_frame_conf,
                                                         &product_frame_ratioimg,
                                                         &product_frame_ratioimg_stats,
                                                         &product_frame_bpm);

            if (retval != 0)
                return(-1);
            continue;
        } else if (casu_is_dummy(casu_fits_get_ehu(ps.mdark))) {
            cpl_msg_info(fctid,
                         "Master dark extension %" CPL_SIZE_FORMAT " is a dummy",
                         (cpl_size)j);
            retval = hawki_twilight_flat_combine_lastbit(j,framelist,parlist,
                                                         isfirst, &ps, &cs,
                                                         &product_frame_mean_twi,
                                                         &product_frame_conf,
                                                         &product_frame_ratioimg,
                                                         &product_frame_ratioimg_stats,
                                                         &product_frame_bpm);
            if (retval != 0)
                return(-1);
            continue;
        }

        /* Loop for each image and dark correct */

        cpl_msg_info(fctid,"Dark correcting extension %" CPL_SIZE_FORMAT,
                     (cpl_size)j);
        for (i = 0; i < ps.ngood; i++)
            casu_darkcor((ps.good)[i],ps.mdark,1.0,&status);
                
        /* Call the combine module */

        cpl_msg_info(fctid,
                     "Doing combination for extension %" CPL_SIZE_FORMAT,
                     (cpl_size)j);
        (void)casu_imcombine(ps.good,NULL,ps.ngood,cs.combtype,cs.scaletype,
                             cs.xrej,cs.thresh,"EXPTIME",&(ps.outimage),NULL,
                             &(ps.rejmask),&(ps.rejplus),&(ps.drs),&status);

        /* If these correction and combination routines failed at any stage
           then get out of here */

        if (status == CASU_OK) {
            cs.we_get |= MEANTWI;
            hawki_twilight_flat_combine_normal(j,&ps,&cs);
        } else {
            cpl_msg_info(fctid,"A processing step failed");
        }

        /* Create any dummies and save the products */
        
        retval = hawki_twilight_flat_combine_lastbit(j,framelist,parlist,
                                                     isfirst, &ps, &cs,
                                                     &product_frame_mean_twi,
                                                     &product_frame_conf,
                                                     &product_frame_ratioimg,
                                                     &product_frame_ratioimg_stats,
                                                     &product_frame_bpm);
        if (retval != 0)
            return(-1);

    }
    hawki_twilight_flat_combine_tidy(&ps,0);
    return(0);
}

/*---------------------------------------------------------------------------*/
/**
  @brief    The recipe data products are saved here
  @param    framelist    The input frame list
  @param    parlist      The input recipe parameter list
  @param    isfirst      Set if this is the first extension to be saved
  @param    ps           The memory structure defined in the main routine
  @param    cs           The configuration structure defined in the main routine
  @param    product_frame_mean_twi    The frame for the mean twilight product
  @param    product_frame_conf        The frame for the mean conf map product
  @param    product_frame_ratioimg    The frame for the difference image product
  @param    product_frame_ratioimg_stats The frame for the ratioimg stats table
  @param    product_frame_bpm            The frame for the BPM product
  @return   0 if everything is ok
 */
/*---------------------------------------------------------------------------*/

static int hawki_twilight_flat_combine_save(cpl_frameset *framelist, 
                                            cpl_parameterlist *parlist,
                                            int isfirst, memstruct *ps,
                                            configstruct *cs,
                                            cpl_frame **product_frame_mean_twi,
                                            cpl_frame **product_frame_conf,
                                            cpl_frame **product_frame_ratioimg,
                                            cpl_frame **product_frame_ratioimg_stats,
                                            cpl_frame **product_frame_bpm) {
    cpl_propertylist *plist,*elist,*p;
    int status,i,night;
    float val;
    const char *fctid = "hawki_twilight_flat_combine_save";
    const char *esoout[] = {"twilightcomb.fits","twilightratio.fits",
                            "twilightratiotab.fits","twilightconf.fits",
                            "bpmmap.fits"};
    const char *prettypfx[] = {"flat","ratio","ratiotab","conf","bpm"};
    char outfile[5][BUFSIZ],dateobs[81],filter[16];
    const char *recipeid = "hawki_twilight_flat_combine";

    /* Get info to make pretty names if requested. */

    for (i = 0; i < 5; i++) 
        strcpy(outfile[i],esoout[i]);
    if (cs->prettynames) {
        plist = casu_fits_get_phu(ps->twilights[0]);
        if (hawki_pfits_get_dateobs(plist,dateobs) != CASU_OK ||
            hawki_pfits_get_filter(plist,filter) != CASU_OK) {
            cpl_msg_warning(fctid,"Missing header information. Reverting to predictable names");
        } else {
            night = casu_night_from_dateobs(dateobs);
            for (i = 0; i < 5; i++) 
                (void)sprintf(outfile[i],"%s_%s_%08d.fits",filter,prettypfx[i],
                              night);
        }
    }

    /* If we need to make a PHU then do that now based on the first frame
       in the input frame list */

    if (isfirst) {

        /* Create a new product frame object and define some tags */

        *product_frame_mean_twi = cpl_frame_new();
        cpl_frame_set_filename(*product_frame_mean_twi,outfile[0]);
        cpl_frame_set_tag(*product_frame_mean_twi,HAWKI_PRO_TWILIGHT_FLAT);
        cpl_frame_set_type(*product_frame_mean_twi,CPL_FRAME_TYPE_IMAGE);
        cpl_frame_set_group(*product_frame_mean_twi,CPL_FRAME_GROUP_PRODUCT);
        cpl_frame_set_level(*product_frame_mean_twi,CPL_FRAME_LEVEL_FINAL);

        /* Set up the PHU header */

        plist = casu_fits_get_phu(ps->twilights[0]);
        hawki_dfs_set_product_primary_header(plist,*product_frame_mean_twi,
                                             framelist,parlist,
                                             recipeid,
                                             "PRO-1.15",NULL,0);

        /* 'Save' the PHU image */                         

        if (cpl_image_save(NULL,outfile[0],CPL_TYPE_UCHAR,plist,
                           CPL_IO_DEFAULT) != CPL_ERROR_NONE) {
            cpl_msg_error(fctid,"Cannot save product PHU");
            cpl_frame_delete(*product_frame_mean_twi);
            return(-1);
        }
        cpl_frameset_insert(framelist,*product_frame_mean_twi);

        /* Create a new product frame object and define some tags for the
           output confidence map */

        *product_frame_conf = cpl_frame_new();
        cpl_frame_set_filename(*product_frame_conf,outfile[3]);
        cpl_frame_set_tag(*product_frame_conf,HAWKI_PRO_CONF);
        cpl_frame_set_type(*product_frame_conf,CPL_FRAME_TYPE_IMAGE);
        cpl_frame_set_group(*product_frame_conf,CPL_FRAME_GROUP_PRODUCT);
        cpl_frame_set_level(*product_frame_conf,CPL_FRAME_LEVEL_FINAL);

        /* Set up the PHU header */

        plist = casu_fits_get_phu(ps->twilights[0]);
        hawki_dfs_set_product_primary_header(plist,*product_frame_conf,
                                             framelist,parlist,
                                             recipeid,"PRO-1.15",
                                             NULL,0);

        /* 'Save' the PHU image */                         

        if (cpl_image_save(NULL,outfile[3],CPL_TYPE_UCHAR,plist,
                           CPL_IO_DEFAULT) != CPL_ERROR_NONE) {
            cpl_msg_error(fctid,"Cannot save product PHU");
            cpl_frame_delete(*product_frame_conf);
            return(-1);
        }
        cpl_frameset_insert(framelist,*product_frame_conf);

        /* Create a new product frame object and define some tags for the
           output bad pixel mask */

        *product_frame_bpm = cpl_frame_new();
        cpl_frame_set_filename(*product_frame_bpm,outfile[4]);
        cpl_frame_set_tag(*product_frame_bpm,HAWKI_PRO_BPM);
        cpl_frame_set_type(*product_frame_bpm,CPL_FRAME_TYPE_IMAGE);
        cpl_frame_set_group(*product_frame_bpm,CPL_FRAME_GROUP_PRODUCT);
        cpl_frame_set_level(*product_frame_bpm,CPL_FRAME_LEVEL_FINAL);

        /* Set up the PHU header */

        plist = casu_fits_get_phu(ps->twilights[0]);
        hawki_dfs_set_product_primary_header(plist,*product_frame_bpm,
                                             framelist,parlist,
                                             recipeid,"PRO-1.15",
                                             NULL,0);

        /* 'Save' the PHU image */                         

        if (cpl_image_save(NULL,outfile[4],CPL_TYPE_UCHAR,plist,
                           CPL_IO_DEFAULT) != CPL_ERROR_NONE) {
            cpl_msg_error(fctid,"Cannot save product PHU");
            cpl_frame_delete(*product_frame_bpm);
            return(-1);
        }
        cpl_frameset_insert(framelist,*product_frame_bpm);

        /* Create a new product frame object for the difference image */

        if (cs->we_expect & RATIMG) {
            *product_frame_ratioimg = cpl_frame_new();
            cpl_frame_set_filename(*product_frame_ratioimg,outfile[1]);
            cpl_frame_set_tag(*product_frame_ratioimg,
                              HAWKI_PRO_RATIOIMG_TWILIGHT_FLAT);
            cpl_frame_set_type(*product_frame_ratioimg,CPL_FRAME_TYPE_IMAGE);
            cpl_frame_set_group(*product_frame_ratioimg,
                                CPL_FRAME_GROUP_PRODUCT);
            cpl_frame_set_level(*product_frame_ratioimg,CPL_FRAME_LEVEL_FINAL);

               /* Set up the PHU header */

            plist = casu_fits_get_phu(ps->twilights[0]);
            hawki_dfs_set_product_primary_header(plist,*product_frame_ratioimg,
                                                 framelist,parlist,
                                                 recipeid,"PRO-1.15",
                                                 NULL,0);

            /* 'Save' the PHU image */                         

            if (cpl_image_save(NULL,outfile[1],CPL_TYPE_UCHAR,plist,
                               CPL_IO_DEFAULT) != CPL_ERROR_NONE) {
                cpl_msg_error(fctid,"Cannot save product PHU");
                cpl_frame_delete(*product_frame_ratioimg);
                return(-1);
            }
            cpl_frameset_insert(framelist,*product_frame_ratioimg);
        }

        /* Create a new product frame object for the difference image stats 
           table */

        if (cs->we_expect & STATS_TAB) {
            *product_frame_ratioimg_stats = cpl_frame_new();
            cpl_frame_set_filename(*product_frame_ratioimg_stats,outfile[2]);
            cpl_frame_set_tag(*product_frame_ratioimg_stats,
                              HAWKI_PRO_RATIOIMG_TWILIGHT_FLAT_STATS);
            cpl_frame_set_type(*product_frame_ratioimg_stats,
                               CPL_FRAME_TYPE_TABLE);
            cpl_frame_set_group(*product_frame_ratioimg_stats,
                                CPL_FRAME_GROUP_PRODUCT);
            cpl_frame_set_level(*product_frame_ratioimg_stats,
                                CPL_FRAME_LEVEL_FINAL);

            /* Set up PHU header */

            plist = casu_fits_get_phu(ps->twilights[0]);
            hawki_dfs_set_product_primary_header(plist,
                                                 *product_frame_ratioimg_stats,
                                                 framelist,parlist,
                                                 recipeid,"PRO-1.15",
                                                 NULL,0);

            /* Fiddle with the extension header now */

            elist = casu_fits_get_ehu(ps->twilights[0]);
            p = cpl_propertylist_duplicate(elist);
            casu_merge_propertylists(p,ps->drs);
            if (! (cs->we_get & STATS_TAB))
                casu_dummy_property(p);
            hawki_dfs_set_product_exten_header(p,*product_frame_ratioimg_stats,
                                               framelist,parlist,
                                               recipeid,
                                               "PRO-1.15",NULL);
            status = CASU_OK;
            casu_removewcs(p,&status);

            /* And finally the difference image stats table */

            if (cpl_table_save(ps->ratioimstats,plist,p,outfile[2],
                               CPL_IO_DEFAULT) != CPL_ERROR_NONE) {
                cpl_msg_error(fctid,"Cannot save product table extension");
                cpl_propertylist_delete(p);
                return(-1);
            }
            cpl_propertylist_delete(p);
            cpl_frameset_insert(framelist,*product_frame_ratioimg_stats);
        }
    }

    /* Get the extension property list */

    plist = casu_fits_get_ehu(ps->twilights[0]);
    casu_merge_propertylists(plist,ps->drs);
    cpl_propertylist_update_int(plist,"ESO PRO DATANCOM",ps->ngood);

    /* Fiddle with the header now */

    p = cpl_propertylist_duplicate(plist);
    if (! (cs->we_get & MEANTWI)) 
        casu_dummy_property(p);
    hawki_dfs_set_product_exten_header(p,*product_frame_mean_twi,framelist,
                                       parlist,recipeid,
                                       "PRO-1.15",NULL);
                
    /* Now save the mean twilight flat image extension */

    cpl_propertylist_update_float(p,"ESO QC FLATRMS",cs->flatrms);
    cpl_propertylist_set_comment(p,"ESO QC FLATRMS","RMS of output flat");
    cpl_propertylist_update_float(p,"ESO QC FLAT MEDMIN",cs->minv);
    cpl_propertylist_set_comment(p,"ESO QC FLAT MEDMIN","Ensemble minimum");
    cpl_propertylist_update_float(p,"ESO QC FLAT MEDMAX",cs->maxv);
    cpl_propertylist_set_comment(p,"ESO QC FLAT MEDMAX","Ensemble maximum");
    cpl_propertylist_update_float(p,"ESO QC FLAT MEDMEAN",cs->avev);
    cpl_propertylist_set_comment(p,"ESO QC FLAT MEDMEAN","Ensemble average");
    cpl_propertylist_update_float(p,"ESO QC FLAT MEDMED",cs->medv);
    cpl_propertylist_set_comment(p,"ESO QC FLAT MEDMED","Ensemble median");
    cpl_propertylist_update_float(p,"ESO QC FLAT MEDSTDEV",cs->medsig);
    cpl_propertylist_set_comment(p,"ESO QC FLAT MEDSTDEV","Ensemble sigma");
    cpl_propertylist_update_float(p,"ESO QC FLUX RATIO",cs->fluxratio);
    cpl_propertylist_set_comment(p,"ESO QC FLUX RATIO","max/min ratio");
    val = cs->maxv - cs->minv;
    cpl_propertylist_update_float(p,"ESO QC FLATRNG",val);
    cpl_propertylist_set_comment(p,"ESO QC FLATRNG","Ensemble range");
    cpl_propertylist_update_float(p,"ESO QC TWIPHOT",
                                  cs->photnoise);
    cpl_propertylist_set_comment(p,"ESO QC TWIPHOT",
                                 "[adu] Estimated photon noise");
    cpl_propertylist_update_float(p,"ESO QC TWISNRATIO",
                                  cs->snratio);
    cpl_propertylist_set_comment(p,"ESO QC TWISNRATIO","Estimated S/N ratio");
    if (cpl_image_save(ps->outimage,outfile[0],CPL_TYPE_FLOAT,p,
                       CPL_IO_EXTEND) != CPL_ERROR_NONE) {
        cpl_msg_error(fctid,"Cannot save product image extension");
        cpl_propertylist_delete(p);
        return(-1);
    }
    cpl_propertylist_delete(p);

    /* Now save the twilight ratio image extension */

    if (cs->we_expect & RATIMG) {
        p = cpl_propertylist_duplicate(plist);
        if (! (cs->we_get & RATIMG))
            casu_dummy_property(p);
        cpl_propertylist_update_float(p,"ESO QC FLATRATIO_MED",
                                      cs->flatratio_med);
        cpl_propertylist_set_comment(p,"ESO QC FLATRATIO_MED",
                                     "Median of ratio map");
        cpl_propertylist_update_float(p,"ESO QC FLATRATIO_RMS",
                                      cs->flatratio_rms);
        cpl_propertylist_set_comment(p,"ESO QC FLATRATIO_RMS",
                                     "RMS of ratio map");
        hawki_dfs_set_product_exten_header(p,*product_frame_ratioimg,
                                           framelist,parlist,recipeid,
                                           "PRO-1.15",NULL);
        if (cpl_image_save(ps->ratioimg,outfile[1],CPL_TYPE_FLOAT,p,
                           CPL_IO_EXTEND) != CPL_ERROR_NONE) {
            cpl_propertylist_delete(p);
            cpl_msg_error(fctid,"Cannot save product image extension");
            return(-1);
        }
        cpl_propertylist_delete(p);
    }

    /* Now any further ratio image stats tables */

    if (! isfirst && (cs->we_expect & STATS_TAB)) {
        p = cpl_propertylist_duplicate(plist);
        if (! (cs->we_get & STATS_TAB))
            casu_dummy_property(p);
        hawki_dfs_set_product_exten_header(p,*product_frame_ratioimg_stats,
                                           framelist,parlist,recipeid,
                                           "PRO-1.15",NULL);
        status = CASU_OK;
        casu_removewcs(p,&status);
        if (cpl_table_save(ps->ratioimstats,NULL,p,outfile[2],CPL_IO_EXTEND)
                           != CPL_ERROR_NONE) {
              cpl_msg_error(fctid,"Cannot save product table extension");
            cpl_propertylist_delete(p);
             return(-1);
        }        
        cpl_propertylist_delete(p);
    }

    /* Fiddle with BPM header */

    p = cpl_propertylist_duplicate(plist);
    if (! (cs->we_get & BPM))
        casu_dummy_property(p);
    cpl_propertylist_update_int(p,"ESO QC FLAT NBADPIX",cs->nbad);
    cpl_propertylist_set_comment(p,"ESO QC FLAT NBADPIX",
                                 "Number of bad pixels detected");
    cpl_propertylist_update_float(p,"ESO QC BADFRAC",cs->badfrac);
    cpl_propertylist_set_comment(p,"ESO QC BADFRAC",
                                 "Fraction of bad pixels detected");

    /* Now save the BPM */

    hawki_dfs_set_product_exten_header(p,*product_frame_bpm,framelist,
                                       parlist,recipeid,"PRO-1.15",
                                       NULL);
    if (cpl_image_save(ps->outbpm,outfile[4],CPL_TYPE_UCHAR,p,
                       CPL_IO_EXTEND) != CPL_ERROR_NONE) {
        cpl_msg_error(fctid,"Cannot save product image extension");
        cpl_propertylist_delete(p);
        return(-1);
    }
    cpl_propertylist_delete(p);

    /* Fiddle with the confidence map header now */

    casu_merge_propertylists(plist,ps->drs2);
    p = cpl_propertylist_duplicate(plist);
    if (! (cs->we_get & CONFMAP)) 
        casu_dummy_property(p);

    /* Now save the confidence map */

    hawki_dfs_set_product_exten_header(p,*product_frame_conf,framelist,
                                       parlist,recipeid,"PRO-1.15",
                                       NULL);
    if (cpl_image_save(ps->outconf,outfile[3],CPL_TYPE_INT,p,
                       CPL_IO_EXTEND) != CPL_ERROR_NONE) {
        cpl_msg_error(fctid,"Cannot save product image extension");
        cpl_propertylist_delete(p);
        return(-1);
    }
    cpl_propertylist_delete(p);

    /* Get out of here */

    return(0);
}

/*---------------------------------------------------------------------------*/
/**
  @brief    Fill undefined products with dummy products
  @param    ps           The memory structure defined in the main routine
  @param    cs           The configuration structure defined in the main routine
 */
/*---------------------------------------------------------------------------*/

static void hawki_twilight_flat_combine_dummy_products(memstruct *ps,
                                                       configstruct *cs) {
    cpl_image *test;

    /* See if you even need to be here */

    if (cs->we_get == cs->we_expect)
        return;

    /* First an output combined twilight frame */

    if (! (cs->we_get & MEANTWI)) {
        ps->outimage = casu_dummy_image(ps->twilights[0]);
        cs->flatrms = 0.0;
    }

    /* Now the confidence map */

    if (! (cs->we_get & CONFMAP)) {
        ps->outconf = casu_dummy_image(ps->twilights[0]);
        cs->flatrms = 0.0;
    }

    /* Do a ratio image */

    if ((cs->we_expect & RATIMG) && ! (cs->we_get & RATIMG)) {
        cs->flatratio_med = 0.0;
        cs->flatratio_rms = 0.0;
        ps->ratioimg = casu_dummy_image(ps->twilights[0]);
    }

    /* Do the bad pixel mask */

    if ((cs->we_expect & BPM) && ! (cs->we_get & BPM)) {
        cs->nbad = 0;
        cs->badfrac = 0.0;
        test = casu_dummy_image(ps->twilights[0]);
        ps->outbpm = cpl_image_cast(test,CPL_TYPE_INT);
        freeimage(test);
    }

    /* If a ratio image stats table is required, then do that now */
   
    if ((cs->we_expect & STATS_TAB) && ! (cs->we_get & STATS_TAB)) 
        ps->ratioimstats = hawki_create_diffimg_stats(0);

    return;
}

/*---------------------------------------------------------------------------*/
/**
  @brief    Normalise the twilight frame and create ratio image and stats table
  @param    jext         the extension number
  @param    ps           The memory structure defined in the main routine
  @param    cs           The configuration structure defined in the main routine
 */
/*---------------------------------------------------------------------------*/

static void hawki_twilight_flat_combine_normal(int jext, memstruct *ps,
                                               configstruct *cs) {
    int nx,ny,ncells,status,*ba,*bpmim;
    long i,npi;
    unsigned char *bpm;
    float *idata,med,sig,gdiff,grms;
    casu_mask *bpmmask;
    cpl_array *bpmarray;
    casu_fits *mfimage;
    const char *fctid="hawki_twilight_flat_combine_normal";

    /* Create the bad pixel mask */
    
    nx = cpl_image_get_size_x(ps->outimage); 
    ny = cpl_image_get_size_y(ps->outimage); 
    npi = nx*ny;
    status = CASU_OK;
    (void)casu_genbpm(ps->good,ps->ngood,ps->outimage,5.0,5.0,"EXPTIME",
                      &bpmarray,&(cs->nbad),&(cs->badfrac),&status);
    if (status == CASU_OK) {
        cs->we_get |= BPM;
        ba = cpl_array_get_data_int(bpmarray);
        bpm = cpl_malloc(npi*sizeof(*bpm));
        bpmim = cpl_malloc(npi*sizeof(*bpmim));
        for (i = 0; i < npi; i++) {
            bpm[i] = (unsigned char)ba[i];
            bpmim[i] = ba[i];
        }
        cpl_array_delete(bpmarray);
        bpmmask = casu_mask_wrap_bpm(bpm,nx,ny);
        ps->outbpm = cpl_image_wrap_int((cpl_size)nx,(cpl_size)ny,bpmim);
    } else {
        bpm = cpl_calloc(npi,sizeof(*bpm));
        bpmmask = casu_mask_wrap_bpm(bpm,nx,ny);        
        cpl_msg_info(fctid,
                     "Bad pixel map creation failed extension %" CPL_SIZE_FORMAT,
                     (cpl_size)jext);
        status = CASU_OK;
    }

    /* Now make the confidence map */

    (void)casu_mkconf(ps->outimage,(char *)"None Available",bpmmask,
                      &(ps->outconf),&(ps->drs2),&status);
    if (status == CASU_OK) 
        cs->we_get |= CONFMAP;
    else {
        cpl_msg_info(fctid,
                     "Confidence map creation failed extension %" CPL_SIZE_FORMAT,
                     (cpl_size)jext);
        status = CASU_OK;
    }
    
    /* Work out the RMS of the mean twilight frame */

    idata = cpl_image_get_data(ps->outimage);
    casu_medsig(idata,bpm,npi,&med,&sig);

    /* Fill in the bad bits... */

    for (i = 0; i < npi; i++)
        if (bpm[i])
            idata[i] = med;

    /* Divide through by the median */

    cpl_propertylist_update_float(ps->drs,"ESO DRS MEDFLAT",med);
    cpl_propertylist_set_comment(ps->drs,"ESO DRS MEDFLAT",
                                 "Median value before normalisation");
    cpl_propertylist_update_float(ps->drs,"ESO QC FLAT NORM",med);
    cpl_propertylist_set_comment(ps->drs,"ESO QC FLAT NORM",
                                 "Median value before normalisation");
    cpl_image_divide_scalar(ps->outimage,med);
    casu_medmad(idata,bpm,npi,&med,&sig);
    sig *= 1.48;
    cs->flatrms = sig;

    /* Load up the master twilight flat */

    if (ps->master_twilight_flat != NULL) {
        mfimage = casu_fits_load(ps->master_twilight_flat,CPL_TYPE_FLOAT,jext);
        if (mfimage == NULL) {
            cpl_msg_info(fctid,
                         "Master twilight extension %" CPL_SIZE_FORMAT " won't load",
                         (cpl_size)jext);
        } else if (casu_is_dummy(casu_fits_get_ehu(mfimage))) {
            cpl_msg_info(fctid,
                         "Master twilight extension %" CPL_SIZE_FORMAT " is a dummy",
                         (cpl_size)jext);
            freefits(mfimage);
        }
    } else
        mfimage = NULL;

    /* Create a ratio image. */

    cs->flatratio_med = 0.0;
    cs->flatratio_rms = 0.0;
    ncells = cs->ncells;
    hawki_difference_image(casu_fits_get_image(mfimage),ps->outimage,bpm,
                           ncells,2,&gdiff,&grms,&(ps->ratioimg),
                           &(ps->ratioimstats));
    freefits(mfimage);
    casu_mask_delete(bpmmask);
    cs->flatratio_med = gdiff;
    cs->flatratio_rms = grms;
    if (ps->ratioimg != NULL)
        cs->we_get |= RATIMG;
    if (ps->ratioimstats != NULL)
        cs->we_get |= STATS_TAB;
    return;
}

/*---------------------------------------------------------------------------*/
/**
  @brief    Do make dummys and save
  @param    jext         the image extension in question
  @param    framelist    the input frame list
  @param    parlist      the input recipe parameter list
  @param    isfirst      true if this is the first extension
  @param    ps           The memory structure defined in the main routine
  @param    cs           The configuration structure defined in the main routine
  @param    product_frame_mean_twi   The frame for the mean twilight product
  @param    product_frame_conf       The frame for the mean twilight product
  @param    product_frame_ratioimg    The frame for the difference image product
  @param    product_frame_ratioimg_stats The frame for the diffimg stats table
  @param    product_frame_bpm        The frame for the bpm product
 @return   0 if everything is ok
 */
/*---------------------------------------------------------------------------*/

static int hawki_twilight_flat_combine_lastbit(int jext, 
                                               cpl_frameset *framelist,
                                               cpl_parameterlist *parlist,
                                               int isfirst, memstruct *ps,
                                               configstruct *cs,
                                               cpl_frame **product_frame_mean_twi,
                                               cpl_frame **product_frame_conf,
                                               cpl_frame **product_frame_ratioimg,
                                               cpl_frame **product_frame_ratioimg_stats,
                                               cpl_frame **product_frame_bpm) {

    int retval;
    const char *fctid="hawki_twilight_flat_combine_lastbit";

    /* Make whatever dummy products you need */

    hawki_twilight_flat_combine_dummy_products(ps,cs);

    /* Save everything */

    cpl_msg_info(fctid,
                 "Saving products for extension %" CPL_SIZE_FORMAT,
                 (cpl_size)jext);
    retval = hawki_twilight_flat_combine_save(framelist,parlist,isfirst,ps,cs,
                                              product_frame_mean_twi,
                                              product_frame_conf,
                                              product_frame_ratioimg,
                                              product_frame_ratioimg_stats,
                                              product_frame_bpm);
    if (retval != 0) {
        hawki_twilight_flat_combine_tidy(ps,0);
        return(-1);
    }

    /* Free some stuff up */

    hawki_twilight_flat_combine_tidy(ps,1);
    return(0);
}

/*---------------------------------------------------------------------------*/
/**
  @brief    Modify the input frame list to remove over-/under-exposed images
  @param    list         The input and output frameset
  @param    cs           The configuration structure defined in the main routine
  @return   0 if everything is ok
 */
/*---------------------------------------------------------------------------*/

static int hawki_twilight_flat_cull(cpl_frameset **list, configstruct *cs) {
    int i,n,j,gotone,live,stopnow;
    cpl_propertylist *p;
    cpl_frame *frm,*frmnew;
    cpl_frameset *newfrmset;
    cpl_image *im;
    double val;
    const char *fctid = "hawki_twilight_flat_cull";

    /* Loop through for each detector until we get one that works */

    frm = cpl_frameset_get_position(*list,0);
    n = cpl_frameset_get_size(*list);
    gotone = 0;
    for (i = 1; i <= HAWKI_NEXTN; i++) {
        p = cpl_propertylist_load(cpl_frame_get_filename(frm),i);
        hawki_pfits_get_detlive(p,&live);
        cpl_propertylist_delete(p);
        if (! live) 
            continue;
        gotone = 1;
        
        /* Loop for all of the frames and see which ones fall inside the 
           thresholds. NB: we're going backwards here. Most of the twilights
           are taken at sunset. If it's too bright, then the counts on the
           detectors actually start going back down again and these can
           mistakenly be included in the defined window even though they
           are saturated. */

        newfrmset = cpl_frameset_new();
        stopnow = 0;
        for (j = n-1; j >= 0; j--) {
            frm = cpl_frameset_get_position(*list,j);
            im = cpl_image_load(cpl_frame_get_filename(frm),CPL_TYPE_FLOAT,0,
                                (cpl_size)i);
            val = cpl_image_get_median_window(im,768,768,1791,1791);
            if (! stopnow && val > cs->hthr) 
                stopnow = 1;
            if (val > cs->lthr && val < cs->hthr && ! stopnow) {
                frmnew = cpl_frame_duplicate(frm);
                cpl_frameset_insert(newfrmset,frmnew);
            }
            cpl_image_delete(im);
        }
        cpl_frameset_delete(*list);
        cpl_frameset_sort(newfrmset,hawki_compare_fnames);
        *list = newfrmset;
        break;
    }
    n = cpl_frameset_get_size(*list);
    
    /* If we don't have one, then all the detectors are dead */

    if (! gotone) {
        cpl_msg_error(fctid,"All detectors flagged as dead");
        return(1);
    } else if (n == 0) {
        cpl_msg_error(fctid,"No images within thresholds %g and %g",
                      cs->lthr,cs->hthr);
        return(1);
    } else {
        return(0);
    }
}

/*---------------------------------------------------------------------------*/
/**
  @brief    Compare function for sorting by filename
  @param    f1           First frame
  @param    f2           Second frame
 */
/*---------------------------------------------------------------------------*/

static int hawki_compare_fnames(const cpl_frame *f1, const cpl_frame *f2) {
    return(strcmp(cpl_frame_get_filename(f1),cpl_frame_get_filename(f2)));
}

/*---------------------------------------------------------------------------*/
/**
  @brief    Initialise the pointers in the memory structure
  @param    ps           The memory structure defined in the main routine
 */
/*---------------------------------------------------------------------------*/

static void hawki_twilight_flat_combine_init(memstruct *ps) {
    ps->labels = NULL;
    ps->twilightlist = NULL;
    ps->twilights = NULL;
    ps->good = NULL;
    ps->master_dark = NULL;
    ps->master_twilight_flat = NULL;
    ps->master_mask = NULL;
    ps->outimage = NULL;
    ps->outconf = NULL;
    ps->outbpm = NULL;
    ps->drs = NULL;
    ps->drs2 = NULL;
    ps->rejmask = NULL;
    ps->rejplus = NULL;
    ps->ratioimg = NULL;
    ps->ratioimstats = NULL;
    ps->mdark = NULL;
}

/*---------------------------------------------------------------------------*/
/**
  @brief    Free any allocated workspace in the memory structure
  @param    ps           The memory structure defined in the main routine
  @param    level        The level of the tidy
 */
/*---------------------------------------------------------------------------*/

static void hawki_twilight_flat_combine_tidy(memstruct *ps, int level) {
    freeimage(ps->outimage);
    freeimage(ps->outconf);
    freeimage(ps->outbpm);
    freefitslist(ps->twilights,ps->ntwilights);
    freepropertylist(ps->drs);
    freepropertylist(ps->drs2);
    freespace(ps->rejmask);
    freespace(ps->rejplus);
    freeimage(ps->ratioimg);
    freetable(ps->ratioimstats);
    freefits(ps->mdark);
    if (level == 1)
        return;

    freespace(ps->good);
    freespace(ps->labels);
    freeframeset(ps->twilightlist);
    freeframe(ps->master_dark);
    freeframe(ps->master_twilight_flat);
    freemask(ps->master_mask);
}

/**@}*/

/*

$Log: hawki_twilight_flat_combine.c,v $
Revision 1.15  2015/10/24 09:21:20  jim
superficial change

Revision 1.14  2015/10/15 11:23:15  jim
modified qc

Revision 1.13  2015/09/11 09:31:20  jim
Changed parameter declarations from _value to _range where appropriate

Revision 1.12  2015/08/07 13:07:06  jim
Fixed copyright to ESO

Revision 1.11  2015/06/08 09:34:23  jim
changed default value for hthr

Revision 1.10  2015/01/29 12:03:01  jim
Modified comments. Modified command line arguments to use strings for
enums where that makes sense. Also removed some support routines to
hawki_utils.c

Revision 1.9  2014/12/11 12:24:09  jim
lots of upgrades

Revision 1.8  2014/04/09 11:08:21  jim
Get rid of a couple of compiler moans

Revision 1.7  2014/03/26 16:08:21  jim
Confidence maps are now stored as floating point

Revision 1.6  2013/11/21 09:38:14  jim
detabbed

Revision 1.5  2013/11/07 10:05:59  jim
Fixed calls to tidy routine

Revision 1.4  2013-10-24 09:33:41  jim
Added prettynames

Revision 1.3  2013-08-30 09:10:06  jim
fixed a few tabs

Revision 1.2  2013-08-29 17:30:26  jim
Altered to remove remaining globals

Revision 1.1.1.1  2013-08-27 12:07:48  jim
Imported


*/
