/* $Id: fors_subtract_sky.c,v 1.8 2013-04-24 14:14:13 cgarcia Exp $
 *
 * This file is part of the FORS Data Reduction Pipeline
 * Copyright (C) 2002-2010 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 */

/*
 * $Author: cgarcia $
 * $Date: 2013-04-24 14:14:13 $
 * $Revision: 1.8 $
 * $Name: not supported by cvs2svn $
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <math.h>
#include <cpl.h>
#include <moses.h>
#include <fors_dfs.h>

static int fors_subtract_sky_create(cpl_plugin *);
static int fors_subtract_sky_exec(cpl_plugin *);
static int fors_subtract_sky_destroy(cpl_plugin *);
static int fors_subtract_sky(cpl_parameterlist *, cpl_frameset *);

static char fors_subtract_sky_description[] =
"This recipe is used to subtract the sky emission from unrebinned slit\n"
"spectra. This is obtained by robust fitting (i.e., excluding the signal\n"
"from possible point-like objects in slit) of the emission along the CCD\n"
"columns within each spectrum). This method doesn't work if extended\n"
"objects are in slit (it really destroys the object spectra), and is\n"
"not applicable to LSS data. The input scientific frames are produced\n"
"by the recipes fors_remove_bias and fors_flatfield.\n"
"\n"
"This recipe cannot be applied to LSS or long-slit like data (MOS/MXU with\n"
"all slits at the same offset). No automatic recipe is available for this.\n"
"Please refer to the FORS Pipeline User's Manual for more details.\n"
"\n"
"In the table below the MXU acronym can be alternatively read as MOS, and\n"
"SCI as STD.\n\n"
"Input files:\n\n"
"  DO category:               Type:       Explanation:         Required:\n"
"  SCIENCE_UNBIAS_MXU\n"
"  or SCIENCE_UNFLAT_MXU\n"
"  or STANDARD_UNBIAS_MXU\n"
"  or STANDARD_UNFLAT_MXU     Calib       Frame with sky lines    Y\n"
"  CURV_COEFF_MXU             Calib       Spectral curvature      Y\n"
"  SLIT_LOCATION_MXU          Calib       Slit location on CCD    Y\n"
"  GRISM_TABLE                Calib       Grism table             .\n\n"
"Output files:\n\n"
"  DO category:               Data type:  Explanation:\n"
"  UNMAPPED_SCI_MXU\n"
"  or UNMAPPED_STD_MXU        FITS image  Sky subtracted scientific frame\n"
"  UNMAPPED_SKY_SCI_MXU\n"
"  or UNMAPPED_SKY_STD_MXU    FITS image  Subtracted sky frame\n\n";

#define fors_subtract_sky_exit(message)       \
{                                             \
if ((const char *)message != NULL) cpl_msg_error(recipe, message);  \
cpl_image_delete(spectra);                    \
cpl_image_delete(skymap);                     \
cpl_table_delete(grism_table);                \
cpl_table_delete(maskslits);                  \
cpl_table_delete(slits);                      \
cpl_table_delete(polytraces);                 \
cpl_propertylist_delete(header);              \
cpl_msg_indent_less();                        \
return -1;                                    \
}

#define fors_subtract_sky_exit_memcheck(message)   \
{                                               \
if ((const char *)message != NULL) cpl_msg_info(recipe, message);     \
printf("free spectra (%p)\n", spectra);         \
cpl_image_delete(spectra);                      \
printf("free skymap (%p)\n", skymap);           \
cpl_image_delete(skymap);                       \
printf("free grism_table (%p)\n", grism_table); \
cpl_table_delete(grism_table);                  \
printf("free maskslits (%p)\n", maskslits);     \
cpl_table_delete(maskslits);                    \
printf("free slits (%p)\n", slits);             \
cpl_table_delete(slits);                        \
printf("free polytraces (%p)\n", polytraces);   \
cpl_table_delete(polytraces);                   \
printf("free header (%p)\n", header);           \
cpl_propertylist_delete(header);                \
cpl_msg_indent_less();                          \
return 0;                                       \
}


/**
 * @brief    Build the list of available plugins, for this module. 
 *
 * @param    list    The plugin list
 *
 * @return   0 if everything is ok, -1 otherwise
 *
 * Create the recipe instance and make it available to the application 
 * using the interface. This function is exported.
 */

int cpl_plugin_get_info(cpl_pluginlist *list)
{
    cpl_recipe *recipe = cpl_calloc(1, sizeof *recipe );
    cpl_plugin *plugin = &recipe->interface;

    cpl_plugin_init(plugin,
                    CPL_PLUGIN_API,
                    FORS_BINARY_VERSION,
                    CPL_PLUGIN_TYPE_RECIPE,
                    "fors_subtract_sky",
                    "Subtract sky from scientific spectra",
                    fors_subtract_sky_description,
                    "Carlo Izzo",
                    PACKAGE_BUGREPORT,
    "This file is currently part of the FORS Instrument Pipeline\n"
    "Copyright (C) 2002-2010 European Southern Observatory\n\n"
    "This program is free software; you can redistribute it and/or modify\n"
    "it under the terms of the GNU General Public License as published by\n"
    "the Free Software Foundation; either version 2 of the License, or\n"
    "(at your option) any later version.\n\n"
    "This program is distributed in the hope that it will be useful,\n"
    "but WITHOUT ANY WARRANTY; without even the implied warranty of\n"
    "MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n"
    "GNU General Public License for more details.\n\n"
    "You should have received a copy of the GNU General Public License\n"
    "along with this program; if not, write to the Free Software Foundation,\n"
    "Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA\n",
                    fors_subtract_sky_create,
                    fors_subtract_sky_exec,
                    fors_subtract_sky_destroy);

    cpl_pluginlist_append(list, plugin);
    
    return 0;
}


/**
 * @brief    Setup the recipe options    
 *
 * @param    plugin  The plugin
 *
 * @return   0 if everything is ok
 *
 * Defining the command-line/configuration parameters for the recipe.
 */

static int fors_subtract_sky_create(cpl_plugin *plugin)
{
    cpl_recipe    *recipe;
    cpl_parameter *p;

    /* 
     * Check that the plugin is part of a valid recipe 
     */

    if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
        recipe = (cpl_recipe *)plugin;
    else 
        return -1;

    /* 
     * Create the (empty) parameters list in the cpl_recipe object 
     */

    recipe->parameters = cpl_parameterlist_new(); 

    /*
     * Dispersion
     */

    p = cpl_parameter_new_value("fors.fors_subtract_sky.dispersion",
                                CPL_TYPE_DOUBLE,
                                "Expected spectral dispersion (Angstrom/pixel)",
                                "fors.fors_subtract_sky",
                                0.0);
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "dispersion");
    cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(recipe->parameters, p);

    /*
     * Start wavelength for spectral extraction
     */

    p = cpl_parameter_new_value("fors.fors_subtract_sky.startwavelength",
                                CPL_TYPE_DOUBLE,
                                "Start wavelength in spectral extraction",
                                "fors.fors_subtract_sky",
                                0.0);
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "startwavelength");
    cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(recipe->parameters, p);

    /*
     * End wavelength for spectral extraction
     */

    p = cpl_parameter_new_value("fors.fors_subtract_sky.endwavelength",
                                CPL_TYPE_DOUBLE,
                                "End wavelength in spectral extraction",
                                "fors.fors_subtract_sky",
                                0.0);
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "endwavelength");
    cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(recipe->parameters, p);

    /*
     * Cosmic rays removal
     */

    p = cpl_parameter_new_value("fors.fors_subtract_sky.cosmics",
                                CPL_TYPE_BOOL,
                                "Eliminate cosmic rays hits",
                                "fors.fors_subtract_sky",
                                FALSE);
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "cosmics");
    cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(recipe->parameters, p);

    return 0;
}


/**
 * @brief    Execute the plugin instance given by the interface
 *
 * @param    plugin  the plugin
 *
 * @return   0 if everything is ok
 */

static int fors_subtract_sky_exec(cpl_plugin *plugin)
{
    cpl_recipe *recipe;
    
    if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
        recipe = (cpl_recipe *)plugin;
    else 
        return -1;

    return fors_subtract_sky(recipe->parameters, recipe->frames);
}


/**
 * @brief    Destroy what has been created by the 'create' function
 *
 * @param    plugin  The plugin
 *
 * @return   0 if everything is ok
 */

static int fors_subtract_sky_destroy(cpl_plugin *plugin)
{
    cpl_recipe *recipe;
    
    if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
        recipe = (cpl_recipe *)plugin;
    else 
        return -1;

    cpl_parameterlist_delete(recipe->parameters); 

    return 0;
}


/**
 * @brief    Interpret the command line options and execute the data processing
 *
 * @param    parlist     The parameters list
 * @param    frameset    The set-of-frames
 *
 * @return   0 if everything is ok
 */

static int fors_subtract_sky(cpl_parameterlist *parlist, 
                               cpl_frameset *frameset)
{

    const char *recipe = "fors_subtract_sky";


    /*
     * Input parameters
     */

    double      dispersion;
    double      startwavelength;
    double      endwavelength;
    int         cosmics;

    /*
     * CPL objects
     */

    cpl_image        *spectra     = NULL;
    cpl_image        *skymap      = NULL;
    cpl_table        *grism_table = NULL;
    cpl_table        *polytraces  = NULL;
    cpl_table        *slits       = NULL;
    cpl_table        *maskslits   = NULL;
    cpl_propertylist *header      = NULL;

    /*
     * Auxiliary variables
     */

    char        version[80];
    const char *slit_location_tag;
    const char *input_tag;
    const char *curv_coeff_tag;
    const char *unmapped_tag;
    const char *unmapped_sky_tag;
    int         nframes;
    int         rebin;
    int         nslits;
    int         treat_as_lss;
    int         i;
    double      reference;
    double      gain;
    double     *xpos;
    double      mxpos;
    int         mxu, mos, lss;
    int         rec_scib;
    int         rec_stdb;
    int         rec_scif;
    int         rec_stdf;
    int         nslits_out_det = 0;

    char       *instrume = NULL;


    cpl_msg_set_indentation(2);


    /*
     * Get configuration parameters
     */

    cpl_msg_info(recipe, "Recipe %s configuration parameters:", recipe);
    cpl_msg_indent_more();
    
    if (cpl_frameset_count_tags(frameset, "GRISM_TABLE") > 1)
        fors_subtract_sky_exit("Too many in input: GRISM_TABLE"); 

    grism_table = dfs_load_table(frameset, "GRISM_TABLE", 1);

    dispersion = dfs_get_parameter_double(parlist,
                    "fors.fors_subtract_sky.dispersion", grism_table);

    if (dispersion <= 0.0)
        fors_subtract_sky_exit("Invalid spectral dispersion value");

    startwavelength = dfs_get_parameter_double(parlist,
                    "fors.fors_subtract_sky.startwavelength", grism_table);
    if (startwavelength > 1.0)
        if (startwavelength < 3000.0 || startwavelength > 13000.0)
            fors_subtract_sky_exit("Invalid wavelength");

    endwavelength = dfs_get_parameter_double(parlist,
                    "fors.fors_subtract_sky.endwavelength", grism_table);
    if (endwavelength > 1.0) {
        if (endwavelength < 3000.0 || endwavelength > 13000.0)
            fors_subtract_sky_exit("Invalid wavelength");
        if (startwavelength < 1.0)
            fors_subtract_sky_exit("Invalid wavelength interval");
    }

    if (startwavelength > 1.0)
        if (endwavelength - startwavelength <= 0.0)
            fors_subtract_sky_exit("Invalid wavelength interval");

    cosmics = dfs_get_parameter_bool(parlist, 
                                     "fors.fors_subtract_sky.cosmics", NULL);

    cpl_table_delete(grism_table); grism_table = NULL;

    if (cpl_error_get_code())
        fors_subtract_sky_exit("Failure reading the configuration parameters");


    cpl_msg_indent_less();
    cpl_msg_info(recipe, "Check input set-of-frames:");
    cpl_msg_indent_more();

    mxu  = cpl_frameset_count_tags(frameset, "SLIT_LOCATION_MXU");
    mos  = cpl_frameset_count_tags(frameset, "SLIT_LOCATION_MOS");
    lss  = cpl_frameset_count_tags(frameset, "SLIT_LOCATION_LSS");

    if (lss)
        fors_subtract_sky_exit("Use this recipe just with MOS/MXU data.");

    nframes = mos + mxu;

    if (nframes == 0) {
        fors_subtract_sky_exit("Missing input slit location table");
    }
    if (nframes > 1) {
        cpl_msg_error(recipe, 
                      "Too many input slit location tables (%d > 1)", nframes);
        fors_subtract_sky_exit(NULL);
    }

    if (mxu)
        curv_coeff_tag = "CURV_COEFF_MXU";
    else
        curv_coeff_tag = "CURV_COEFF_MXU";

    
    nframes = cpl_frameset_count_tags(frameset, curv_coeff_tag);

    if (nframes == 0) {
        cpl_msg_error(recipe, "Missing input %s", curv_coeff_tag);
        fors_subtract_sky_exit(NULL);
    }
    if (nframes > 1) {
        cpl_msg_error(recipe, "Too many input %s (%d > 1)", curv_coeff_tag,
                      nframes);
        fors_subtract_sky_exit(NULL);
    }

    if (mxu) {
        rec_scib = cpl_frameset_count_tags(frameset, "SCIENCE_UNBIAS_MXU");
        rec_stdb = cpl_frameset_count_tags(frameset, "STANDARD_UNBIAS_MXU");
        rec_scif = cpl_frameset_count_tags(frameset, "SCIENCE_UNFLAT_MXU");
        rec_stdf = cpl_frameset_count_tags(frameset, "STANDARD_UNFLAT_MXU");
    }
    else {
        rec_scib = cpl_frameset_count_tags(frameset, "SCIENCE_UNBIAS_MOS");
        rec_stdb = cpl_frameset_count_tags(frameset, "STANDARD_UNBIAS_MOS");
        rec_scif = cpl_frameset_count_tags(frameset, "SCIENCE_UNFLAT_MOS");
        rec_stdf = cpl_frameset_count_tags(frameset, "STANDARD_UNFLAT_MOS");
    }

    nframes = rec_scib + rec_stdb + rec_scif + rec_stdf;

    if (nframes == 0) {
        fors_subtract_sky_exit("Missing input scientific spectra");
    }
    if (nframes > 1) {
        cpl_msg_error(recipe, "Too many input scientific spectra (%d > 1)", 
                      nframes);
        fors_subtract_sky_exit(NULL);
    }

    if (rec_scib) {
        if (mxu) {
            input_tag         = "SCIENCE_UNBIAS_MXU";
            slit_location_tag = "SLIT_LOCATION_MXU";
            unmapped_tag      = "UNMAPPED_SCI_MXU";
            unmapped_sky_tag  = "UNMAPPED_SKY_SCI_MXU";
        }
        else {
            input_tag         = "SCIENCE_UNBIAS_MOS";
            slit_location_tag = "SLIT_LOCATION_MOS";
            unmapped_tag      = "UNMAPPED_SCI_MOS";
            unmapped_sky_tag  = "UNMAPPED_SKY_SCI_MOS";
        }
    }
    else if (rec_stdb) {
        if (mxu) {
            input_tag         = "STANDARD_UNBIAS_MXU";
            slit_location_tag = "SLIT_LOCATION_MXU";
            unmapped_tag      = "UNMAPPED_STD_MXU";
            unmapped_sky_tag  = "UNMAPPED_SKY_STD_MXU";
        }
        else {
            input_tag         = "STANDARD_UNBIAS_MOS";
            slit_location_tag = "SLIT_LOCATION_MOS";
            unmapped_tag      = "UNMAPPED_STD_MOS";
            unmapped_sky_tag  = "UNMAPPED_SKY_STD_MOS";
        }
    }
    else if (rec_scif) {
        if (mxu) {
            input_tag         = "SCIENCE_UNFLAT_MXU";
            slit_location_tag = "SLIT_LOCATION_MXU";
            unmapped_tag      = "UNMAPPED_SCI_MXU";
            unmapped_sky_tag  = "UNMAPPED_SKY_SCI_MXU";
        }
        else {   
            input_tag         = "SCIENCE_UNFLAT_MOS";
            slit_location_tag = "SLIT_LOCATION_MOS";
            unmapped_tag      = "UNMAPPED_SCI_MOS";
            unmapped_sky_tag  = "UNMAPPED_SKY_SCI_MOS";
        }
    }
    else if (rec_stdf) {
        if (mxu) {
            input_tag         = "STANDARD_UNFLAT_MXU";
            slit_location_tag = "SLIT_LOCATION_MXU";
            unmapped_tag      = "UNMAPPED_STD_MXU";
            unmapped_sky_tag  = "UNMAPPED_SKY_STD_MXU";
        }
        else {   
            input_tag         = "STANDARD_UNFLAT_MOS";
            slit_location_tag = "SLIT_LOCATION_MOS";
            unmapped_tag      = "UNMAPPED_STD_MOS";
            unmapped_sky_tag  = "UNMAPPED_SKY_STD_MOS";
        }
    }


    header = dfs_load_header(frameset, input_tag, 0);

    if (header == NULL)
        fors_subtract_sky_exit("Cannot load scientific frame header");

    if (mos)
        maskslits = mos_load_slits_fors_mos(header, &nslits_out_det);
    else
        maskslits = mos_load_slits_fors_mxu(header);

    /*
     * Check if all slits have the same X offset: if not, abort!
     */

    treat_as_lss = fors_mos_is_lss_like(maskslits, nslits_out_det);

    cpl_table_delete(maskslits); maskslits = NULL;

    if (treat_as_lss)
        fors_subtract_sky_exit("This recipe cannot process MOS/MXU "
                               "data with all slits at the same offset.");


    if (!dfs_equal_keyword(frameset, "ESO INS GRIS1 ID"))
        cpl_msg_warning(cpl_func,"Input frames are not from the same grism");

    if (!dfs_equal_keyword(frameset, "ESO INS FILT1 ID"))
        cpl_msg_warning(cpl_func,"Input frames are not from the same filter");

    if (!dfs_equal_keyword(frameset, "ESO DET CHIP1 ID"))
        cpl_msg_warning(cpl_func,"Input frames are not from the same chip");


    /*
     * Get the reference wavelength and the rebin factor along the
     * dispersion direction from the reference frame
     */

    instrume = (char *)cpl_propertylist_get_string(header, "INSTRUME");
    if (instrume == NULL)
        fors_subtract_sky_exit("Missing keyword INSTRUME in reference frame "
                            "header");

    if (instrume[4] == '1')
        snprintf(version, 80, "%s/%s", "fors1", VERSION);
    if (instrume[4] == '2')
        snprintf(version, 80, "%s/%s", "fors2", VERSION);

    reference = cpl_propertylist_get_double(header, "ESO INS GRIS1 WLEN");

    if (cpl_error_get_code() != CPL_ERROR_NONE)
        fors_subtract_sky_exit("Missing keyword ESO INS GRIS1 WLEN "
                            "in reference frame header");

    if (reference < 3000.0)   /* Perhaps in nanometers... */
        reference *= 10;

    if (reference < 3000.0 || reference > 13000.0) {
        cpl_msg_error(recipe, "Invalid central wavelength %.2f read from "
                      "keyword ESO INS GRIS1 WLEN in reference frame header",
                      reference);
        fors_subtract_sky_exit(NULL);
    }

    cpl_msg_info(recipe, "The central wavelength is: %.2f", reference);

    rebin = cpl_propertylist_get_int(header, "ESO DET WIN1 BINX");

    if (cpl_error_get_code() != CPL_ERROR_NONE)
        fors_subtract_sky_exit("Missing keyword ESO DET WIN1 BINX "
                            "in reference frame header");

    if (rebin != 1) {
        dispersion *= rebin;
        cpl_msg_warning(recipe, "The rebin factor is %d, and therefore the "
                        "working dispersion used is %f A/pixel", rebin,
                        dispersion);
    }

    if (cosmics) {
        gain = cpl_propertylist_get_double(header, "ESO DET OUT1 CONAD");

        if (cpl_error_get_code() != CPL_ERROR_NONE)
            fors_subtract_sky_exit("Missing keyword ESO DET OUT1 CONAD in "
                                   "scientific frame header");

        cpl_msg_info(recipe, "The gain factor is: %.2f e-/ADU", gain);
    }


    cpl_msg_indent_less();
    cpl_msg_info(recipe, "Load input frames...");
    cpl_msg_indent_more();

    spectra = dfs_load_image(frameset, input_tag, CPL_TYPE_FLOAT, 0, 0);
    if (spectra == NULL)
        fors_subtract_sky_exit("Cannot load input scientific frame");

    slits = dfs_load_table(frameset, slit_location_tag, 1);
    if (slits == NULL)
        fors_subtract_sky_exit("Cannot load slits location table");

    polytraces = dfs_load_table(frameset, curv_coeff_tag, 1);
    if (polytraces == NULL)
        fors_subtract_sky_exit("Cannot load spectral curvature table");

    cpl_msg_indent_less();
    cpl_msg_info(recipe, "Local sky determination...");
    cpl_msg_indent_more();
    skymap = mos_subtract_sky(spectra, slits, polytraces, reference,
                              startwavelength, endwavelength, dispersion);

    cpl_table_delete(polytraces); polytraces = NULL;
    cpl_table_delete(slits); slits = NULL;

    if (cosmics) {
        cpl_msg_info(recipe, "Removing cosmic rays...");
        mos_clean_cosmics(spectra, gain, -1., -1.);
    }

    if (dfs_save_image(frameset, spectra, unmapped_tag,
                       header, parlist, recipe, version))
        fors_subtract_sky_exit(NULL);

    cpl_image_delete(spectra); spectra = NULL;

    if (dfs_save_image(frameset, skymap, unmapped_sky_tag,
                       header, parlist, recipe, version))
        fors_subtract_sky_exit(NULL);

    cpl_image_delete(skymap); skymap = NULL;

    cpl_propertylist_delete(header); header = NULL;

    return 0;
}