/* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set sw=2 sts=2 et cin: */
/*
 * This file is part of the MUSE Instrument Pipeline
 * Copyright (C) 2007-2014 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

/*----------------------------------------------------------------------------*
 *                              Includes                                      *
 *----------------------------------------------------------------------------*/
#define _DEFAULT_SOURCE
#define _BSD_SOURCE /* get mkdtemp() and setenv() from stdlib.h */
#include <unistd.h> /* access() */
#include <stdlib.h>
#include <stdio.h>
#include <float.h>
#include <math.h>
#include <sys/stat.h>
#include <string.h>
#ifdef HAVE_GETTIMEOFDAY
#include <time.h>
#include <sys/time.h>
#endif

#include <muse.h>
#include "muse_test_processing_z.h"

#ifdef DO_NOT_CPL_TEST
#define cpl_test(A)
#define cpl_test_eq(A, B)
#define cpl_test_eq_string(A, B)
#define cpl_test_nonnull(A)
#define cpl_test_init(A, B) cpl_init(CPL_INIT_DEFAULT)
#define cpl_test_end(A) (A)
#endif

#define DIR_TEMPLATE "/tmp/muse_test_processing_XXXXXX"

static void
muse_test_processing_prepare_test(char *aDir)
{
  // create (raw) MUSE-like image
  cpl_image *raw = cpl_image_fill_test_create(4160, 4164);
  /* offset background, so that not everything is marked as saturated */
  cpl_image_add_scalar(raw, 10.);
  /* extension header */
  cpl_propertylist *hdr = cpl_propertylist_new();
  cpl_propertylist_append_string(hdr, "DATE-OBS", "2016-09-14T12:30:00.001");
  cpl_propertylist_append_int(hdr, "DUMMY", 1);
  cpl_propertylist_append_int(hdr, "ESO DET BINX", 1);
  cpl_propertylist_append_int(hdr, "ESO DET BINY", 1);
  cpl_propertylist_append_int(hdr, "ESO DET READ CURID", 1);
  cpl_propertylist_append_string(hdr, "ESO DET READ CURNAME", "SCI1.0");
  char *file = cpl_sprintf("%s/raw.fits", aDir);
  cpl_image_save(NULL, file, CPL_TYPE_SHORT, hdr, CPL_IO_CREATE);
  cpl_propertylist_delete(hdr);
  /* extension header */
  hdr = cpl_propertylist_new();
  cpl_propertylist_append_string(hdr, "EXTNAME", "CHAN01");
  cpl_propertylist_append_string(hdr, "ESO DET CHIP NAME", "CHAN01");
  cpl_propertylist_append_string(hdr, "ESO DET CHIP ID", "psyche");
  cpl_propertylist_append_string(hdr, "ESO DET CHIP DATE",
                                 "2014-02-21T12:18:14.207");
  cpl_propertylist_append_int(hdr, "ESO DET OUT1 NX", 2048);
  cpl_propertylist_append_int(hdr, "ESO DET OUT1 NY", 2056);
  cpl_propertylist_append_int(hdr, "ESO DET OUT1 OVSCX", 5);
  cpl_propertylist_append_int(hdr, "ESO DET OUT1 OVSCY", 5);
  cpl_propertylist_append_int(hdr, "ESO DET OUT1 PRSCX", 32);
  cpl_propertylist_append_int(hdr, "ESO DET OUT1 PRSCY", 20);
  cpl_propertylist_append_int(hdr, "ESO DET OUT1 X", 1);
  cpl_propertylist_append_int(hdr, "ESO DET OUT1 Y", 1);
  cpl_propertylist_append_float(hdr, "ESO DET OUT1 GAIN", 0.71);
  cpl_propertylist_append_int(hdr, "ESO DET OUT2 NX", 2048);
  cpl_propertylist_append_int(hdr, "ESO DET OUT2 NY", 2056);
  cpl_propertylist_append_int(hdr, "ESO DET OUT2 OVSCX", 5);
  cpl_propertylist_append_int(hdr, "ESO DET OUT2 OVSCY", 5);
  cpl_propertylist_append_int(hdr, "ESO DET OUT2 PRSCX", 32);
  cpl_propertylist_append_int(hdr, "ESO DET OUT2 PRSCY", 20);
  cpl_propertylist_append_int(hdr, "ESO DET OUT2 X", 4096);
  cpl_propertylist_append_int(hdr, "ESO DET OUT2 Y", 1);
  cpl_propertylist_append_float(hdr, "ESO DET OUT2 GAIN", 0.72);
  cpl_propertylist_append_int(hdr, "ESO DET OUT3 NX", 2048);
  cpl_propertylist_append_int(hdr, "ESO DET OUT3 NY", 2056);
  cpl_propertylist_append_int(hdr, "ESO DET OUT3 OVSCX", 5);
  cpl_propertylist_append_int(hdr, "ESO DET OUT3 OVSCY", 5);
  cpl_propertylist_append_int(hdr, "ESO DET OUT3 PRSCX", 32);
  cpl_propertylist_append_int(hdr, "ESO DET OUT3 PRSCY", 20);
  cpl_propertylist_append_int(hdr, "ESO DET OUT3 X", 4096);
  cpl_propertylist_append_int(hdr, "ESO DET OUT3 Y", 4112);
  cpl_propertylist_append_float(hdr, "ESO DET OUT3 GAIN", 0.73);
  cpl_propertylist_append_int(hdr, "ESO DET OUT4 NX", 2048);
  cpl_propertylist_append_int(hdr, "ESO DET OUT4 NY", 2056);
  cpl_propertylist_append_int(hdr, "ESO DET OUT4 OVSCX", 5);
  cpl_propertylist_append_int(hdr, "ESO DET OUT4 OVSCY", 5);
  cpl_propertylist_append_int(hdr, "ESO DET OUT4 PRSCX", 32);
  cpl_propertylist_append_int(hdr, "ESO DET OUT4 PRSCY", 20);
  cpl_propertylist_append_int(hdr, "ESO DET OUT4 X", 1);
  cpl_propertylist_append_int(hdr, "ESO DET OUT4 Y", 4112);
  cpl_propertylist_append_float(hdr, "ESO DET OUT4 GAIN", 0.74);
  cpl_image_save(raw, file, CPL_TYPE_SHORT, hdr, CPL_IO_EXTEND);
  cpl_free(file);
  cpl_image_delete(raw);
  cpl_propertylist_delete(hdr);

  // create generic table
  cpl_table *table = cpl_table_new(5);
  cpl_table_new_column(table, "column", CPL_TYPE_DOUBLE);
  cpl_table_fill_column_window(table, "column", 0, 5, DBL_EPSILON);
  file = cpl_sprintf("%s/table.fits", aDir);
  cpl_table_save(table, NULL, NULL, file, CPL_IO_CREATE);
  cpl_free(file);
  cpl_table_delete(table);

  // create an IFU-specific table (IFU 1)
  table = cpl_table_new(10);
  cpl_table_new_column(table, "IFU", CPL_TYPE_INT);
  cpl_table_fill_column_window(table, "IFU", 0, 9, 1);
  cpl_propertylist *header = cpl_propertylist_new();
  cpl_propertylist_append_string(header, "EXTNAME", "CHAN01");
  cpl_propertylist_append_string(header, "PIPEFILE", "tableifu.fits");
  file = cpl_sprintf("%s/tableifu.fits", aDir);
  /* save with header in primary data unit */
  cpl_table_save(table, header, NULL, file, CPL_IO_CREATE);
  cpl_free(file);
  cpl_table_delete(table);
  cpl_propertylist_delete(header);

  // create a basic FITS header
  header = cpl_propertylist_new();
  cpl_propertylist_append_string(header, "INSTRUME", "MUSE");
  cpl_propertylist_append_double(header, "VALUE", 99.99);
  file = cpl_sprintf("%s/header.fits", aDir);
  cpl_propertylist_save(header, file, CPL_IO_CREATE);
  cpl_free(file);
  cpl_propertylist_delete(header);
} /* muse_test_processing_prepare_test() */

static int
muse_test_processing_frames_sort_valid_ifus_in_exposure(cpl_table *aTable,
                                                        int aExposure)
{
  /* loop through all IFUs and count the number of valid IFUs found */
  int i, nvalid = 0;
  for (i = 1; i <= 24; i++) {
    char *colname = cpl_sprintf("%02d", i);
    if (cpl_table_is_valid(aTable, colname, aExposure - 1)) {
      nvalid++;
    }
    cpl_free(colname);
  }
  return nvalid;
} /* muse_test_processing_frames_sort_valid_ifus_in_exposure() */

static void
muse_test_processing_frames_sort_test(const char *aDir, cpl_recipe *aRecipe)
{
#ifdef HAVE_GETTIMEOFDAY /* fail/skip, if we don't have that function, see below */
  /*
   * create 3 test exposures, with 24 files each (one for each IFU)
   * and add them to 3 different framesets (one with all of them and two
   * where something is wrong (missing IFU and wrong tag)
   */
  cpl_frameset *fall = cpl_frameset_new(), /* all files included */
               *f1 = cpl_frameset_new(), /* missing IFUs */
               *f2 = cpl_frameset_new(); /* wrong tag for some IFUs */
  cpl_propertylist *header = cpl_propertylist_new();
  cpl_table *table = cpl_table_new(0);
  const char *tag = "TEST_FILE_TAG";

  /* loop over all IFUs for 3 exposures and create the test files */
  int n;
  for (n = 1; n <= 3; n++) {
    /* create typical FITS format for DATE-OBS */
    struct timeval tv;
    gettimeofday(&tv, NULL);
    char date[24]; /* full UTC in ISO format plus ms and null byte */
    strftime(date, 20, "%FT%T", gmtime(&tv.tv_sec));
    /* write milliseconds to the same string, starting at the null byte */
    snprintf(date+19, 5, ".%03ld", tv.tv_usec/1000);

    cpl_msg_debug(__func__, "exposure %d: date=\"%s\"", n, date);
    cpl_propertylist_append_string(header, "DATE-OBS", date);

    int i;
    for (i = 1; i <= 24; i++) {
      /* change header to fit */
      char *chan = cpl_sprintf("CHAN%02d", i);
      cpl_propertylist_append_string(header, "EXTNAME", chan);
      cpl_free(chan);

      char *outfile = cpl_sprintf("%s/test_table_%1d_%02d.fits", aDir, n, i);
      cpl_table_save(table, header, NULL, outfile, CPL_IO_CREATE);

      /* create frame for this file and add it to (some of) the framesets */
      cpl_frame *frame = cpl_frame_new();
      cpl_frame_set_filename(frame, outfile);
      cpl_frame_set_tag(frame, tag);
      cpl_frameset_insert(fall, frame); /* always insert into this frameset */
      if (i != 13 && i != 20) { /* exclude these IFUs from the other set */
        cpl_frame *frame1 = cpl_frame_duplicate(frame);
        cpl_frameset_insert(f1, frame1); /* always insert into this frameset */
      }
      cpl_frame *frame2 = cpl_frame_duplicate(frame);
      if (i % n) { /* add wrong tags for some frames for this frameset */
        cpl_frame_set_tag(frame2, "WRONG_TAG");
      }
      cpl_frameset_insert(f2, frame2); /* always insert into this frameset */

      cpl_propertylist_erase(header, "EXTNAME");
      cpl_free(outfile);
    } /* for i */
    cpl_propertylist_erase(header, "DATE-OBS");

    /* the ms of the date strings will be different because of processing *
     * time, but just to be safe, sleep for at least 1s in addition       */
    //sleep(1);
  } /* for n */
  cpl_table_delete(table);
  cpl_propertylist_delete(header);

  /* create new processing structure, adapted for the needs here */
  muse_processing *proc = muse_processing_new("frames_sort", aRecipe);
  cpl_frameset_delete(proc->inframes); /* input frames are set below */
  /* need to replace tag with our tag, for the following tests to work */
  cpl_array_set_string(proc->intags, 0, tag);

  /*
   * now finally run the tests, replacing the empty with
   * a filled input frameset for each run
   */
  proc->inframes = cpl_frameset_duplicate(fall);
  table = muse_processing_sort_exposures(proc);
  cpl_test(cpl_table_get_nrow(table) == 3);
  cpl_test(muse_test_processing_frames_sort_valid_ifus_in_exposure(table, 1) == 24);
  cpl_test(muse_test_processing_frames_sort_valid_ifus_in_exposure(table, 2) == 24);
  cpl_test(muse_test_processing_frames_sort_valid_ifus_in_exposure(table, 3) == 24);
  cpl_table_delete(table);

  cpl_frameset_delete(proc->inframes);
  proc->inframes = cpl_frameset_duplicate(f1);
  table = muse_processing_sort_exposures(proc);
  cpl_test(cpl_table_get_nrow(table) == 3);
  /* 2 IFUs should be missing for all exposures */
  cpl_test(muse_test_processing_frames_sort_valid_ifus_in_exposure(table, 1) == 22);
  cpl_test(muse_test_processing_frames_sort_valid_ifus_in_exposure(table, 2) == 22);
  cpl_test(muse_test_processing_frames_sort_valid_ifus_in_exposure(table, 3) == 22);
  cpl_table_delete(table);

  cpl_frameset_delete(proc->inframes);
  proc->inframes = cpl_frameset_duplicate(f2);
  table = muse_processing_sort_exposures(proc);
  cpl_test(cpl_table_get_nrow(table) == 3);
  cpl_test(muse_test_processing_frames_sort_valid_ifus_in_exposure(table, 1) == 24);
  cpl_test(muse_test_processing_frames_sort_valid_ifus_in_exposure(table, 2) == 12); /* every second one missing */
  cpl_test(muse_test_processing_frames_sort_valid_ifus_in_exposure(table, 3) == 8); /* only every third is there */
  cpl_table_delete(table);

  /* remove all files from the disk, using the frameset that contains all frames */
  cpl_frameset_iterator *fiter = cpl_frameset_iterator_new(fall);
  cpl_frame *frame = cpl_frameset_iterator_get(fiter);
  while (frame) {
    /* using this with cpl_test_zero() might be a good idea, but then *
     * we would test more the test procedure than the function        */
    remove(cpl_frame_get_filename(cpl_frameset_iterator_get_const(fiter)));
    cpl_frameset_iterator_advance(fiter, 1);
    frame = cpl_frameset_iterator_get(fiter);
  } /* while */
  cpl_frameset_iterator_delete(fiter);

  /* cleanup of frames and this processing object */
  cpl_frameset_delete(fall);
  cpl_frameset_delete(f1);
  cpl_frameset_delete(f2);
  muse_processing_delete(proc);
#else /* no gettimeofday() available */
  cpl_msg_error(__func__, "gettimeofday() not available, cannot run this test!");
#endif
} /* muse_test_processing_frames_sort_test() */

/*----------------------------------------------------------------------------*/
/**
  @brief    Short test program to check the muse processing framework.
 */
/*----------------------------------------------------------------------------*/
int main(int argc, char **argv)
{
  UNUSED_ARGUMENTS(argc, argv);
  cpl_test_init(PACKAGE_BUGREPORT, CPL_MSG_INFO);

  char dirtemplate[FILENAME_MAX] = DIR_TEMPLATE;
  char *dir = mkdtemp(dirtemplate);
  int retval = chdir(dir);
  // test preparation: build input and test images
  muse_test_processing_prepare_test(dir);

  // here we just simulate a framework.
  // initialize plugin list
  cpl_pluginlist *list = cpl_pluginlist_new();
  cpl_plugin_get_info(list);
  cpl_test_eq(cpl_pluginlist_get_size(list), 1);
  cpl_plugin *plugin = cpl_pluginlist_get_first(list);
  cpl_test_nonnull(plugin);
  cpl_test_eq(cpl_plugin_get_type(plugin), CPL_PLUGIN_TYPE_RECIPE);
  cpl_recipe *recipe = (cpl_recipe *)plugin;

  // initialize plugin
  cpl_plugin_func init_func = cpl_plugin_get_init(plugin);
  cpl_test(init_func != NULL);
  retval = init_func(plugin);
  cpl_test_eq(retval, CPL_ERROR_NONE);

  // test muse_processing_recipeinfo(), when no frames are there yet
  cpl_errorstate state = cpl_errorstate_get();
  muse_processing_recipeinfo(plugin);
  cpl_test(cpl_errorstate_is_equal(state));

  // init (input) frames
  recipe->frames = cpl_frameset_new();
  // create a RAW frame
  char *file1 = cpl_sprintf("%s/raw.fits", dir);
  cpl_frame *frame = cpl_frame_new();
  cpl_frame_set_tag(frame, "RAW");
  cpl_frame_set_group(frame, CPL_FRAME_GROUP_RAW);
  cpl_frame_set_filename(frame, file1);
  cpl_frameset_insert(recipe->frames, frame);

  // and two tables
  char *file2 = cpl_sprintf("%s/table.fits", dir);
  frame = cpl_frame_new();
  cpl_frame_set_tag(frame, "TABLE");
  cpl_frame_set_group(frame, CPL_FRAME_GROUP_CALIB);
  cpl_frame_set_filename(frame, file2);
  cpl_frameset_insert(recipe->frames, frame);
  char *file3 = cpl_sprintf("%s/tableifu.fits", dir);
  frame = cpl_frame_new();
  cpl_frame_set_tag(frame, "TABLE_IFU");
  cpl_frame_set_group(frame, CPL_FRAME_GROUP_CALIB);
  cpl_frame_set_filename(frame, file3);
  cpl_frameset_insert(recipe->frames, frame);

  // and a bare FITS header
  char *file4 = cpl_sprintf("%s/header.fits", dir);
  frame = cpl_frame_new();
  cpl_frame_set_tag(frame, "HEADER");
  cpl_frame_set_group(frame, CPL_FRAME_GROUP_CALIB);
  cpl_frame_set_filename(frame, file4);
  cpl_frameset_insert(recipe->frames, frame);

#if 0
  cpl_frameset_dump(recipe->frames, stdout);
  fflush(stdout);
  char *cmd = cpl_sprintf("ls -la %s", dir);
  system(cmd);
  cpl_free(cmd);
#endif

  // test muse_processing_recipeinfo(), now that the frames are there
  state = cpl_errorstate_get();
  muse_processing_recipeinfo(plugin);
  cpl_test(cpl_errorstate_is_equal(state));
  // again with some environment variables set and a non-default parameter
  cpl_test_zero(setenv("OMP_NUM_THREADS", "5", 1));
  cpl_test_zero(setenv("OMP_DYNNAMIC", "TRUE", 1)); // with typo
  cpl_test_zero(setenv("MUSE_CPL_ERRORSTATE_NDUMP", "500", 1));
  cpl_test_zero(setenv("MUSE_DAR_CORRECT_METHOD", "Ciddor", 1));
  cpl_test_zero(setenv("MUSE_EXPERT", "1", 1)); // non-existant!
  cpl_parameter *p = cpl_parameterlist_find(recipe->parameters,
                                            "muse.muse_test_processing.astring");
  char *pstring = cpl_strdup(cpl_parameter_get_string(p));
  cpl_parameter_set_string(p, "BLA");
  state = cpl_errorstate_get();
  muse_processing_recipeinfo(plugin);
  cpl_test(cpl_errorstate_is_equal(state));
  // check visually, that the log does not print the ones with typos
  cpl_test_zero(unsetenv("OMP_NUM_THREADS"));
  cpl_test_zero(unsetenv("OMP_DYNNAMIC"));
  cpl_test_zero(unsetenv("MUSE_CPL_ERRORSTATE_NDUMP"));
  cpl_test_zero(unsetenv("MUSE_DAR_CORRECT_METHOD"));
  cpl_test_zero(unsetenv("MUSE_EXPERT"));
  cpl_parameter_set_string(p, pstring);
  cpl_free(pstring);
  // case for early-return
  state = cpl_errorstate_get();
  muse_processing_recipeinfo(NULL);
  cpl_test(cpl_errorstate_is_equal(state));

  // run plugin once
  cpl_plugin_func exec_func = cpl_plugin_get_exec(plugin);
  cpl_test(exec_func != NULL);
  retval = exec_func(plugin);
  cpl_test_eq(retval, CPL_ERROR_NONE);

  // run plugin again, with debug-level output
  cpl_msg_set_level(CPL_MSG_DEBUG);
  retval = exec_func(plugin);
  cpl_test_eq(retval, CPL_ERROR_NONE);
  cpl_msg_set_level(CPL_MSG_INFO);

  // before removing it, use the still existing recipe to do
  // the frames-sort test that needs a new processing structure
  muse_test_processing_frames_sort_test(dir, recipe);

  // cleanup
  cpl_plugin_func deinit_func = cpl_plugin_get_deinit(plugin);
  cpl_test(deinit_func != NULL);
  retval = deinit_func(plugin);
  cpl_test_eq(retval, CPL_ERROR_NONE);
  cpl_frameset_delete(recipe->frames);

  cpl_pluginlist_delete(list);
  muse_processinginfo_delete(recipe);
  cpl_test_zero(remove(file1));
  cpl_free(file1);
  cpl_test_zero(remove(file2));
  cpl_free(file2);
  cpl_test_zero(remove(file3));
  cpl_free(file3);
  cpl_test_zero(remove(file4));
  cpl_free(file4);

  // this removes the dir created for the results, and it also (implicitely)
  // checks that there are no more files.
  retval = chdir(".."); // avoid EBUSY on rmdir
  retval = rmdir(dir);
  cpl_test_eq(retval, 0);

  return cpl_test_end(0);
} /* main() */

/*----------------------------------------------------------------------------*/
/**
  @brief    Dummy compute routine.
  @param    aProcessing   the processing structure
  @param    aParams       the parameters list
  @return   CPL_ERROR_NONE if all tests succeeded, another CPL error otherwise
 */
/*----------------------------------------------------------------------------*/
int
muse_test_processing_compute(muse_processing *aProcessing,
                             muse_test_processing_params_t *aParams)
{
  // test parameters. They're set in muse_test_processing.xml
  cpl_test_eq(aParams->adouble, 3);
  cpl_test_eq(aParams->aboolean, FALSE);
  cpl_test_eq_string(aParams->astring, "DEF");
  cpl_test_eq(aParams->anenum, 1);

  // check loading of raw data
  muse_imagelist *images = muse_basicproc_load(aProcessing, aParams->nifu,
                                               NULL);
  cpl_test_nonnull(images);
  cpl_ensure(images, CPL_ERROR_NULL_INPUT, CPL_ERROR_NULL_INPUT);

  // just copy the input image
  muse_image *image = muse_image_new();
  image->data = cpl_image_duplicate(muse_imagelist_get(images, 0)->data);
  image->stat = cpl_image_duplicate(muse_imagelist_get(images, 0)->stat);
  image->dq = cpl_image_duplicate(muse_imagelist_get(images, 0)->dq);
  image->header = cpl_propertylist_new();
  cpl_propertylist_append_double(image->header, "ADOUBLE", 1.0);
  cpl_propertylist_append_string(image->header, "BUNIT", "adu");
  int retval;

  // loading of non-existing optional input is implicitly
  // checked during recipe initialization

  // check loading of tables
  cpl_table *table = muse_processing_load_ctable(aProcessing, "TABLE", 0);
  cpl_test_error(CPL_ERROR_NONE);
  cpl_test_nonnull(table);
  cpl_table_delete(table);
  table = muse_processing_load_ctable(aProcessing, "TABLE_IFU", 1); /* IFU-specific */
  cpl_test_error(CPL_ERROR_NONE);
  cpl_test_nonnull(table);
  cpl_table_delete(table);
  // failure case for generic file
  // (this and the following failure cases implicitly also
  // test muse_processing_load_table())
  cpl_table *table2 = muse_processing_load_ctable(aProcessing,
                                                  "TABLE_NOT_PRESENT", 0);
  cpl_test_error(CPL_ERROR_NONE); /* still no error, since this may be optional */
  cpl_test_null(table2);
  // failure case for table with IFU
  table2 = muse_processing_load_ctable(aProcessing, "TABLE_NOT_PRESENT", 10);
  cpl_test_error(CPL_ERROR_NONE); /* no error, since this may be optional */
  cpl_test_null(table2);
  // failure case of non-existant file
  // to do that, add another entry to the inframes and remove it again afterwards
  cpl_frameset *fin = cpl_frameset_duplicate(aProcessing->inframes); // save them
  cpl_frame *ffake = cpl_frame_new();
  cpl_frame_set_filename(ffake, "non-exiting.fits");
  cpl_frame_set_tag(ffake, "TABLE_FAKE_PRESENT");
  cpl_frameset_insert(aProcessing->inframes, ffake);
  table2 = muse_processing_load_ctable(aProcessing, "TABLE_FAKE_PRESENT", 0);
  cpl_test_error(CPL_ERROR_FILE_IO);
  cpl_test_null(table2);
  cpl_frameset_delete(aProcessing->inframes);
  aProcessing->inframes = fin;
  fin = NULL;

  // check loading of propertylists (like ASTROMETRY_WCS)
  cpl_propertylist *header = muse_processing_load_header(aProcessing, "HEADER");
  cpl_test_error(CPL_ERROR_NONE);
  cpl_test_nonnull(header);
  cpl_propertylist_delete(header);

  // check saving of MUSE images
  retval = muse_processing_save_image(aProcessing, aParams->nifu,
                                      image, "processing_muse_image");
  cpl_test_error(CPL_ERROR_NONE);

  // check saving of CPL images
  retval = muse_processing_save_cimage(aProcessing, aParams->nifu,
                                       image->data, image->header,
                                       "processing_cpl_image");
  cpl_test_error(CPL_ERROR_NONE);

  // check saving of CPL tables
  table = cpl_table_new(0); // will trigger case "has no rows"
  retval = muse_processing_save_table(aProcessing, aParams->nifu,
                                      table, image->header,
                                      "processing_cpl_table",
                                      MUSE_TABLE_TYPE_CPL);
  cpl_test_error(CPL_ERROR_NONE);
  cpl_table_delete(table);

  // check saving of MUSE tables
  muse_table *mtable = muse_table_new();
  mtable->table = cpl_table_new(1);
  mtable->header = cpl_propertylist_new(); // do not set AFLOAT, to trigger warning
  cpl_propertylist_append_string(mtable->header, "OBJECT", "A test MUSE table");
  cpl_propertylist_append_string(mtable->header, "ATABLE", "A stipud string...");
  retval = muse_processing_save_table(aProcessing, 0, mtable, NULL,
                                      "processing_muse_table",
                                      MUSE_TABLE_TYPE_MUSE);
  cpl_test_error(CPL_ERROR_NONE);
  // test failure case
  cpl_table_delete(mtable->table);
  mtable->table = NULL;
  retval = muse_processing_save_table(aProcessing, 0, mtable, NULL,
                                      "processing_muse_table_BAD",
                                      MUSE_TABLE_TYPE_MUSE);
  cpl_test_error(CPL_ERROR_NULL_INPUT);
  muse_table_delete(mtable);

  // check saving of FITS headers
  retval = muse_processing_save_header(aProcessing, aParams->nifu,
                                       image->header, "processing_header");
  cpl_test_error(CPL_ERROR_NONE);
  muse_image_delete(image);
  muse_imagelist_delete(images);

  // check saving of MUSE pixel tables
  muse_pixtable *pt = cpl_malloc(sizeof(muse_pixtable));
  /* give it a positive size to not fail (when saving as image) */
  pt->table = muse_cpltable_new(muse_pixtable_def, 5);
  pt->header = cpl_propertylist_new();
  pt->ffspec = NULL;
  cpl_propertylist_append_string(pt->header, "PIXEL TABLE TYPE",
                                 "ITS COMPLICATED"); /* for fun */
  retval = muse_processing_save_table(aProcessing, aParams->nifu,
                                      pt, NULL,
                                      "processing_pixel_table",
                                      MUSE_TABLE_TYPE_PIXTABLE);
  cpl_test_error(CPL_ERROR_NONE);
  muse_pixtable_delete(pt);

  // check saving of MUSE datacubes
  muse_datacube *cube = cpl_calloc(1, sizeof(muse_datacube));
  cube->header = cpl_propertylist_new();
  /* set some FITS keywords that saving of a cube requires */
  cpl_propertylist_append_string(cube->header, "OBJECT", "ANOBJECT");
  cpl_propertylist_append_string(cube->header, "BUNIT", "count");
  cube->data = cpl_imagelist_new();
  cpl_image *plane = cpl_image_new(1, 1, CPL_TYPE_FLOAT);
  cpl_imagelist_set(cube->data, plane, 0);
  cube->stat = cpl_imagelist_duplicate(cube->data);
  retval = muse_processing_save_cube(aProcessing, -1, cube,
                                      "processing_data_cube",
                                      MUSE_CUBE_TYPE_FITS);
  cpl_test_error(CPL_ERROR_NONE);
  // try saving a cube with IDP headers (supposed to fail!)
  cpl_propertylist_append_string(cube->header, "DATE-OBS",
                                 "2016-09-14T12:30:00.002");
  retval = muse_processing_save_cube(aProcessing, -1, cube,
                                      "processing_data_cube_with_idp_BAD",
                                      MUSE_CUBE_TYPE_SDP);
  cpl_test_error(CPL_ERROR_DATA_NOT_FOUND); /* missing important headers */
  muse_datacube_delete(cube);

  // check saving of MUSE masks
  muse_mask *mask = muse_mask_new();
  mask->header = cpl_propertylist_new();
  cpl_propertylist_append_string(mask->header, "OBJECT", "A MASK");
  mask->mask = cpl_mask_new(10, 10);
  cpl_mask_set(mask->mask, 3, 3, CPL_BINARY_1);
  retval = muse_processing_save_mask(aProcessing, -1, mask,
                                     "processing_mask");
  cpl_test_error(CPL_ERROR_NONE);
  muse_mask_delete(mask);
  mask = NULL;

  // check loading the MUSE mask back (this only works on the 2nd run!)
  cpl_errorstate state = cpl_errorstate_get();
  mask = muse_processing_load_mask(aProcessing, "processing_mask");
  if (mask) {
    cpl_test_error(CPL_ERROR_NONE);
    cpl_test_nonnull(mask);
    cpl_test_nonnull(mask->header);
    cpl_test_nonnull(mask->mask);
    muse_mask_delete(mask);
    /* in this case, remove the loaded frame again from the used frames, *
     * so that the CPL DFS functions tested below work the same way      */
    cpl_frameset_erase(aProcessing->usedframes, "processing_mask");
  } else {
    cpl_errorstate_set(state);
  }

  // check the output frameset
  cpl_test_eq(cpl_frameset_get_size(aProcessing->outframes), 8);
  cpl_frame *frame;
  frame = cpl_frameset_find(aProcessing->outframes, "processing_muse_image");
  cpl_test_nonnull(frame);
  if (frame != NULL) {
    cpl_test_eq_string(cpl_frame_get_filename(frame),
                       "processing_muse_image-01.fits");
    cpl_test_eq(cpl_frame_get_group(frame), CPL_FRAME_GROUP_PRODUCT);
    cpl_test_eq(cpl_frame_get_level(frame), CPL_FRAME_LEVEL_FINAL);
    cpl_test_eq(cpl_frame_get_type(frame), CPL_FRAME_TYPE_IMAGE);
    cpl_test_eq(access(cpl_frame_get_filename(frame), F_OK), 0);
  }
  frame = cpl_frameset_find(aProcessing->outframes, "processing_cpl_image");
  cpl_test_nonnull(frame);
  if (frame != NULL) {
    cpl_test_eq_string(cpl_frame_get_filename(frame),
                       "processing_cpl_image_0001-01.fits");
    cpl_test_eq(cpl_frame_get_group(frame), CPL_FRAME_GROUP_PRODUCT);
    cpl_test_eq(cpl_frame_get_level(frame), CPL_FRAME_LEVEL_TEMPORARY);
    cpl_test_eq(cpl_frame_get_type(frame), CPL_FRAME_TYPE_IMAGE);
    cpl_test_eq(access(cpl_frame_get_filename(frame), F_OK), 0);
  }
  frame = cpl_frameset_find(aProcessing->outframes, "processing_cpl_table");
  cpl_test_nonnull(frame);
  if (frame != NULL) {
    cpl_test_eq_string(cpl_frame_get_filename(frame),
                       "processing_cpl_table-01.fits");
    cpl_test_eq(cpl_frame_get_group(frame), CPL_FRAME_GROUP_PRODUCT);
    cpl_test_eq(cpl_frame_get_level(frame), CPL_FRAME_LEVEL_FINAL);
    cpl_test_eq(cpl_frame_get_type(frame), CPL_FRAME_TYPE_TABLE);
    cpl_test_eq(access(cpl_frame_get_filename(frame), F_OK), 0);
  }
  frame = cpl_frameset_find(aProcessing->outframes, "processing_pixel_table");
  cpl_test_nonnull(frame);
  if (frame != NULL) {
    cpl_test_eq_string(cpl_frame_get_filename(frame),
                       "processing_pixel_table-01.fits");
    cpl_test_eq(cpl_frame_get_group(frame), CPL_FRAME_GROUP_PRODUCT);
    cpl_test_eq(cpl_frame_get_level(frame), CPL_FRAME_LEVEL_FINAL);
    cpl_test_eq(cpl_frame_get_type(frame), CPL_FRAME_TYPE_TABLE);
    cpl_test_eq(access(cpl_frame_get_filename(frame), F_OK), 0);
  }
  frame = cpl_frameset_find(aProcessing->outframes, "processing_header");
  cpl_test_nonnull(frame);
  if (frame != NULL) {
    cpl_test_eq_string(cpl_frame_get_filename(frame),
                       "processing_header-01.fits");
    cpl_test_eq(cpl_frame_get_group(frame), CPL_FRAME_GROUP_PRODUCT);
    cpl_test_eq(cpl_frame_get_level(frame), CPL_FRAME_LEVEL_FINAL);
    cpl_test_eq(cpl_frame_get_type(frame), CPL_FRAME_TYPE_IMAGE);
    cpl_test_eq(access(cpl_frame_get_filename(frame), F_OK), 0);
  }
  frame = cpl_frameset_find(aProcessing->outframes, "processing_data_cube");
  cpl_test_nonnull(frame);
  if (frame != NULL) {
    cpl_test_eq_string(cpl_frame_get_filename(frame),
                       "processing_data_cube.fits");
    cpl_test_eq(cpl_frame_get_group(frame), CPL_FRAME_GROUP_PRODUCT);
    cpl_test_eq(cpl_frame_get_level(frame), CPL_FRAME_LEVEL_FINAL);
    cpl_test_eq(cpl_frame_get_type(frame), CPL_FRAME_TYPE_IMAGE);
    cpl_test_eq(access(cpl_frame_get_filename(frame), F_OK), 0);
  }
  frame = cpl_frameset_find(aProcessing->outframes, "processing_mask");
  cpl_test_nonnull(frame);
  if (frame != NULL) {
    cpl_test_eq_string(cpl_frame_get_filename(frame), "processing_mask.fits");
    cpl_test_eq(cpl_frame_get_group(frame), CPL_FRAME_GROUP_PRODUCT);
    cpl_test_eq(cpl_frame_get_level(frame), CPL_FRAME_LEVEL_FINAL);
    cpl_test_eq(cpl_frame_get_type(frame), CPL_FRAME_TYPE_IMAGE);
    cpl_test_eq(access(cpl_frame_get_filename(frame), F_OK), 0);
  }

  /* clean up */
  cpl_frameset_iterator *fiter = cpl_frameset_iterator_new(aProcessing->outframes);
  frame = cpl_frameset_iterator_get(fiter);
  while (frame) {
    cpl_msg_debug(__func__, "deleting \"%s\"", cpl_frame_get_filename(frame));
    cpl_test(remove(cpl_frame_get_filename(frame)) == 0);
    cpl_frameset_iterator_advance(fiter, 1);
    frame = cpl_frameset_iterator_get(fiter);
  } /* while */
  cpl_frameset_iterator_delete(fiter);

  /* Test some failure cases. All these functions just output   *
   * ERROR messages, they don't set error states by themselves. */
  state = cpl_errorstate_get();
  muse_processing_delete(NULL);
  cpl_test(cpl_errorstate_is_equal(state));
  /* try to create a frame, but first fail without processing struct */
  cpl_propertylist *hdr = cpl_propertylist_new();
  cpl_propertylist_append_string(hdr, "DATE-OBS", "2013-08-26T15:55:55.555");
  cpl_propertylist_append_string(hdr, "MUSE PRIVATE SOMETHING", "am I here?");
  state = cpl_errorstate_get();
  cpl_frame *testframe = muse_processing_new_frame(NULL, 10, hdr, "BAD_TAG",
                                                   CPL_FRAME_TYPE_ANY);
  cpl_test(!cpl_errorstate_is_equal(state)
           && cpl_error_get_code() == CPL_ERROR_NULL_INPUT);
  cpl_errorstate_set(state);
  cpl_test_null(testframe);
  /* this will issue a WARNING, because BAD_TAG is not in the XML */
  testframe = muse_processing_new_frame(aProcessing, 10, hdr, "BAD_TAG",
                                        CPL_FRAME_TYPE_ANY);
  cpl_test_nonnull(testframe); /* this time it should have succeeded */
  cpl_test(!cpl_propertylist_has(hdr, "MUSE PRIVATE SOMETHING")); /* be removed */
  /* now use this frame to generate a failure when appending used */
  state = cpl_errorstate_get();
  muse_processing_append_used(NULL, testframe, CPL_FRAME_GROUP_RAW, 1);
  cpl_test(cpl_errorstate_is_equal(state)); /* just silently fails */
  cpl_frame_delete(testframe);
  cpl_propertylist_delete(hdr);
  /* one last check, of muse_processing_check_input() */
  state = cpl_errorstate_get();
  cpl_test(muse_processing_check_input(NULL, 10) == CPL_ERROR_NULL_INPUT);
  cpl_test(cpl_errorstate_is_equal(state)); /* no other error can be propagated */

  return retval;
} /* muse_test_processing_compute() */