/* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set sw=2 sts=2 et cin: */
/*
 * This file is part of the MUSE Instrument Pipeline
 * Copyright (C) 2015-2016 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

/*---------------------------------------------------------------------------*
 *                             Includes                                      *
 *---------------------------------------------------------------------------*/
#include <string.h>
#include <math.h>
#include <cpl.h>
#include <muse.h>
#include "muse_lingain_z.h"

/* XXX: Do not add the residual non-linearity model description   *
 * to the final product until the model has been fully validated. *
 * At the moment there are still discrepancies compared to the    *
 * prototype implementation.                                      */
#undef NONLINEARITY_MODEL

typedef void (*muse_free_func)(void *);

struct muse_measurand_t {
    double value;
    double error;
};

typedef struct muse_measurand_t muse_measurand_t;

struct muse_gain_fit_t
{
    muse_measurand_t gain;
    double signal_range[2];
};

typedef struct muse_gain_fit_t muse_gain_fit_t;

struct muse_linearity_fit_t
{
    cpl_array *coefficients;
    double range[2];
    double rms;
};

typedef struct muse_linearity_fit_t muse_linearity_fit_t;

/*---------------------------------------------------------------------------*
 *                             Functions code                                *
 *---------------------------------------------------------------------------*/

/* XXX: Make this a public function. Could go to muse_utils? */
static void
muse_vfree(void **array, cpl_size size, muse_free_func deallocator)
{
  if (array) {
    cpl_size idx;
    for (idx = 0; idx < size; ++idx) {
      if (deallocator) {
        deallocator(array[idx]);
      }
    }
    cpl_free(array);
  }
  return;
}

static void
muse_linearity_fit_clear(muse_linearity_fit_t *aFit)
{
  if (aFit) {
    if (aFit->coefficients) {
      cpl_array_delete(aFit->coefficients);
    }
  }
  return;
}

/*---------------------------------------------------------------------------*/
/**
  @brief    Validate the recipe parameters.
  @param    aParams  the recipe parameters to validate.
  @return   CPL_TRUE if all parameters are valid, otherwise CPL_FALSE.

  The function checks whether the provided recipe parameters are valid,
  in the sense that the given values can be used without double checking
  basic properties like, for instance, the upper limit of a range is larger
  than the lower limit.
 */
/*---------------------------------------------------------------------------*/
static cpl_boolean
muse_lingain_validate_parameters(muse_lingain_params_t *aParams)
{
  if (aParams->ybox <= 0) {
    cpl_msg_error(__func__, "Invalid measurement window size: window must "
                  "be larger than 0!");
    return CPL_FALSE;
  }

  if (aParams->xgap < 0) {
    cpl_msg_error(__func__, "Invalid tracing edge offset: offset is "
                  "less than 0!");
    return CPL_FALSE;
  }

  if (aParams->xborder < 0) {
    cpl_msg_error(__func__, "Invalid offset from detector edge: offset is "
                  "less than 0!");
    return CPL_FALSE;
  }

  if (aParams->order < 0) {
    cpl_msg_error(__func__, "Invalid polynomial fit order: polynomial "
                  "degree is less than 0!");
    return CPL_FALSE;
  }

  if (aParams->toffset < 0.) {
    cpl_msg_error(__func__, "Invalid exposure time offset: offset is less "
                  "than 0.!");
    return CPL_FALSE;
  }

  if (aParams->sigma < 0.) {
    cpl_msg_error(__func__, "Invalid sigma clipping threshold used for "
                  "signal value cleaning: threshold is less than 0.!");
    return CPL_FALSE;
  }

  if (aParams->signalmin < 0.) {
    cpl_msg_error(__func__, "Invalid signal level grid: minimum signal is "
                  "less than 0.!");
    return CPL_FALSE;
  }

  if (aParams->signalbin <= 0.) {
    cpl_msg_error(__func__, "Invalid signal level grid: signal bin size "
                  "must larger than 0.!");
    return CPL_FALSE;
  }

  if (aParams->signalmax < aParams->signalmin + aParams->signalbin) {
    cpl_msg_error(__func__, "Invalid signal level grid: maximum signal is "
                  "too small for chosen minimum value and bin size!");
    return CPL_FALSE;
  }

  if (aParams->gainlimit < 0.) {
    cpl_msg_error(__func__, "Invalid minimum signal used for gain fit: "
                  "minimum signal is less than 0.!");
    return CPL_FALSE;
  }

  if (aParams->gainsigma < 0.) {
    cpl_msg_error(__func__, "Invalid sigma clipping threshold used for "
                  "gain value cleaning: threshold is less than 0.!");
    return CPL_FALSE;
  }

  if (aParams->ctsmin < 0.) {
    cpl_msg_error(__func__, "Invalid signal level grid used for "
                  "non-linearity analysis: minimum signal is "
                  "less than 0.!");
    return CPL_FALSE;
  }

  if (aParams->ctsbin <= 0.) {
    cpl_msg_error(__func__, "Invalid signal level grid used for "
                  "non-linearity analysis: signal bin size must larger "
                  "than 0.!");
    return CPL_FALSE;
  }

  if (aParams->ctsmax < aParams->ctsmin + aParams->ctsbin) {
    cpl_msg_error(__func__, "Invalid signal level grid used for "
                  "non-linearity analysis: maximum signal is "
                  "too small for chosen minimum value and bin size!");
    return CPL_FALSE;
  }

  if (aParams->linearmin < 0.) {
    cpl_msg_error(__func__, "Invalid signal range used for residual "
                  "non-linearity fit: minimum signal is less than 0.!");
    return CPL_FALSE;
  }

  if (aParams->linearmax <= aParams->linearmin) {
    cpl_msg_error(__func__, "Invalid signal range used for residual "
                  "non-linearity fit: maximum signal is too small for "
                  "the cosen minimum value!");
    return CPL_FALSE;
  }

  return CPL_TRUE;
}

/*---------------------------------------------------------------------------*/
/**
  @brief    Load and pre-process the raw images (or skip loading them).
  @param    aProcessing  the processing structure
  @param    aIFU         the IFU number
  @return   The imagelist of pre-reduced exposures.
 */
/*---------------------------------------------------------------------------*/
static muse_imagelist *
muse_lingain_load_images(muse_processing *aProcessing, unsigned short aIFU)
{
  muse_imagelist *images;

  /* Look for the master bias of the current IFU and determine the      *
   * basic processing parameters so that a consistent set of parameters *
   * is used here.                                                      */
  cpl_frame *bias = muse_frameset_find_master(aProcessing->inframes,
                                              MUSE_TAG_MASTER_BIAS, aIFU);

  /* Get primary header containing the processing options used during *
   * the master bias creation.                                        */
  cpl_propertylist *properties = cpl_propertylist_load(cpl_frame_get_filename(bias), 0);
  cpl_frame_delete(bias);
  muse_basicproc_params *bpars = muse_basicproc_params_new_from_propertylist(properties);
  cpl_propertylist_delete(properties);

  images = muse_basicproc_load(aProcessing, aIFU, bpars);
  muse_basicproc_params_delete(bpars);

  return images;
}

/*---------------------------------------------------------------------------*/
/**
  @brief    Create an index table for the exposures of a linearity sequence.
  @param    aList  the image list
  @return   The index table of the exposures.

  The function creates a sorted index table for the input image list. The
  images in the list are sorted by exposure time and observation time, both
  in ascending order.
 */
/*---------------------------------------------------------------------------*/
static cpl_table *
muse_lingain_sort_images(const muse_imagelist *aList)
{
  cpl_ensure(aList, CPL_ERROR_NULL_INPUT, NULL);

  cpl_errorstate status = cpl_errorstate_get();

  cpl_table *exposures = cpl_table_new(aList->size);

  cpl_table_new_column(exposures, "INDEX", CPL_TYPE_INT);
  cpl_table_new_column(exposures, "TYPE", CPL_TYPE_STRING);
  cpl_table_new_column(exposures, "MJDOBS", CPL_TYPE_DOUBLE);
  cpl_table_new_column(exposures, "EXPTIME", CPL_TYPE_DOUBLE);

  unsigned int iexposure;
  for (iexposure = 0; iexposure < aList->size; ++iexposure) {
    cpl_propertylist *header = muse_imagelist_get((muse_imagelist *)aList, iexposure)->header;
    cpl_table_set_int(exposures, "INDEX", iexposure, iexposure);
    if (strncmp(muse_pfits_get_dpr_type(header), "BIAS,DETCHECK", 14) == 0) {
      cpl_table_set_string(exposures, "TYPE", iexposure,
                           MUSE_TAG_LINEARITY_BIAS);
    } else {
      cpl_table_set_string(exposures, "TYPE", iexposure,
                           MUSE_TAG_LINEARITY_FLAT);
    }
    cpl_table_set_double(exposures, "MJDOBS", iexposure,
                         muse_pfits_get_mjdobs(header));
    cpl_table_set_double(exposures, "EXPTIME", iexposure,
                         muse_pfits_get_exptime(header));
  }

  /* This assumes that the bias frames always have an exposure time *
   * which is smaller than that of the shortest flat field exposure *
   * and thus are sorted into the table as the first entries.       */
  cpl_propertylist *order = cpl_propertylist_new();
  cpl_propertylist_append_bool(order, "EXPTIME", CPL_FALSE);
  cpl_propertylist_append_bool(order, "MJDOBS", CPL_FALSE);

  cpl_table_sort(exposures, order);
  cpl_propertylist_delete(order);

  if (!cpl_errorstate_is_equal(status)) {
    cpl_table_delete(exposures);
    return NULL;
  }

  /* Validate exposure table: there should be pairs of the same type and *
   * with the same exposure time                                         */
  cpl_boolean invalid = CPL_FALSE;
  cpl_size npairs = (cpl_size)(aList->size / 2);
  cpl_size ipair;
  for (ipair = 0; ipair < npairs; ++ipair) {
    register cpl_size jpair = 2 * ipair;
    register cpl_size kpair = jpair + 1;
    if (strcmp(cpl_table_get_string(exposures, "TYPE", jpair),
               cpl_table_get_string(exposures, "TYPE", kpair)) != 0) {
      cpl_msg_error(__func__, "Invalid pair of exposures: Type of exposure "
                    "does not match: Got '%s', expected '%s'!",
                    cpl_table_get_string(exposures, "TYPE", jpair),
                    cpl_table_get_string(exposures, "TYPE", kpair));
      invalid = CPL_TRUE;
      break;
    }
    if ((cpl_table_get_double(exposures, "EXPTIME", jpair, NULL) -
        cpl_table_get_double(exposures, "EXPTIME", kpair, NULL)) > 0.1) {
      cpl_msg_error(__func__, "Invalid pair of exposures: exposure times "
                    "do not match: Got '%.4f', expected '%.4f'!",
                    cpl_table_get_double(exposures, "EXPTIME", jpair, NULL),
                    cpl_table_get_double(exposures, "EXPTIME", kpair, NULL));
      invalid = CPL_TRUE;
      break;
    }
  }

  if (invalid) {
    cpl_table_delete(exposures);
    cpl_error_set(__func__, CPL_ERROR_ILLEGAL_OUTPUT);
    return NULL;
  }

  /* If there is an orphan exposure left, remove it from the table */
  cpl_size nexposures = 2 * npairs;
  if (aList->size > nexposures) {
    cpl_table_erase_window(exposures, nexposures, aList->size - nexposures);
  }

  return exposures;
}

/*---------------------------------------------------------------------------*/
/**
  @brief    Create a list of measurement windows for the linearity sequence
            and a list of traces.
  @param    aTrace        The traces of the slices to be used
  @param    aExposureList The index table of valid exposures
  @param    aList         The image list
  @param    aParams       The parameter list
  @return   The list measurement windows positioned along the slices.
 */
/*---------------------------------------------------------------------------*/
static cpl_table *
muse_lingain_create_windowlist(const cpl_table *aTrace,
                               const cpl_table *aExposureList,
                               const muse_imagelist *aList,
                               muse_lingain_params_t *aParams)
{
  cpl_ensure(aTrace && aExposureList && aList && aParams,
             CPL_ERROR_NULL_INPUT, NULL);

  cpl_table *windowlist = cpl_table_new(0);
  cpl_table_new_column(windowlist, "Quadrant", CPL_TYPE_INT);
  cpl_table_new_column(windowlist, "SliceNo", CPL_TYPE_INT);
  cpl_table_new_column(windowlist, "Xmin", CPL_TYPE_INT);
  cpl_table_new_column(windowlist, "Ymin", CPL_TYPE_INT);
  cpl_table_new_column(windowlist, "Xmax", CPL_TYPE_INT);
  cpl_table_new_column(windowlist, "Ymax", CPL_TYPE_INT);

  /* Get quadrant definitions from the first image in the exposure list */
  /* XXX: Verify that all exposures have the same quadrant definitions? */

  int idx = cpl_table_get_int(aExposureList, "INDEX", 0, NULL);

  cpl_errorstate status = cpl_errorstate_get();

  muse_image *image = muse_imagelist_get((muse_imagelist *)aList, idx);
  cpl_size firstrow = 0;

  unsigned char iquadrant;
  for (iquadrant = 1; iquadrant <= 4; ++iquadrant) {
    cpl_size *qwindow = muse_quadrants_get_window(image, iquadrant);

    unsigned short islice;
    for (islice = 1; islice <= kMuseSlicesPerCCD; ++islice) {
      cpl_errorstate _status = cpl_errorstate_get();
      cpl_polynomial **traces = muse_trace_table_get_polys_for_slice(aTrace,
                                                                     islice);
      /* If no tracing polynomial is found for the current slice index *
       * issue a warning and continue with the next slice.             */
      if (!traces) {
        cpl_msg_warning(__func__, "slice %2hu of IFU %d: tracing polynomials "
                                  "missing!", islice, aParams->nifu);
        muse_trace_polys_delete(traces);
        cpl_errorstate_set(_status);
        continue;
      }
      cpl_size torder = 0;
      double ledge = cpl_polynomial_get_coeff(traces[MUSE_TRACE_LEFT], &torder);
      double redge = cpl_polynomial_get_coeff(traces[MUSE_TRACE_RIGHT], &torder);
      if ((ledge < qwindow[0] + aParams->xborder) ||
          (redge > qwindow[1] - aParams->xborder)) {
        muse_trace_polys_delete(traces);
        continue;
      }
      else {
        unsigned int nwindows = (qwindow[3] - qwindow[2] + 1) / aParams->ybox;

        /* Add nwindows empty rows for the next batch of windows */
        cpl_table_set_size(windowlist, firstrow + nwindows);

        unsigned int iwindow;
        for (iwindow = 0; iwindow < nwindows; ++iwindow) {
          cpl_size irow = firstrow + iwindow;

          int ymin = iwindow * aParams->ybox + qwindow[2];
          int ymax = ymin + aParams->ybox - 1;

          double ymid = ymin + 0.5 * aParams->ybox;
          double xleft = cpl_polynomial_eval_1d(traces[MUSE_TRACE_LEFT], ymid, NULL);
          double xright = cpl_polynomial_eval_1d(traces[MUSE_TRACE_RIGHT], ymid, NULL);

          int xmin = (int)rint(xleft) + aParams->xgap;
          int xmax = (int)rint(xright) - aParams->xgap;

          cpl_table_set_int(windowlist, "Quadrant", irow, iquadrant);
          cpl_table_set_int(windowlist, "SliceNo", irow, islice);
          cpl_table_set_int(windowlist, "Xmin", irow, xmin);
          cpl_table_set_int(windowlist, "Xmax", irow, xmax);
          cpl_table_set_int(windowlist, "Ymin", irow, ymin);
          cpl_table_set_int(windowlist, "Ymax", irow, ymax);
        }
        firstrow += nwindows;
      }

      muse_trace_polys_delete(traces);
    }

    cpl_free(qwindow);
  }

  if (!cpl_errorstate_is_equal(status)) {
    cpl_table_delete(windowlist);
    return NULL;
  }

  return windowlist;
}

/*---------------------------------------------------------------------------*/
/**
  @brief    Create an array of window indices located in a given quadrant.
  @param    aWindowList  The window list to be queried.
  @param    aQuadrant    Detector quadrant to be selected.
  @return   The array of window indices or NULL if no windows were found.

  Note that any selection which has been applied to the window list prior
  to calling this function will be reset. On return all rows in the window
  list are selected!
 */
/*---------------------------------------------------------------------------*/
static cpl_array *
muse_lingain_quadrant_get_windows(cpl_table *aWindowList,
                                  unsigned char aQuadrant)
{
  cpl_table_select_all(aWindowList);

  /* Determine the window indices for the current detector quadrant */
  cpl_size nwindows = cpl_table_and_selected_int(aWindowList, "Quadrant",
                                                 CPL_EQUAL_TO, aQuadrant);
  if (nwindows == 0) {
    return NULL;
  }

  cpl_array *windex = cpl_array_new(nwindows, CPL_TYPE_SIZE);

  cpl_size iwindow = 0;
  cpl_size irow;
  for (irow = 0; irow < cpl_table_get_nrow(aWindowList); ++irow) {
      if (cpl_table_is_selected(aWindowList, irow)) {
          cpl_array_set_cplsize(windex, iwindow, irow);
          ++iwindow;
      }
  }

  /* Reset window list selection */
  cpl_table_select_all(aWindowList);

  return windex;
}

/*---------------------------------------------------------------------------*/
/**
  @brief    Extract the data given by a list of window indices.
  @param    aGainTable  The gain table from which the data is extracted.
  @param    aWindows    The array of window indices to extract.
  @param    aColumn     The name of the data column from which the data is
                        taken.
  @param    aRow        The row index from which the data is taken.
  @return   The array of extracted data values or NULL if an error occurs.

  TBD
 */
/*---------------------------------------------------------------------------*/
static cpl_array *
muse_lingain_quadrant_extract_data(const cpl_table *aGainTable,
                                   const cpl_array *aWindows,
                                   const char *aName,
                                   cpl_size aRow)
{

  const cpl_array *src = cpl_table_get_array(aGainTable, aName, aRow);

  if (!src) {
    return NULL;
  }

  cpl_size nwindows = cpl_array_get_size(aWindows);
  cpl_array *data = cpl_array_new(nwindows, CPL_TYPE_DOUBLE);

  cpl_size iwindow;
  for (iwindow = 0; iwindow < nwindows; ++iwindow) {
    cpl_size jwindow = cpl_array_get_cplsize(aWindows, iwindow, NULL);
    int invalid = 0;
    double value = cpl_array_get_double(src, jwindow, &invalid);

    if (!invalid) {
      cpl_array_set_double(data, iwindow, value);
    } else {
      cpl_array_set_invalid(data, iwindow);
    }
  }

  return data;

}

/*---------------------------------------------------------------------------*/
/**
  @brief    Create a list of RON measurements from a list of windows
            and a list of linearity sequence images.
  @param    aWindowList   The list of measurement windows to use.
  @param    aExposureList The index table of valid exposures
  @param    aImageList    The image list
  @param    aParams       The parameter list
  @return   A table with the computed per window RON values, or NULL if an
            error occurs.

  The function computes for each measurement window defined in the input
  window list a RON value using the standard deviation measured on the
  difference of the first pair of biases found in the exposure list.
  An error estimate of the RON is calculated using the standard deviation
  of the standard deviation.

  The function resets any existing selection of the exposure list!
 */
/*---------------------------------------------------------------------------*/
static cpl_table *
muse_lingain_window_get_ron(const cpl_table *aWindowList,
                            cpl_table *aExposureList,
                            muse_imagelist *aImageList,
                            muse_lingain_params_t *aParams)
{
  cpl_table_select_all(aExposureList);
  cpl_size nbias = cpl_table_and_selected_string(aExposureList, "TYPE", CPL_EQUAL_TO,
                                                 MUSE_TAG_LINEARITY_BIAS);
  if (nbias > 2) {
    cpl_msg_warning(__func__, "Found more than 2 (%" CPL_SIZE_FORMAT ") "
                    "input images of type '%s' for IFU %u! Only the first 2 "
                    "images will be used!", nbias, MUSE_TAG_LINEARITY_BIAS,
                    aParams->nifu);
  }

  cpl_size ibias = 0;
  cpl_size iexposure;
  for (iexposure = 0; iexposure < cpl_table_get_nrow(aExposureList); ++iexposure) {
    if (cpl_table_is_selected(aExposureList, iexposure)) {
      ibias = cpl_table_get_int(aExposureList, "INDEX", iexposure, NULL);
      break;
    }
  }
  cpl_table_select_all(aExposureList);

  /* Get bias images to compute the RON per window on the difference image *
   * of the two bias images. Reject bad image pixels according to the data *
   * quality information so that they don't affect the results.            */
  muse_image *bias1 = muse_imagelist_get(aImageList, ibias);
  muse_image *bias2 = muse_imagelist_get(aImageList, ibias + 1);

  muse_image_reject_from_dq(bias1);
  muse_image_reject_from_dq(bias2);

  cpl_image *dbias = cpl_image_subtract_create(bias1->data, bias2->data);

  cpl_table *ron = cpl_table_new(cpl_table_get_nrow(aWindowList));
  cpl_table_new_column(ron, "Ron", CPL_TYPE_DOUBLE);
  cpl_table_new_column(ron, "RonErr", CPL_TYPE_DOUBLE);

  cpl_size iwindow;
  for (iwindow = 0; iwindow < cpl_table_get_nrow(aWindowList); ++ iwindow) {
    int xmin = cpl_table_get_int(aWindowList, "Xmin", iwindow, NULL);
    int ymin = cpl_table_get_int(aWindowList, "Ymin", iwindow, NULL);
    int xmax = cpl_table_get_int(aWindowList, "Xmax", iwindow, NULL);
    int ymax = cpl_table_get_int(aWindowList, "Ymax", iwindow, NULL);

    /* The ron is estimated from the standard deviation of the difference *
     * image. The standard deviation from the difference image has noise  *
     * contributions from both images and needs to be scaled by the       *
     * square root of 0.5 to get the noise contribution of a single       *
     * image. The scale factor results from inverting the error           *
     * propagation assuming both images have the same error, the RON.     */
    double _ron = sqrt(0.5) * cpl_image_get_stdev_window(dbias, xmin, ymin,
                                                         xmax, ymax);

    /* The error of the ron is estimated from the standard deviation of the *
     * standard deviation of the difference image. Since computing the ron  *
     * using cpl_image_get_stdev_window() ignores bad pixels, the following *
     * computation takes this into account when using sample size npixel.   */
    double npixel = (xmax - xmin + 1) * (ymax - ymin + 1);
    const cpl_mask *mask = cpl_image_get_bpm_const(dbias);

    if (mask) {
      cpl_size ninvalid = cpl_mask_count_window(mask, xmin, ymin, xmax, ymax);
      npixel -= ninvalid;
    }
    npixel *= 0.5; 
    
    /* Compute the error of the ron as unbiased estimator of the standard *
     * deviation of the standard deviation of the difference image. Note  *
     * that the final expression just scales the ron value, so that the   *
     * scaling by sqrt(0.5) is already taken into account.                */
    double g = lgamma(npixel) - lgamma(npixel - 0.5);
    double g1 = exp(-g);
    double g2 = exp(g);
    double _ronerr = _ron * g1 * sqrt(npixel - 0.5 - g2 * g2);

    cpl_table_set_double(ron, "Ron", iwindow, _ron);
    cpl_table_set_double(ron, "RonErr", iwindow, _ronerr);
  }
  cpl_image_delete(dbias);

  return ron;
}

/*---------------------------------------------------------------------------*/
/**
  @brief    Create a list of local signal and gain measurements for a set
            of measurement windows.
  @param    aWindowList   The list of measurement windows to use.
  @param    aRon          Array of RON measurements for the set of windows
  @param    aExposureList The index table of valid exposures
  @param    aImageList    The image list
  @param    aParams       The parameter list
  @return   A table with the computed per window signal, variance and gain
            estimates, or NULL if an error occurs.

  The function computes for each measurement window defined in the input
  window list an estimate for the signal, its variance and the (inverse)
  gain (i.e. signal / variance) for each pair of flat fields found in the
  exposure list.

  Flat field pairs for which the overall flux level of its two members
  differs too much, are skipped.

  The function resets any existing selection of the exposure list!
 */
/*---------------------------------------------------------------------------*/
static cpl_table *
muse_lingain_window_get_gain(const cpl_table *aWindowList,
                             const cpl_array *aRon,
                             cpl_table *aExposureList,
                             muse_imagelist *aImageList,
                             muse_lingain_params_t *aParams)
{

  /* Create a local table containing only the flat field images of the *
   * linearity sequence.                                               */
  cpl_table_select_all(aExposureList);
  cpl_size nflats = cpl_table_and_selected_string(aExposureList, "TYPE", CPL_EQUAL_TO,
                                                 MUSE_TAG_LINEARITY_FLAT);
  if (nflats % 2) {
    cpl_msg_warning(__func__, "Found odd number of input images (%" CPL_SIZE_FORMAT
                    ") input images of type '%s' for IFU %u! Exposure series "
                    "may be incorrect!", nflats, MUSE_TAG_LINEARITY_FLAT,
                    aParams->nifu);
  }

  cpl_table *flats = cpl_table_extract_selected(aExposureList);
  cpl_table_select_all(aExposureList);

  /* Create output table containing the gain measurements */
  cpl_errorstate state = cpl_errorstate_get();

  cpl_size npairs   = nflats / 2;
  cpl_size nwindows = cpl_table_get_nrow(aWindowList);

  cpl_table *gain = cpl_table_new(npairs);
  cpl_table_new_column(gain, "ExpTime", CPL_TYPE_DOUBLE);
  cpl_table_new_column_array(gain, "Signal", CPL_TYPE_DOUBLE, nwindows);
  cpl_table_new_column_array(gain, "Variance", CPL_TYPE_DOUBLE, nwindows);
  cpl_table_new_column_array(gain, "Gain", CPL_TYPE_DOUBLE, nwindows);

  if (!cpl_errorstate_is_equal(state)) {
    cpl_table_delete(gain);
    cpl_table_delete(flats);
    return NULL;
  }

  /* Get median RON and its standard deviation for the computation of the *
   * gain for each measurement window.                                    */
  double mron_value = cpl_array_get_median(aRon);
  double mron_sdev  = cpl_array_get_stdev(aRon);

  /* Compute a gain estimate for each pair of input flat fields and each *
   * measurement window                                                  */
  cpl_size kpair = 0;
  cpl_size ipair;
  for (ipair = 0; ipair < npairs; ++ipair) {
    cpl_size iexposure = 2 * ipair;
    cpl_size jexposure = iexposure + 1;
    int iflat = cpl_table_get_int(flats, "INDEX", iexposure, NULL);
    int jflat = cpl_table_get_int(flats, "INDEX", jexposure, NULL);
    muse_image *flat1 = muse_imagelist_get(aImageList, iflat);
    muse_image *flat2 = muse_imagelist_get(aImageList, jflat);

    double exptime1 = cpl_table_get_double(flats, "EXPTIME", iexposure, NULL);
    double exptime2 = cpl_table_get_double(flats, "EXPTIME", jexposure, NULL);
    double flux1 = cpl_image_get_mean(flat1->data);
    double flux2 = cpl_image_get_mean(flat2->data);

    if (fabs((flux1 - flux2) / flux1) > aParams->fluxtol) {
      cpl_msg_warning(__func__, "Inconsistent overall flux level (%.4f, "
                      "%.4f) detected for flat field pair (%d, %d) of IFU %u "
                      "with exposure time %.4f! Skipping this pair!",
                      flux1, flux2, iflat, jflat, aParams->nifu, exptime1);
      continue;
    }

    cpl_table_set_double(gain, "ExpTime", kpair, 0.5 * (exptime1 + exptime2));

    /* Exclude known bad pixels from the following analysis */
    muse_image_reject_from_dq(flat1);
    muse_image_reject_from_dq(flat2);

    /* Compute gain estimate for each measurement window. Use the union  *
     * of the two data quality masks so that the measurement is done for *
     * the same pixels in both images. If sigma clipping is enabled the  *
     * computations are done on the cleaned data.                        */
    cpl_array *_signal   = cpl_array_new(nwindows, CPL_TYPE_DOUBLE);
    cpl_array *_variance = cpl_array_new(nwindows, CPL_TYPE_DOUBLE);
    cpl_array *_gain     = cpl_array_new(nwindows, CPL_TYPE_DOUBLE);

    if (aParams->sigma > 0.) {

      cpl_size iwindow;
      for (iwindow = 0; iwindow < nwindows; ++iwindow) {
        int xmin = cpl_table_get_int(aWindowList, "Xmin", iwindow, NULL);
        int ymin = cpl_table_get_int(aWindowList, "Ymin", iwindow, NULL);
        int xmax = cpl_table_get_int(aWindowList, "Xmax", iwindow, NULL);
        int ymax = cpl_table_get_int(aWindowList, "Ymax", iwindow, NULL);

        /* Extract the image region of the window from both flat fields. *
         * Use the union of the bad pixel masks of the subimages for any *
         * further analysis.                                             */
        cpl_image *_flat1 = cpl_image_extract(flat1->data,
                                              xmin, ymin, xmax, ymax);
        cpl_image *_flat2 = cpl_image_extract(flat2->data,
                                              xmin, ymin, xmax, ymax);
        cpl_mask *mask = cpl_image_get_bpm(_flat1);
        cpl_mask_or(mask, cpl_image_get_bpm(_flat2));
        cpl_image_reject_from_mask(_flat2, mask);

        cpl_errorstate _state = cpl_errorstate_get();

        double median1 = cpl_image_get_median(_flat1);
        double median2 = cpl_image_get_median(_flat2);

        if (!cpl_errorstate_is_equal(_state)) {
          cpl_errorstate_set(_state);

          cpl_array_set_invalid(_signal, iwindow);
          cpl_array_set_invalid(_variance, iwindow);
          cpl_array_set_invalid(_gain, iwindow);

          cpl_image_delete(_flat2);
          cpl_image_delete(_flat1);

          continue;
        }

        double clip1 = aParams->sigma * cpl_image_get_stdev(_flat1);
        double clip2 = aParams->sigma * cpl_image_get_stdev(_flat2);

        /* Remove outliers from the flat field pixel values by creating *
         * a bad pixel mask for these pixels. The resulting mask used   *
         * for the following analysis is then the union of all involved *
         * bad pixels masks.                                            */
        cpl_mask *mask1 =
            cpl_mask_threshold_image_create(_flat1,
                                            median1 - clip1,
                                            median1 + clip1);
        cpl_mask *mask2 =
            cpl_mask_threshold_image_create(_flat2,
                                            median2 - clip2,
                                            median2 + clip2);

        /* The created masks need to be inverted to flag pixel values *
         * outside of the defined range as bad, so that they can be   *
         * combined and used as mask for the flat fields.             */
        cpl_mask_not(mask1);
        cpl_mask_not(mask2);
        cpl_mask_or(mask1, mask2);
        cpl_mask_delete(mask2);

        cpl_mask_or(mask, mask1);
        cpl_mask_delete(mask1);

        /* Apply the mask, also to the second member of the flat field  *
         * pair. The subtraction operation propagates the bad pixel map *
         * to the difference image.                                     */
        cpl_image_reject_from_mask(_flat2, mask);
        cpl_image *dflat = cpl_image_subtract_create(_flat1, _flat2);

        if ((median1 < kMuseSaturationLimit) && (median2 < kMuseSaturationLimit)) {

          _state = cpl_errorstate_get();

          double s1 = cpl_image_get_mean(_flat1);
          double s2 = cpl_image_get_mean(_flat2);

          if (!cpl_errorstate_is_equal(_state)) {
            cpl_errorstate_set(_state);
            cpl_array_set_invalid(_signal, iwindow);
            cpl_array_set_invalid(_variance, iwindow);
            cpl_array_set_invalid(_gain, iwindow);
          } else {
            double s = 0.5 * (s1 + s2);
            double v = 1.;
            double g = 0.;

            if ((cpl_array_get_double(aRon, iwindow, NULL) - mron_value) <= 5. * mron_sdev) {
              double sdev = cpl_image_get_stdev(dflat);
              v = 0.5 * sdev * sdev - mron_value * mron_value;
              g = s / v;
            }
            cpl_array_set_double(_signal, iwindow, s);
            cpl_array_set_double(_variance, iwindow, v);
            cpl_array_set_double(_gain, iwindow, g);
          }
        } else {
          cpl_array_set_invalid(_signal, iwindow);
          cpl_array_set_invalid(_variance, iwindow);
          cpl_array_set_invalid(_gain, iwindow);
        }

        cpl_image_delete(dflat);
        cpl_image_delete(_flat2);
        cpl_image_delete(_flat1);
      }

    } else {

      /* Create difference image using the whole image only once, instead    *
       * of doing the subtraction for each window. Pixels having a non-zero  *
       * data quality flag are ignored. As a side effect the subtraction     *
       * creates a bad pixel mask for the result image which is the union    *
       * of the two input mask, and thus can be used as bad pixel mask for   *
       * the per window computations on both flat fields of the current pair */

      cpl_image *dflat = cpl_image_subtract_create(flat1->data, flat2->data);

      /* Use the union of the bad pixel masks for both input flat fields */
      cpl_image_reject_from_mask(flat1->data, cpl_image_get_bpm(dflat));
      cpl_image_reject_from_mask(flat2->data, cpl_image_get_bpm(dflat));

      cpl_size iwindow;
      for (iwindow = 0; iwindow < nwindows; ++iwindow) {
        int xmin = cpl_table_get_int(aWindowList, "Xmin", iwindow, NULL);
        int ymin = cpl_table_get_int(aWindowList, "Ymin", iwindow, NULL);
        int xmax = cpl_table_get_int(aWindowList, "Xmax", iwindow, NULL);
        int ymax = cpl_table_get_int(aWindowList, "Ymax", iwindow, NULL);

        double median1 = cpl_image_get_median_window(flat1->data,
                                                     xmin, ymin, xmax, ymax);
        double median2 = cpl_image_get_median_window(flat2->data,
                                                     xmin, ymin, xmax, ymax);

        if ((median1 < kMuseSaturationLimit) && (median2 < kMuseSaturationLimit)) {
          double s1 = cpl_image_get_mean_window(flat1->data,
                                                xmin, ymin, xmax, ymax);
          double s2 = cpl_image_get_mean_window(flat2->data,
                                                xmin, ymin, xmax, ymax);
          double s = 0.5 * (s1 + s2);
          double v = 1.;
          double g = 0.;

          if ((cpl_array_get_double(aRon, iwindow, NULL) - mron_value) <= 5. * mron_sdev) {
            double sdev = cpl_image_get_stdev_window(dflat,
                                                     xmin, ymin, xmax, ymax);
            v = 0.5 * sdev * sdev - mron_value * mron_value;
            g = s / v;
          }
          cpl_array_set_double(_signal, iwindow, s);
          cpl_array_set_double(_variance, iwindow, v);
          cpl_array_set_double(_gain, iwindow, g);
        } else {
          cpl_array_set_invalid(_signal, iwindow);
          cpl_array_set_invalid(_variance, iwindow);
          cpl_array_set_invalid(_gain, iwindow);
        }
      }

      cpl_image_delete(dflat);

    }

    cpl_table_set_array(gain, "Signal", kpair, _signal);
    cpl_table_set_array(gain, "Variance", kpair, _variance);
    cpl_table_set_array(gain, "Gain", kpair, _gain);
    ++kpair;

    cpl_array_delete(_signal);
    cpl_array_delete(_variance);
    cpl_array_delete(_gain);
  }

  cpl_table_delete(flats);

  /* Remove pairs which have been rejected */
  cpl_table_erase_invalid_rows(gain);
  if (cpl_table_get_nrow(gain) == 0) {
    cpl_table_delete(gain);
    gain = NULL;
  }

  return gain;
}

/*---------------------------------------------------------------------------*/
/**
  @brief    Compute the RON value for a single detector quadrant.
  @param    aRon       An array to store the result of the RON computations
  @param    aRonTable  The list of RON values measured for each measurement
                       window.
  @param    aWindows   The list of window indices to use.
  @param    aParams    The parameter list
  @return   CPL_ERROR_NONE on success and an error code in case of failure.

  The function computes the RON as the weighted mean of the the readout
  noise measurements together with its propagated error. In addition the
  median and the MAD of the individual RON measurements are calculated.
  The results of both computations are stored in the first to array elements
  of the @em aRon argument. The RON computed as the weighted mean and its
  error are stored in the first element of the result array. The median RON
  and the MAD are stored in the second array element the result object. 
 */
/*---------------------------------------------------------------------------*/
static cpl_error_code
muse_lingain_quadrant_get_ron(muse_measurand_t *aRon,
                              const cpl_table *aRonTable,
                              const cpl_array *aWindows,
                              muse_lingain_params_t *aParams)
{
  /* Currently not used */
  CPL_UNUSED(aParams);

  /* Create a continuous array for the measured RON values and its error *
   * estimate.                                                           */
  cpl_size nwindows = cpl_array_get_size(aWindows);

  cpl_array *qron    = cpl_array_new(nwindows, CPL_TYPE_DOUBLE);
  cpl_array *qronerr = cpl_array_new(nwindows, CPL_TYPE_DOUBLE);

  cpl_size iwindow;
  for (iwindow = 0; iwindow < nwindows; ++iwindow) {
    cpl_size jwindow = cpl_array_get_cplsize(aWindows, iwindow, NULL);

    int vnull = 0;
    int enull = 0;
    double v = cpl_table_get_double(aRonTable, "Ron", jwindow, &vnull);
    double e = cpl_table_get_double(aRonTable, "RonErr", jwindow, &enull);

    if ((vnull == 0) && (enull == 0)) {
      cpl_array_set_double(qron, iwindow, v);
      cpl_array_set_double(qronerr, iwindow, e);
    } else {
      cpl_array_set_invalid(qron, iwindow);
      cpl_array_set_invalid(qronerr, iwindow);
    }
  }

  if (cpl_array_count_invalid(qron) == nwindows) {
    cpl_msg_debug(__func__, "Calculating RON failed. No data available!");

    cpl_array_delete(qron);
    cpl_array_delete(qronerr);

    return CPL_ERROR_DATA_NOT_FOUND;
  }

  /* Compute the median and the MAD of the RON measurements */
  double ronmedian = cpl_array_get_median(qron);

  cpl_array *mad = cpl_array_duplicate(qron);
  cpl_array_subtract_scalar(mad, ronmedian);
  cpl_array_abs(mad);

  double ronmad = cpl_array_get_median(mad);

  cpl_array_delete(mad);

  /* Compute the RON as the weighted mean of the individual *
   * RON measurements. The uncertainty of the RON is its    *
   * propagated error.                                      */ 
  cpl_array *wvalues = cpl_array_duplicate(qron);
  cpl_array *weights = cpl_array_duplicate(qronerr);
  cpl_array_power(weights, -2.);
  cpl_array_multiply(wvalues, weights);

  double ron = 0.;
  double ronerr = 0.;
  for (cpl_size i = 0; i < cpl_array_get_size(wvalues); ++i) {
    int invalid[2] = {0, 0};
    double _ron = cpl_array_get(wvalues, i, &invalid[0]);
    double _ronerr = cpl_array_get(weights, i, &invalid[1]);
    if ((invalid[0]) == 0 && (invalid[1] == 0)) {
      ron += _ron;
      ronerr += _ronerr;
    }
  }
  cpl_array_delete(wvalues);
  cpl_array_delete(weights);

  if (ronerr > 0.) {
    ron /= ronerr;
    ronerr = 1. / sqrt(ronerr);
  } else {
    ron = -1.;
    ronerr = -1.;
  }

  aRon[0].value = ron;
  aRon[0].error = ronerr;
  aRon[1].value = ronmedian;
  aRon[1].error = ronmad;

  cpl_array_delete(qron);
  cpl_array_delete(qronerr);

  return CPL_ERROR_NONE;
}

/*---------------------------------------------------------------------------*/
/**
  @brief    Compute the gain value for a single detector quadrant.
  @param    aGain       The result of the gain computation
  @param    aGainTable  The list of gain values measured for each
                        measurement window and each exposure time.
  @param    aWindows    The list of window indices to use.
  @param    aParams     The parameter list
  @return   CPL_ERROR_NONE on success and an error code in case of failure.

  The function computes the gain as the 0th-order coefficient of the
  1st-order polynomial model fitted to the gain measurements. The
  computed gain and the RMS of the fit are stored in the aGain object.
 */
/*---------------------------------------------------------------------------*/
static cpl_error_code
muse_lingain_quadrant_get_gain(muse_gain_fit_t *aGain,
                               const cpl_table *aGainTable,
                               const cpl_array *aWindows,
                               muse_lingain_params_t *aParams)
{
  /* Create flat arrays for the signal and the gain from the gain table *
   * and the windows for a single quadrant. The signal is converted to  *
   * logarithmic scale, and all data elements for which the signal or   *
   * the gain are invalid are marked as such in the destination array.  */
  cpl_size nrows    = cpl_table_get_nrow(aGainTable);
  cpl_size nwindows = cpl_array_get_size(aWindows);
  cpl_size ndata  = nrows * nwindows;

  cpl_array *qsignal = cpl_array_new(ndata, CPL_TYPE_DOUBLE);
  cpl_array *qgain   = cpl_array_new(ndata, CPL_TYPE_DOUBLE);

  cpl_size irow;
  for (irow = 0; irow < nrows; ++irow) {
    cpl_size iwindow;
    cpl_size stride = irow * nwindows;

    const cpl_array *_qsignal = cpl_table_get_array(aGainTable,
                                                    "Signal", irow);
    const cpl_array *_qgain   = cpl_table_get_array(aGainTable,
                                                    "Gain", irow);

    for (iwindow = 0; iwindow < nwindows; ++iwindow) {
      cpl_size jwindow = cpl_array_get_cplsize(aWindows, iwindow, NULL);

      int snull = 0;
      int gnull = 0;
      double s = cpl_array_get_double(_qsignal, jwindow, &snull);
      double g = cpl_array_get_double(_qgain, jwindow, &gnull);

      cpl_size idata = stride + iwindow;
      if (((snull == 0) && (gnull == 0)) && (s > 0.)) {
        cpl_array_set_double(qsignal, idata, log10(s));
        cpl_array_set_double(qgain, idata, g);
      } else {
        cpl_array_set_invalid(qsignal, idata);
        cpl_array_set_invalid(qgain, idata);
      }
    }
  }

  /* Create the bins for the gain and the abscissa for the gain fits. */
  cpl_size nbins = (cpl_size)((aParams->signalmax - aParams->signalmin) /
      aParams->signalbin + 0.5);
  cpl_array *gx = cpl_array_new(nbins, CPL_TYPE_DOUBLE);
  cpl_array *gy = cpl_array_new(nbins, CPL_TYPE_DOUBLE);

  cpl_size ibin;
  for (ibin = 0; ibin < nbins; ++ibin) {
    double bmin = aParams->signalmin + ibin * aParams->signalbin;
    double bmax = bmin + aParams->signalbin;
    double bmid = 0.5 * (bmin + bmax);

    if (bmid < log10(aParams->gainlimit)) {
      cpl_array_set_invalid(gx, ibin);
      cpl_array_set_invalid(gy, ibin);
      continue;
    }

    /* Change signal scale back to linear for the fit later on. */
    cpl_array_set(gx, ibin, pow(10., bmid));

    cpl_array *gdata = cpl_array_duplicate(qgain);

    cpl_size idata;
    for (idata = 0; idata < ndata; ++idata) {
      /* Mark elements outside of the current bin boundaries as *
       * invalid, so that they are ignored in the computations. */
      int invalid = 0;
      double s = cpl_array_get_double(qsignal, idata, &invalid);

      if (invalid || (s < bmin) || (s >= bmax)) {
        cpl_array_set_invalid(gdata, idata);
      }
    }

    /* Skip empty bins */
    if (cpl_array_count_invalid(gdata) == ndata) {
      cpl_array_set_invalid(gx, ibin);
      cpl_array_set_invalid(gy, ibin);
      cpl_array_delete(gdata);
      continue;
    }

    /* Get median and standard deviation of the remaining elements.   *
     * and remove outliers from the gain data before the final median *
     * gain value for this bin is calculated.                         */
    double median = cpl_array_get_median(gdata);
    double stdev  = cpl_array_get_stdev(gdata);

    stdev *= aParams->gainsigma;
    for (idata = 0; idata < ndata; ++idata) {
      /* Mark elements outside of the current bin boundaries as *
       * invalid, so that they are ignored in the computations. */
      int invalid = 0;
      double g = cpl_array_get_double(gdata, idata, &invalid);

      if (invalid || (g <= median - stdev) || (g >= median + stdev)) {
        cpl_array_set_invalid(gdata, idata);
      }
    }

    if (cpl_array_count_invalid(gdata) == ndata) {
      cpl_array_set_invalid(gx, ibin);
      cpl_array_set_invalid(gy, ibin);
    } else {
      /* XXX: For debugging, may be removed later */
      double gmean = cpl_array_get_mean(gdata);

      cpl_array_set_double(gy, ibin, gmean);
    }

    cpl_array_delete(gdata);
  }

  cpl_array_delete(qgain);
  cpl_array_delete(qsignal);


  /* Remove invalid bins prior to fitting a straight line to the gain data */
  muse_cplarray_erase_invalid(gx);
  muse_cplarray_erase_invalid(gy);

  cpl_size nx = cpl_array_get_size(gx);
  cpl_size ny = cpl_array_get_size(gy);

  if ((nx == 0) || (ny == 0)) {

    cpl_msg_debug(__func__, "Fitting the gain relation failed. No data "
                  "available!");
    cpl_array_delete(gy);
    cpl_array_delete(gx);

    return CPL_ERROR_DATA_NOT_FOUND;
  }

  if (nx != ny) {
    cpl_msg_debug(__func__, "Fitting the gain relation failed. Gain "
                  "and signal data sets do not match!");
    cpl_array_delete(gy);
    cpl_array_delete(gx);

    return CPL_ERROR_INCOMPATIBLE_INPUT;
  }

  /* One extra data point is required so that residuals can be computed! */
  if (nx < 3) {
    cpl_msg_debug(__func__, "Fitting the gain relation failed. Insufficient "
                  "data points; cannot fit a first order polynomial!");
    cpl_array_delete(gy);
    cpl_array_delete(gx);

    return CPL_ERROR_DATA_NOT_FOUND;
  }

  /* Fit a straight line to the cleaned, mean gain values. */
  double *_x = cpl_array_get_data_double(gx);
  double *_y = cpl_array_get_data_double(gy);

  cpl_matrix *x = cpl_matrix_wrap(1, nx, _x);
  cpl_vector *y = cpl_vector_wrap(ny, _y);
  cpl_boolean symetric = CPL_FALSE;
  const cpl_size mindeg = 0;
  const cpl_size maxdeg = 1;

  cpl_polynomial *gfit = cpl_polynomial_new(1);
  cpl_error_code ecode = cpl_polynomial_fit(gfit, x, &symetric, y, NULL,
                                            CPL_FALSE, &mindeg, &maxdeg);

  cpl_vector_unwrap(y);
  cpl_matrix_unwrap(x);

  if (ecode != CPL_ERROR_NONE) {
    cpl_polynomial_delete(gfit);
    cpl_array_delete(gy);
    cpl_array_delete(gx);

    cpl_msg_debug(__func__, "Fitting the gain relation failed. Fitting "
                  "first order polynomial to gain data failed!");

    aGain->gain.value = 0.;
    aGain->gain.error = 0.;
    aGain->signal_range[0] = 0.;
    aGain->signal_range[1] = 0.;
    return CPL_ERROR_ILLEGAL_OUTPUT;
  }

  aGain->signal_range[0] = cpl_array_get_min(gx);
  aGain->signal_range[1] = cpl_array_get_max(gx);

  double rms = 0.;
  cpl_size ix;
  for (ix = 0; ix < nx; ++ix) {
    double rx = cpl_array_get_double(gx, ix, NULL);
    double ry = cpl_array_get_double(gy, ix, NULL);
    double ryf = cpl_polynomial_eval_1d(gfit, rx, NULL);
    rms += (ry - ryf) * (ry - ryf);
  }
  rms = sqrt(rms);

  cpl_array_delete(gy);
  cpl_array_delete(gx);

  const cpl_size order = 0;
  aGain->gain.value = cpl_polynomial_get_coeff(gfit, &order);
  aGain->gain.error = rms;

  cpl_polynomial_delete(gfit);

  return CPL_ERROR_NONE;
}

/*---------------------------------------------------------------------------*/
/**
  @brief    Compute the residual non-linearity for a single detector quadrant.
  @param    aGainTable    The list of gain values measured for each
                          measurement window and each exposure time.
  @param    aWindows      The list of window indices to use.
  @param    aGain         Detector gain factor for the quadrant being processed.
  @param    aParams       The parameter list
  @return   CPL_ERROR_NONE on success and an error code in case of failure.

  TBD
 */
/*---------------------------------------------------------------------------*/
static cpl_error_code
muse_lingain_quadrant_get_nonlinearity(muse_linearity_fit_t *aFit,
                                       const cpl_table *aGainTable,
                                       const cpl_array *aWindows,
                                       double aGain,
                                       muse_lingain_params_t *aParams)
{

  /* Build the log10(signal) grid. Using these signal bins to   *
   * determine the set of measurement windows falling into each *
   * bin from the exposure with the longest exposure time. The  *
   * selected windows are then used for the analysis of all     *
   * exposure times                                             */

  cpl_size ntimes = cpl_table_get_nrow(aGainTable);
  cpl_size irow;
  cpl_error_code ecode = cpl_table_get_column_maxpos(aGainTable, "ExpTime",
                                                     &irow);
  if (ecode != CPL_ERROR_NONE) {
    cpl_msg_debug(__func__, "Maximum exposure time cannot be determined from "
                  "linearity data set of IFU %u!", aParams->nifu);
    return ecode;
  }

  if (irow != ntimes - 1) {
    cpl_msg_debug(__func__, "Linearity data set of IFU %u is not sorted in "
                  "ascending order of the exposure time!", aParams->nifu);
    return CPL_ERROR_ILLEGAL_INPUT;
  }

  /* Get the signal array of the longest exposure and use this signal *
   * (in log10(ADU)) to select measurement windows for each signal    *
   * bin. Correct for the gain, since bin boundaries are given in     *
   * log10(counts).                                                   */
  cpl_array *sigref =
      muse_lingain_quadrant_extract_data(aGainTable, aWindows, "Signal", irow);
  cpl_array_logarithm(sigref, 10.);

  cpl_size nsignal = cpl_array_get_size(sigref);
  cpl_size nlevels = (cpl_size)((aParams->ctsmax - aParams->ctsmin) /
      aParams->ctsbin + 0.5);

  cpl_array **counts = cpl_calloc(nlevels, sizeof *counts);
  const double qgain = log10(aGain);
  cpl_size klevel = 0;
  cpl_size ilevel;
  for (ilevel = 0; ilevel < nlevels; ++ilevel) {
    double shigh  = aParams->ctsmax - ilevel * aParams->ctsbin;
    double slow   = shigh - aParams->ctsbin;

    shigh -= qgain;
    slow  -= qgain;

    /* For each bin create an array of window indices which satisfy *
     * the selection criterion.                                     */
    cpl_size *sindex = cpl_malloc(nsignal * sizeof *sindex);
    cpl_size kdata = 0;
    cpl_size idata;
    for (idata = 0; idata < nsignal; ++idata) {
      int invalid = 0;
      double s = cpl_array_get_double(sigref, idata, &invalid);

      if (!invalid && ((s >= slow) && (s < shigh))) {
        sindex[kdata] = cpl_array_get_cplsize(aWindows, idata, NULL);
        ++kdata;
      }
    }

    if (kdata == 0) {
      cpl_free(sindex);
      cpl_msg_debug(__func__, "No valid measurement found within signal "
                    "range [%.6f, %.6f[ [log10(counts)] of the linearity "
                    "data set of IFU %u!", slow + qgain, shigh + qgain,
                    aParams->nifu);
      continue;
    } else {

      /* Compute the mean signal [ADU] from the selected windows for *
       * each exposure time individually. If no valid signal data is *
       * present for an exposure time the respective mean signal     *
       * element is marked as invalid to be able to exclude it       *
       * during the following analysis.                              */
      cpl_array *smean = cpl_array_new(ntimes, CPL_TYPE_DOUBLE);
      cpl_array *windex = cpl_array_wrap_cplsize(sindex, kdata);
      cpl_size itime;
      for (itime = 0; itime < ntimes; ++itime) {
        cpl_array *qsignal =
            muse_lingain_quadrant_extract_data(aGainTable, windex,
                                               "Signal", itime);
        if (qsignal && (cpl_array_count_invalid(qsignal) < kdata)) {
          double mean = cpl_array_get_mean(qsignal);
          cpl_array_set_double(smean, itime, mean);
        } else {
          cpl_array_set_invalid(smean, itime);
        }
        cpl_array_delete(qsignal);
      }
      cpl_array_unwrap(windex);

      /* Note that here counts is still in ADU, despite counts is usually *
       * used after the signal is converted to electrons!                 */
      counts[klevel] = smean;
      ++klevel;
    }
    cpl_free(sindex);
  }

  cpl_array_delete(sigref);

  /* Get the array of the exposure times and add time offset */
  cpl_array *exptime = cpl_array_new(ntimes, CPL_TYPE_DOUBLE);

  cpl_size itime;
  for (itime = 0; itime < ntimes; ++itime) {
    double _exptime = cpl_table_get_double(aGainTable, "ExpTime",
                                           itime, NULL);
    cpl_array_set_double(exptime, itime, _exptime);
  }
  cpl_array_add_scalar(exptime, aParams->toffset);

  /* Compute count rate from the determined mean signals */
  cpl_array **crate = cpl_calloc(klevel, sizeof *crate);
  for (ilevel = 0; ilevel < klevel; ++ilevel) {
    crate[ilevel] = cpl_array_duplicate(counts[ilevel]);
    cpl_array_divide(crate[ilevel], exptime);
  }

  /* Select count rates located within the limits of the desired linear  *
   * range, and select the corresponding exposure times. The count rates *
   * should be constant for all exposure times. Thus, a constant is      *
   * fitted to the relation exposure time vs. count rate, and the        *
   * residuals are computed.                                             */

  /* Convert linear range limit to ADU */
  double linmin = pow(10., aParams->linearmin - qgain);
  double linmax = pow(10., aParams->linearmax - qgain);

  cpl_array **cresidual = cpl_malloc(klevel * sizeof *cresidual);

  for (ilevel = 0; ilevel < klevel; ++ilevel) {
    cpl_array *_exptime = cpl_array_duplicate(exptime);
    cpl_array *_crate   = cpl_array_duplicate(crate[ilevel]);

    for (itime = 0; itime < ntimes; ++itime) {
      int invalid = 0;
      double cnts = cpl_array_get_double(counts[ilevel], itime, &invalid);

      if (invalid || ((cnts < linmin) || (cnts >= linmax))) {
          cpl_array_set_invalid(_exptime, itime);
          cpl_array_set_invalid(_crate, itime);
      }
    }

    if (cpl_array_get_size(_crate) == cpl_array_count_invalid(_crate)) {

      /* If all data at this count level is invalid mark the residuals *
       * as unavailable.                                               */
      cresidual[ilevel] = NULL;

      cpl_array_delete(_crate);
      cpl_array_delete(_exptime);

    } else {

      /* XXX: Fitting a zero order polynomial with constant weights in    *
       *      a least square sense is equivalent to calculating the mean! *
       *      Any reason for using a true fit?                            */
#ifdef USE_POLYFIT

      /* Remove invalid data prior to fitting a constant to the count rates */
      muse_cplarray_erase_invalid(_exptime);
      muse_cplarray_erase_invalid(_crate);

      cpl_size nt = cpl_array_get_size(_exptime);
      cpl_size nc = cpl_array_get_size(_crate);

      /* By design the 2 arrays should have the same number of elements. *
       * Thus the following condition should never be true!              */
      if (nt != nc) {
        cpl_msg_debug(__func__, "Fitting a constant to the count rate data "
                      "failed. Exposure time and count rate data sets do not "
                      "match!");
        cpl_array_delete(_crate);
        cpl_array_delete(_exptime);

        muse_vfree((void **)cresidual, klevel, (muse_free_func)cpl_array_delete);
        muse_vfree((void **)crate, klevel, (muse_free_func)cpl_array_delete);
        muse_vfree((void **)counts, nlevels, (muse_free_func)cpl_array_delete);

        cpl_array_delete(exptime);

        return CPL_ERROR_INCOMPATIBLE_INPUT;
      }

      /* Fit a constant to the cleaned, count rate values. */
      double *_et = cpl_array_get_data_double(_exptime);
      double *_cr = cpl_array_get_data_double(_crate);

      cpl_matrix *et = cpl_matrix_wrap(1, nt, _et);
      cpl_vector *cr = cpl_vector_wrap(nc, _cr);
      cpl_boolean symetric = CPL_FALSE;
      const cpl_size mindeg = 0;
      const cpl_size maxdeg = 0;

      cpl_polynomial *crfit = cpl_polynomial_new(1);
      cpl_error_code ecode = cpl_polynomial_fit(crfit, et, &symetric, cr, NULL,
                                                CPL_FALSE, &mindeg, &maxdeg);

      cpl_vector_unwrap(et);
      cpl_matrix_unwrap(cr);

      cpl_array_delete(_crate);
      cpl_array_delete(_exptime);

      if (ecode != CPL_ERROR_NONE) {
        cpl_polynomial_delete(crfit);
        cpl_msg_debug(__func__, "Fitting a constant to the count rate level "
                      "failed. Fitting a zero order polynomial to count rate "
                      "data failed!");

        muse_vfree((void **)cresidual, klevel, (muse_free_func)cpl_array_delete);
        muse_vfree((void **)crate, klevel, (muse_free_func)cpl_array_delete);
        muse_vfree((void **)counts, nlevels, (muse_free_func)cpl_array_delete);

        cpl_array_delete(exptime);

        return CPL_ERROR_ILLEGAL_OUTPUT;
      }

      const cpl_size order = 0;
      double crlevel = cpl_polynomial_get_coeff(crfit, &order);
      cpl_polynomial_delete(crfit);
#else
      double crlevel = cpl_array_get_mean(_crate);

      cpl_array_delete(_crate);
      cpl_array_delete(_exptime);
#endif
      /* Calculate the count rate residuals */
      cresidual[ilevel] = cpl_array_duplicate(crate[ilevel]);
      cpl_array_subtract_scalar(cresidual[ilevel], crlevel);
      cpl_array_divide(cresidual[ilevel], crate[ilevel]);
    }
  }

  cpl_array_delete(exptime);


  /* Fit the final non-linearity relation, i.e. signal in ADU vs. residual *
   * non-linearity.                                                        */

  /* Create signal bins for which the mean residual will be computed. *
   * Reuse the bin settings used for the gain computation.            */
  double signalmax = aParams->signalmax;
  double signalmin = aParams->signalmin;
  double signalbin = aParams->signalbin;

  cpl_size nbins = (cpl_size)((signalmax - signalmin) / signalbin + 0.5);
  cpl_array *lx  = cpl_array_new(nbins, CPL_TYPE_DOUBLE);
  cpl_array *ly  = cpl_array_new(nbins, CPL_TYPE_DOUBLE);
  cpl_array *lyse = cpl_array_new(nbins, CPL_TYPE_DOUBLE);

  cpl_size ibin;
  for (ibin = 0; ibin < nbins; ++ibin) {
    double bmin = signalmin + ibin * signalbin;
    double bmax = bmin + signalbin;
    double bmid = 0.5 * (bmin + bmax);

    cpl_array_set_double(lx, ibin, bmid);

    bmin = pow(10., bmin);
    bmax = pow(10., bmax);

    double *rdata = cpl_malloc((klevel * ntimes) * sizeof *rdata);
    cpl_size kresidual = 0;
    for (ilevel = 0; ilevel < klevel; ++ilevel) {
      cpl_array *residual = cresidual[ilevel];

      if (residual != NULL) {
        for (itime = 0; itime < ntimes; ++itime) {
          int cinvalid = 0;
          int rinvalid = 0;
          double _counts   = cpl_array_get_double(counts[ilevel], itime, &cinvalid);
          double _residual = cpl_array_get_double(residual, itime, &rinvalid);

          if (!(cinvalid || rinvalid) && ((_counts >= bmin) && (_counts < bmax))) {
            rdata[kresidual] = _residual;
            ++kresidual;
          }
        }
      }
    }
    if (kresidual == 0) {
      cpl_array_set_invalid(lx, ibin);
      cpl_array_set_invalid(ly, ibin);
      cpl_array_set_invalid(lyse, ibin);
    } else {
      cpl_array *r = cpl_array_wrap_double(rdata, kresidual);

      double rmean = cpl_array_get_mean(r);
      double rsdev = cpl_array_get_stdev(r);
      cpl_array_set_double(ly, ibin, rmean);
      cpl_array_set_double(lyse, ibin, rsdev);

      cpl_array_unwrap(r);
    }
    cpl_free(rdata);
  }

  muse_vfree((void **)cresidual, klevel, (muse_free_func)cpl_array_delete);
  muse_vfree((void **)crate, klevel, (muse_free_func)cpl_array_delete);
  muse_vfree((void **)counts, nlevels, (muse_free_func)cpl_array_delete);

  /* Fit a polynomial to the final signal vs. mean residual non-linearity */
  muse_cplarray_erase_invalid(lx);
  muse_cplarray_erase_invalid(ly);
  muse_cplarray_erase_invalid(lyse);

  cpl_size nx = cpl_array_get_size(lx);
  cpl_size ny = cpl_array_get_size(ly);

  if ((nx == 0) || (ny == 0)) {

    cpl_msg_debug(__func__, "Fitting the non-linearity relation failed. "
                  "No data available!");
    cpl_array_delete(lyse);
    cpl_array_delete(ly);
    cpl_array_delete(lx);

    return CPL_ERROR_DATA_NOT_FOUND;
  }

  if (nx != ny) {
    cpl_msg_debug(__func__, "Fitting the non-linearity relation failed. "
                  "Linearity residual and signal data sets do not match!");
    cpl_array_delete(lyse);
    cpl_array_delete(ly);
    cpl_array_delete(lx);

    return CPL_ERROR_INCOMPATIBLE_INPUT;
  }

  /* One extra data point is required so that the residuals can be computed! */
  if (nx < aParams->order + 2) {
    cpl_msg_debug(__func__, "Fitting the non-linearity relation failed. "
                  "Insufficient data points; cannot fit a %d order "
                  "polynomial!", aParams->order);
    cpl_array_delete(lyse);
    cpl_array_delete(ly);
    cpl_array_delete(lx);

    return CPL_ERROR_DATA_NOT_FOUND;
  }

  /* Fit a polynomial to the cleaned, mean residual non-linearity values. */
  double *_x  = cpl_array_get_data_double(lx);
  double *_y  = cpl_array_get_data_double(ly);
  double *_yse = cpl_array_get_data_double(lyse);

  cpl_matrix *x  = cpl_matrix_wrap(1, nx, _x);
  cpl_vector *y  = cpl_vector_wrap(ny, _y);
  cpl_vector *yse = cpl_vector_wrap(ny, _yse);
  cpl_boolean symetric = CPL_FALSE;
  const cpl_size mindeg = 0;
  const cpl_size maxdeg = aParams->order;

  cpl_polynomial *lfit = cpl_polynomial_new(1);
  ecode = cpl_polynomial_fit(lfit, x, &symetric, y, NULL, CPL_FALSE,
                             &mindeg, &maxdeg);

  cpl_vector_unwrap(yse);
  cpl_vector_unwrap(y);
  cpl_matrix_unwrap(x);

  if (ecode != CPL_ERROR_NONE) {
    cpl_polynomial_delete(lfit);
    cpl_array_delete(lyse);
    cpl_array_delete(ly);
    cpl_array_delete(lx);

    cpl_msg_debug(__func__, "Fitting the non-linearity relation failed. "
                  "Fitting %d order polynomial to residual non-linearity "
                  "data failed!", aParams->order);
    aFit->coefficients = NULL;
    aFit->range[0] = 0;
    aFit->range[1] = 0;

    return CPL_ERROR_ILLEGAL_OUTPUT;
  }

  /* Save the results of the polynomial fit: coefficients and range */
  cpl_size ncoeff = aParams->order + 1;
  cpl_array *coefficients = cpl_array_new(ncoeff, CPL_TYPE_DOUBLE);

  cpl_size icoeff;
  for (icoeff = 0; icoeff < ncoeff; ++icoeff) {
    const cpl_size order = icoeff;
    double coeff = cpl_polynomial_get_coeff(lfit, &order);
    cpl_array_set_double(coefficients, icoeff, coeff);
  }

  double rms = 0.;
  cpl_size ix;
  for (ix = 0; ix < nx; ++ix) {
    double rx = cpl_array_get_double(lx, ix, NULL);
    double ry = cpl_array_get_double(ly, ix, NULL);
    double ryf = cpl_polynomial_eval_1d(lfit, rx, NULL);
    rms += (ry - ryf) * (ry - ryf);
  }
  rms = sqrt(rms);
  cpl_polynomial_delete(lfit);

  aFit->coefficients = coefficients;
  aFit->range[0] = cpl_array_get_min(lx);
  aFit->range[1] = cpl_array_get_max(lx);
  aFit->rms      = rms;

  cpl_array_delete(lyse);
  cpl_array_delete(ly);
  cpl_array_delete(lx);

  return CPL_ERROR_NONE;
}

/*---------------------------------------------------------------------------*/
/**
  @brief    Compute a readout noise value for each detector quadrant.
  @param    aRonTable    The list of RON values measured for each
                         measurement window.
  @param    aWindowList  The list of measurement windows.
  @param    aParams      The parameter list
  @return   An table with the computed RON value for each quadrant.

  The function computes a RON value for each detector quadrant of a single
  IFU, using the measurement windows which are located within the quadrant
  boundaries. The RON is computed as the median of the individual readout
  noise measurements, and its error is estimated as its RMS.

  Note that the function still returns the result table of RON values even
  if individual RON values are invalid. However in this case the respective
  RON value is marked as an invalid element. This applies also to the case
  where all RON values are invalid for one or the other reason. It is up to
  the calling function to decided how such results are handled properly in
  a given context.
 */
/*---------------------------------------------------------------------------*/
static cpl_table *
muse_lingain_compute_ron(const cpl_table *aRonTable, cpl_table *aWindowList,
                         muse_lingain_params_t *aParams)
{
  const unsigned char nquadrants = 4;
  unsigned char iquadrant;

  cpl_table *ron = cpl_table_new(nquadrants);

  if (!ron) {
    return NULL;
  }

  cpl_table_new_column(ron, "Ron", CPL_TYPE_DOUBLE);
  cpl_table_new_column(ron, "RonErr", CPL_TYPE_DOUBLE);
  cpl_table_new_column(ron, "RonMedian", CPL_TYPE_DOUBLE);
  cpl_table_new_column(ron, "RonMad", CPL_TYPE_DOUBLE);

  for (iquadrant = 0; iquadrant < nquadrants; ++iquadrant) {
    unsigned int quadrant = iquadrant + 1;

    /* Get window indices for the current detector quadrant */
    cpl_array *windex = muse_lingain_quadrant_get_windows(aWindowList, quadrant);

    if (!windex) {
        cpl_msg_warning(__func__, "No measurement windows defined for "
                        "quadrant %u of IFU %d!", quadrant, aParams->nifu);
        cpl_table_set_invalid(ron, "Ron", iquadrant);
        cpl_table_set_invalid(ron, "RonErr", iquadrant);
        cpl_table_set_invalid(ron, "RonMedian", iquadrant);
        cpl_table_set_invalid(ron, "RonMad", iquadrant);
        continue;
    }

    /* Compute the RON for the current detector quadrant, from the window *
     * indices and the RON measurements table.                            */

    muse_measurand_t qron[2];
    cpl_error_code ecode = muse_lingain_quadrant_get_ron(qron, aRonTable,
                                                         windex, aParams);
    if ((ecode != CPL_ERROR_NONE) || (qron[0].value <= 0.) ||
        (qron[0].error <= 0.)) {
      cpl_msg_warning(__func__, "Detector readout noise value for quadrant "
                      "%u of IFU %d is invalid!", quadrant, aParams->nifu);
      cpl_table_set_invalid(ron, "Ron", iquadrant);
      cpl_table_set_invalid(ron, "RonErr", iquadrant);
      cpl_table_set_invalid(ron, "RonMedian", iquadrant);
      cpl_table_set_invalid(ron, "RonMad", iquadrant);
    } else {
      cpl_msg_info(__func__, "Detector readout noise value for quadrant %u "
                   "of IFU %d: %.6f +/- %.6f [ADU]", quadrant, aParams->nifu,
                   qron[0].value, qron[0].error);
    }

    cpl_table_set_double(ron, "Ron", iquadrant, qron[0].value);
    cpl_table_set_double(ron, "RonErr", iquadrant, qron[0].error);
    cpl_table_set_double(ron, "RonMedian", iquadrant, qron[1].value);
    cpl_table_set_double(ron, "RonMad", iquadrant, qron[1].error);
    cpl_array_delete(windex);
  }

  return ron;
}

/*---------------------------------------------------------------------------*/
/**
  @brief    Compute a gain value for each detector quadrant.
  @param    aGainTable    The list of gain values measured for each
                          measurement window and each exposure time.
  @param    aWindowList   The list of measurement windows.
  @param    aParams       The parameter list
  @return   An table with the computed gain value for each quadrant.

  The function computes a gain value for each detector quadrant of a single
  IFU, using the measurement windows which are located within the quadrant
  boundaries. The gain data from these windows falling within individual
  signal bins is averaged and a linear fit to the gain versus signal relation
  is performed, with the intercept being the gain value for the quadrant.

  Note that the function still returns the result table of gain values even
  if individual gain values are invalid because the linear fit failed.
  However in this case the respective gain value is marked as an invalid
  element. This applies also to the case that all gain values are
  invalid for one or the other reason. It is up to the calling function
  to decided how such results are handled properly in a given context.
 */
/*---------------------------------------------------------------------------*/
static cpl_table *
muse_lingain_compute_gain(const cpl_table *aGainTable, cpl_table *aWindowList,
                          muse_lingain_params_t *aParams)
{
  const unsigned char nquadrants = 4;
  unsigned char iquadrant;

  cpl_table *gain = cpl_table_new(nquadrants);

  if (!gain) {
    return NULL;
  }

  cpl_table_new_column(gain, "Gain", CPL_TYPE_DOUBLE);
  cpl_table_new_column(gain, "FitRms", CPL_TYPE_DOUBLE);
  cpl_table_new_column(gain, "SignalMin", CPL_TYPE_DOUBLE);
  cpl_table_new_column(gain, "SignalMax", CPL_TYPE_DOUBLE);

  for (iquadrant = 0; iquadrant < nquadrants; ++iquadrant) {
    unsigned int quadrant = iquadrant + 1;

    /* Get window indices for the current detector quadrant */
    cpl_array *windex = muse_lingain_quadrant_get_windows(aWindowList, quadrant);

    if (!windex) {
        cpl_msg_warning(__func__, "No measurement windows defined for "
                        "quadrant %u of IFU %d!", quadrant, aParams->nifu);
        cpl_table_set_invalid(gain, "Gain", iquadrant);
        cpl_table_set_invalid(gain, "FitRms", iquadrant);
        cpl_table_set_invalid(gain, "SignalMin", iquadrant);
        cpl_table_set_invalid(gain, "SignalMax", iquadrant);
        continue;
    }

    /* Compute the gain for the current detector quadrant, from the window *
     * indices and the gain measurements table.                            */

    muse_gain_fit_t qgain = {{0., 0.}, {0., 0.}};
    cpl_error_code ecode = muse_lingain_quadrant_get_gain(&qgain, aGainTable,
                                                          windex, aParams);
    if (ecode != CPL_ERROR_NONE) {
      cpl_msg_warning(__func__, "Detector gain value for quadrant %u of "
                      "IFU %d is invalid!", quadrant, aParams->nifu);
      cpl_table_set_invalid(gain, "Gain", iquadrant);
      cpl_table_set_invalid(gain, "FitRms", iquadrant);
      cpl_table_set_invalid(gain, "SignalMin", iquadrant);
      cpl_table_set_invalid(gain, "SignalMax", iquadrant);
    } else {
      cpl_msg_info(__func__, "Detector gain value for quadrant %u of "
                   "IFU %d: %.6f [counts/ADU]", quadrant, aParams->nifu,
                   qgain.gain.value);
    }

    cpl_table_set_double(gain, "Gain", iquadrant, qgain.gain.value);
    cpl_table_set_double(gain, "FitRms", iquadrant, qgain.gain.error);
    cpl_table_set_double(gain, "SignalMin", iquadrant, qgain.signal_range[0]);
    cpl_table_set_double(gain, "SignalMax", iquadrant, qgain.signal_range[1]);
    cpl_array_delete(windex);
  }

  return gain;
}

/*---------------------------------------------------------------------------*/
/**
  @brief    Compute the residual non-linearity for each detector quadrant.
  @param    aGainTable    The list of gain values measured for each
                          measurement window and each exposure time.
  @param    aWindowList   The list of measurement windows.
  @param    aGain         Table of detector gain values for each quadrant.
  @param    aParams       The parameter list
  @return   A table with the fitted coefficients of the polynomial.

  The function determines the relation of residual non-linearity versus
  signal for each detector quadrant using a polynomial fit. The conversion
  from ADU to counts (electrons) is done  using the given gain value which
  must be available before this function is called.
 */
/*---------------------------------------------------------------------------*/
static cpl_table *
muse_lingain_compute_nonlinearity(const cpl_table *aGainTable,
                                  cpl_table *aWindowList,
                                  const cpl_table *aGain,
                                  muse_lingain_params_t *aParams)
{
  const unsigned char nquadrants = 4;
  unsigned char iquadrant;

  cpl_table *linearity = cpl_table_new(nquadrants);
  cpl_table_new_column(linearity, "Gain", CPL_TYPE_DOUBLE);
  cpl_table_new_column(linearity, "SignalMin", CPL_TYPE_DOUBLE);
  cpl_table_new_column(linearity, "SignalMax", CPL_TYPE_DOUBLE);
  cpl_table_new_column_array(linearity, "Coefficients", CPL_TYPE_DOUBLE,
                             aParams->order + 1);
  cpl_table_new_column(linearity, "FitRms", CPL_TYPE_DOUBLE);

  cpl_table_set_column_unit(linearity, "Gain", "counts/ADU)");
  cpl_table_set_column_unit(linearity, "SignalMin", "log10(ADU)");
  cpl_table_set_column_unit(linearity, "SignalMax", "log10(ADU)");

  for (iquadrant = 0; iquadrant < nquadrants; ++iquadrant) {
    unsigned int quadrant = iquadrant + 1;

    /* Get window indices for the current detector quadrant */
    cpl_array *windex = muse_lingain_quadrant_get_windows(aWindowList, quadrant);

    if (!windex) {
        cpl_msg_warning(__func__, "No measurement windows defined for "
                        "quadrant %u of IFU %d!", quadrant, aParams->nifu);
        continue;
    }

    /* Determine the detector non-linearity for the current detector *
     * quadrant, from the window and the gain measurements table.    */

    int invalid = 0;
    double qgain = cpl_table_get_double(aGain, "Gain", iquadrant, &invalid);

    if (invalid) {
      cpl_msg_warning(__func__, "Got invalid gain for quadrant %u of IFU "
                      "%d! Skipping non-linearity computation!", quadrant,
                      aParams->nifu);
      cpl_array_delete(windex);
      continue;
    }

    muse_linearity_fit_t qlinearity = {NULL, {0., 0.}, 0.};
    cpl_error_code ecode =
        muse_lingain_quadrant_get_nonlinearity(&qlinearity, aGainTable,
                                               windex, qgain, aParams);

    if (ecode != CPL_ERROR_NONE) {
      cpl_msg_warning(__func__, "Computation of the detector non-linearity "
                      "for quadrant %u of IFU %d failed!", quadrant,
                      aParams->nifu);
      cpl_table_set_invalid(linearity, "Gain", iquadrant);
      cpl_table_set_invalid(linearity, "SignalMin", iquadrant);
      cpl_table_set_invalid(linearity, "SignalMax", iquadrant);
      cpl_table_set_invalid(linearity, "Coefficients", iquadrant);
      cpl_table_set_invalid(linearity, "FitRms", iquadrant);
    } else {
      cpl_msg_info(__func__, "RMS of residual non-linearity model over "
                   "signal range [%.4f, %.4f] [log10(ADU)] for quadrant "
                   "%u of IFU %d: %.6e", qlinearity.range[0],
                   qlinearity.range[1], quadrant, aParams->nifu,
                   qlinearity.rms);

      cpl_table_set_double(linearity, "Gain", iquadrant, qgain);
      cpl_table_set_double(linearity, "SignalMin", iquadrant,
                           qlinearity.range[0]);
      cpl_table_set_double(linearity, "SignalMax", iquadrant,
                           qlinearity.range[1]);
      cpl_table_set_array(linearity, "Coefficients", iquadrant,
                          qlinearity.coefficients);
      cpl_table_set_double(linearity, "FitRms", iquadrant,
                           qlinearity.rms);
    }
    muse_linearity_fit_clear(&qlinearity);
    cpl_array_delete(windex);
  }

  return linearity;
}

/*----------------------------------------------------------------------------*/
/**
  @brief
  @param    aProcessing  the processing structure
  @param    aParams      the parameters list
  @return   0 if everything is ok, -1 something went wrong

 */
/*----------------------------------------------------------------------------*/
int
muse_lingain_compute(muse_processing *aProcessing,
                     muse_lingain_params_t *aParams)
{
  /* Validate provided recipe parameters */
  if (!muse_lingain_validate_parameters(aParams)) {
    return -1;
  }

  /* Load trace table */
  cpl_table *trace = muse_processing_load_ctable(aProcessing, MUSE_TAG_TRACE_TABLE,
                                                 aParams->nifu);

  if (!trace) {
    cpl_msg_error(__func__, "%s could not be loaded!", MUSE_TAG_TRACE_TABLE);
    return -1;
  }

  /* Load and pre-process the raw images. */

  /* XXX: This includes the bias subtraction, which will be applied to the   *
   * two linearity bias frames too. In principle this is not an issue, since *
   * only their standard deviation is used as RON, but the bias subtraction  *
   * adds noise. Thus, check whether the biases need a special treatment in  *
   * muse_lingain_load_images()!                                             */

  muse_imagelist *images = muse_lingain_load_images(aProcessing, aParams->nifu);
  if (!images) {
    cpl_msg_error(__func__, "Loading and basic processing of the raw input "
                  "images failed for IFU %u!", aParams->nifu);
    cpl_table_delete(trace);
    return -1;
  }

  /* Create the sorted index table for the exposures of the linearity *
   * sequence.                                                        */
  cpl_table *exposures = muse_lingain_sort_images(images);
  if (!exposures) {
    cpl_msg_error(__func__, "Creating a sorted list of exposures failed "
                  "for IFU %u!", aParams->nifu);
    muse_imagelist_delete(images);
    cpl_table_delete(trace);
    return -1;
  }

  /* Create list of measurement windows to be used for sampling the flat *
   * field data                                                          */
  cpl_table *windowlist = muse_lingain_create_windowlist(trace, exposures,
                                                         images, aParams);
  if (!windowlist) {
    cpl_msg_error(__func__, "Creating the list of measurement windows failed "
                  "for IFU %u!", aParams->nifu);
    cpl_table_delete(exposures);
    muse_imagelist_delete(images);
    cpl_table_delete(trace);
    return -1;
  }
  cpl_table_delete(trace);

  /* Measure per window read-out noise on the first 2 bias frames */
  cpl_table *rondata = muse_lingain_window_get_ron(windowlist, exposures, images,
                                                   aParams);
  if (!rondata) {
    cpl_msg_error(__func__, "Calculating the RON for individual measurement "
                  "windows failed for IFU %u!", aParams->nifu);
    cpl_table_delete(windowlist);
    cpl_table_delete(exposures);
    muse_imagelist_delete(images);
    return -1;
  }

  /* Compute the per quadrant readout noise */
  cpl_table *ron = muse_lingain_compute_ron(rondata, windowlist, aParams);
  if (!ron) {
    cpl_msg_error(__func__, "Calculating the detector readout noise failed "
                  "for IFU %u!", aParams->nifu);
    cpl_table_delete(rondata);
    cpl_table_delete(windowlist);
    cpl_table_delete(exposures);
    muse_imagelist_delete(images);
    return -1;
  }

  /* Measure gain value for each measurement window and each flat field *
   * pair of the exposure series.                                       */
  cpl_array *wron = muse_cpltable_extract_column(rondata, "Ron");

  cpl_table *gaindata = muse_lingain_window_get_gain(windowlist, wron,
                                                     exposures, images,
                                                     aParams);
  if (!gaindata) {
    cpl_msg_error(__func__, "Calculating the gain for individual measurement "
                  "windows failed for IFU %u!", aParams->nifu);
    cpl_array_unwrap(wron);
    cpl_table_delete(ron);
    cpl_table_delete(rondata);
    cpl_table_delete(windowlist);
    cpl_table_delete(exposures);
    muse_imagelist_delete(images);
    return -1;
  }
  cpl_array_unwrap(wron);
  cpl_table_delete(rondata);
  cpl_table_delete(exposures);

  /* Compute a gain value for each detector quadrant. */
  cpl_table *gain = muse_lingain_compute_gain(gaindata, windowlist, aParams);

  /* XXX: Check error handling here! Right now terminate the recipe if    *
   *      any gain value is invalid, but this may be relaxed with respect *
   *      to operational needs.                                           */

  if (cpl_table_count_invalid(gain, "Gain") != 0) {
    cpl_msg_error(__func__, "Calculating the detector gain failed for IFU %u!",
                  aParams->nifu);
    cpl_table_delete(gain);
    cpl_table_delete(gaindata);
    cpl_table_delete(ron);
    cpl_table_delete(windowlist);
    muse_imagelist_delete(images);
    return -1;
  }

#ifdef NONLINEARITY_MODEL
  /* Fit the residual detector non-linearity for each quadrant */
  cpl_table *linearity = muse_lingain_compute_nonlinearity(gaindata, windowlist,
                                                           gain, aParams);
  if (cpl_table_has_invalid(linearity, "Coefficients")) {
    cpl_msg_error(__func__, "Computing the detector residual non-linearity "
                  "failed for IFU %u!", aParams->nifu);
    cpl_table_delete(linearity);
    cpl_table_delete(gain);
    cpl_table_delete(gaindata);
    cpl_table_delete(ron);
    cpl_table_delete(windowlist);
    muse_imagelist_delete(images);
    return -1;
  }
#endif

  cpl_table_delete(gaindata);
  cpl_table_delete(windowlist);

  /* Create the linearity table product */
  cpl_propertylist *header =
      cpl_propertylist_duplicate(muse_imagelist_get(images, 0)->header);

  muse_imagelist_delete(images);

  cpl_propertylist_erase_regexp(header,
                                "^SIMPLE$|^BITPIX$|^NAXIS|^EXTEND$|^XTENSION$|"
                                "^DATASUM$|^DATAMIN$|^DATAMAX$|^DATAMD5$|"
                                "^PCOUNT$|^GCOUNT$|^HDUVERS$|^BLANK$|"
                                "^BZERO$|^BSCALE$|^BUNIT$|^CHECKSUM$|^INHERIT$|"
                                "^PIPEFILE$|^ESO PRO |" MUSE_WCS_KEYS "|"
                                MUSE_HDR_OVSC_REGEXP, 0);

  const unsigned char nquadrant = 4;
  unsigned char iquadrant;
  for (iquadrant = 0; iquadrant < nquadrant; ++iquadrant) {
    char keyword[KEYWORD_LENGTH];
    unsigned char _iquadrant = iquadrant + 1;
    int invalid[2] = {0, 0};
    double _ron = cpl_table_get_double(ron, "Ron", iquadrant, &invalid[0]);
    double _ronerr = cpl_table_get_double(ron, "RonErr", iquadrant, NULL);
    double _ronmed = cpl_table_get_double(ron, "RonMedian", iquadrant, NULL);
    double _ronmad = cpl_table_get_double(ron, "RonMad", iquadrant, NULL);
    double _gain = cpl_table_get_double(gain, "Gain", iquadrant, &invalid[1]);
    double _gainerr = cpl_table_get_double(gain, "FitRms", iquadrant, NULL);
    double _conad = (_gain > 0.) ? 1. /_gain : 0.;

    if (invalid[0]) {
      _ron = 0;
      _ronerr = 0.;
      _ronmed = 0.;
      _ronmad = 0.;
    }

    if (invalid[1]) {
      _gain = 0.;
      _gainerr = 0.;
    }

    /* Convert RON measurements to counts (electrons), do error *
     * propagation for the RON.                                 */
    double a = _gain * _ronerr;
    double b = _ron * _gainerr;
    _ronerr = sqrt(a * a + b * b);
    _ron *= _gain;
    _ronmed *= _gain;
    _ronmad *= _gain;


    /* XXX: Note that the unit string in the comment of the header *
     *      keywords ESO DET OUTi GAIN and ESO DET OUTi CONAD are  *
     *      wrong. The actual unit of the GAIN keyword value is    *
     *      [e-/ADU] and the unit of the CONAD keyword value is    *
     *      [ADU/e-]. This is a known issue and would need to be   *
     *      fixed at the ICS level (raw frame header), not here!   */

    snprintf(keyword, KEYWORD_LENGTH, "ESO DET OUT%d GAIN", _iquadrant);
    cpl_propertylist_update_double(header, keyword, _gain);
    //cpl_propertylist_set_comment(header, keyword,
    //                             "[e-/ADU] Conversion ADUs to electrons");

    snprintf(keyword, KEYWORD_LENGTH, "ESO DET OUT%d CONAD", _iquadrant);
    cpl_propertylist_update_double(header, keyword, _conad);
    //cpl_propertylist_set_comment(header, keyword,
    //                             "[ADU/e-] Conversion electrons to ADUs");

    snprintf(keyword, KEYWORD_LENGTH, QC_LINGAIN_RONi, _iquadrant);
    cpl_propertylist_update_double(header, keyword, _ron);

    snprintf(keyword, KEYWORD_LENGTH, QC_LINGAIN_RONi_ERR, _iquadrant);
    cpl_propertylist_update_double(header, keyword, _ronerr);

    snprintf(keyword, KEYWORD_LENGTH, QC_LINGAIN_RONi_MEDIAN, _iquadrant);
    cpl_propertylist_update_double(header, keyword, _ronmed);

    snprintf(keyword, KEYWORD_LENGTH, QC_LINGAIN_RONi_MAD, _iquadrant);
    cpl_propertylist_update_double(header, keyword, _ronmad);

    snprintf(keyword, KEYWORD_LENGTH, QC_LINGAIN_CONADi, _iquadrant);
    cpl_propertylist_update_double(header, keyword, _conad);

    snprintf(keyword, KEYWORD_LENGTH, QC_LINGAIN_GAINi, _iquadrant);
    cpl_propertylist_update_double(header, keyword, _gain);

    snprintf(keyword, KEYWORD_LENGTH, QC_LINGAIN_GAINi_ERR, _iquadrant);
    cpl_propertylist_append_double(header, keyword, _gainerr);

    snprintf(keyword, KEYWORD_LENGTH, QC_LINGAIN_COUNTSi_MIN, _iquadrant);
    cpl_propertylist_append_double(header, keyword,
                                   cpl_table_get_double(gain, "SignalMin",
                                                        iquadrant, NULL));

    snprintf(keyword, KEYWORD_LENGTH, QC_LINGAIN_COUNTSi_MAX, _iquadrant);
    cpl_propertylist_append_double(header, keyword,
                                   cpl_table_get_double(gain, "SignalMax",
                                                        iquadrant, NULL));

#ifdef NONLINEARITY_MODEL
    snprintf(keyword, KEYWORD_LENGTH, MUSE_HDR_NONLINn_LLO, _iquadrant);
    cpl_propertylist_append_double(header, keyword,
                                   cpl_table_get_double(linearity, "SignalMin",
                                                        iquadrant, NULL));
    cpl_propertylist_set_comment(header, keyword,
                                 "[log10(ADU)] Minimum signal used for fit");

    snprintf(keyword, KEYWORD_LENGTH, MUSE_HDR_NONLINn_LHI, _iquadrant);
    cpl_propertylist_append_double(header, keyword,
                                   cpl_table_get_double(linearity, "SignalMax",
                                                        iquadrant, NULL));
    cpl_propertylist_set_comment(header, keyword,
                                 "[log10(ADU)] Maximum signal used for fit");

    int ncoefficient = (int)cpl_table_get_column_depth(linearity,
                                                       "Coefficients");

    snprintf(keyword, KEYWORD_LENGTH, MUSE_HDR_NONLINn_ORDER, _iquadrant);
    cpl_propertylist_append_double(header, keyword, ncoefficient - 1);
    cpl_propertylist_set_comment(header, keyword,
                                 "Order of the polynomial fit");

    const cpl_array *coefficients =
        cpl_table_get_array(linearity, "Coefficients", iquadrant);
    int icoefficient;
    for (icoefficient = 0; icoefficient < ncoefficient; ++icoefficient) {
      char comment[KEYWORD_LENGTH];
      double c = cpl_array_get_double(coefficients, icoefficient, NULL);
      snprintf(keyword, KEYWORD_LENGTH, MUSE_HDR_NONLINn_COEFFo,
               _iquadrant, (unsigned char)icoefficient);
      cpl_propertylist_append_double(header, keyword, c);

      snprintf(comment, KEYWORD_LENGTH,
               "%d order coefficient of the polynomial fit", icoefficient);
      cpl_propertylist_set_comment(header, keyword, comment);
    }

    snprintf(keyword, KEYWORD_LENGTH, QC_LINGAIN_NLFITi_RMS, _iquadrant);
    cpl_propertylist_append_double(header, keyword,
                                   cpl_table_get_double(linearity, "FitRms",
                                                        iquadrant, NULL));
#endif
  }
  cpl_table_delete(ron);

#ifdef NONLINEARITY_MODEL
  cpl_table_erase_column(linearity, "FitRms");

  muse_processing_save_table(aProcessing, aParams->nifu, linearity, header,
                             MUSE_TAG_NONLINGAIN, MUSE_TABLE_TYPE_CPL);
#else
  muse_processing_save_header(aProcessing, aParams->nifu, header,
                              MUSE_TAG_NONLINGAIN);
#endif

  cpl_propertylist_delete(header);
#ifdef NONLINEARITY_MODEL
  cpl_table_delete(linearity);
#endif
  cpl_table_delete(gain);

  return 0;
} /* muse_lingain_compute() */
