/* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set sw=2 sts=2 et cin: */
/*
 * This file is part of the MUSE Instrument Pipeline
 * Copyright (C) 2005-2014 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

/*---------------------------------------------------------------------------*
 *                            Includes                                       *
 *---------------------------------------------------------------------------*/
#include <cpl.h>
#include <math.h>
#include <string.h>

#include "muse_quality.h"
#include "muse_instrument.h"

#include "muse_cplwrappers.h"
#include "muse_pfits.h"
#include "muse_quadrants.h"
#include "muse_tracing.h"
#include "muse_utils.h"
#include "muse_data_format_z.h"

/*---------------------------------------------------------------------------*/
/**
 * @defgroup muse_quality      Quality flag handling
 *
 * These functions determine and handle the Euro3D-like 32bit integer quality
 * flags of all pixels.
 */
/*---------------------------------------------------------------------------*/

/**@{*/

/*---------------------------------------------------------------------------*/
/**
  @brief    Find bad (especially hot) pixels (in a master dark).
  @param    aDark       MUSE dark image to process
  @param    aSigmaLo    sigma for low cut (in terms of median deviation)
  @param    aSigmaHi    sigma for high cut (in terms of median deviation)
  @return   number of bad pixels found or a negative value on error

  This function is thought to operate on bias-subtracted dark images from
  which cosmic rays were removed (by suitable image combination).

  Determine outliers that deviate too much from homogenous noise in each
  of the four CCD quadrants. The low pixels are flagged EURO3D_DEADPIXEL,
  the high (hot) pixels as EURO3D_HOTPIXEL, both are also rejected in the
  bad pixel mask of the cpl_image of the data and stat extensions.
  If one of the sigma arguments is non-positive, the respective cut is ignored.

  Note that incoming bad pixels are excluded from the statistics, so they might
  be marked again as dark or hot and included in the final count.

  @error{set CPL_ERROR_NULL_INPUT and return -1, aDark is NULL}
  @error{set CPL_ERROR_ILLEGAL_INPUT and return -2,
         the data buffers of the data or dq components of aDark could not be accessed}
 */
/*---------------------------------------------------------------------------*/
int
muse_quality_dark_badpix(muse_image *aDark, double aSigmaLo, double aSigmaHi)
{
  cpl_ensure(aDark, CPL_ERROR_NULL_INPUT, -1);
  float *data = cpl_image_get_data_float(aDark->data);
  int *dq = cpl_image_get_data_int(aDark->dq);
  cpl_ensure(data && dq, CPL_ERROR_ILLEGAL_INPUT, -2);
#ifdef MUSE_DARK_PREPARE_TEST_DATA /* extract a region for (automated) testing, adapt header accordingly */
  muse_image *x = muse_image_new();
  x->data = cpl_image_extract(aDark->data, 1780, 1922, 3000, 4112);
  x->dq = cpl_image_extract(aDark->dq, 1780, 1922, 3000, 4112);
  x->stat = cpl_image_extract(aDark->stat, 1780, 1922, 3000, 4112);
  x->header = cpl_propertylist_duplicate(aDark->header);
  cpl_propertylist_update_int(x->header, "ESO DET OUT1 NX", 269); /* [1780:2048,1922:2056] */
  cpl_propertylist_update_int(x->header, "ESO DET OUT1 NY", 135);
  cpl_propertylist_update_int(x->header, "ESO DET OUT2 NX", 952); /* [2049:3000,1922:2056] */
  cpl_propertylist_update_int(x->header, "ESO DET OUT2 NY", 135);
  cpl_propertylist_update_int(x->header, "ESO DET OUT3 NX", 269); /* [1780:2048,2057:4112] */
  /* stays the same: cpl_propertylist_update_int(x->header, "ESO DET OUT3 NY", 2056); */
  cpl_propertylist_update_int(x->header, "ESO DET OUT4 NX", 952); /* [2049:3000,2057:4112] */
  /* stays the same: cpl_propertylist_update_int(x->header, "ESO DET OUT4 NY", 2056); */
  const char *fn = "muse_test_quality_dark.fits";
  muse_image_save(x, fn);
  cpl_msg_debug(__func__, "saved as \"%s\"", fn);
  muse_image_delete(x);
#endif

  /* transfer already known bad pixels into the cpl_mask of the image, *
   * so that they don't influence the mean/stdev values computed below */
  int nbad = muse_quality_image_reject_using_dq(aDark->data, aDark->dq,
                                                aDark->stat);
  cpl_msg_debug(__func__, "%d incoming bad pixels", nbad);
  cpl_binary *databpm = cpl_mask_get_data(cpl_image_get_bpm(aDark->data)),
             *statbpm = NULL;
  if (aDark->stat) {
    statbpm = cpl_mask_get_data(cpl_image_get_bpm(aDark->stat));
  }

  int nlo = 0, nhi = 0;
  unsigned char n;
  for (n = 1; n <= 4; n++) {
    cpl_size *w = muse_quadrants_get_window(aDark, n);
    cpl_stats_mode smode = CPL_STATS_MEDIAN | CPL_STATS_MEDIAN_DEV
                         | CPL_STATS_MIN | CPL_STATS_MAX;
    cpl_stats *s = cpl_stats_new_from_image_window(aDark->data, smode,
                                                   w[0], w[2], w[1], w[3]);
    double median = cpl_stats_get_median(s),
           mdev = cpl_stats_get_median_dev(s),
           min = cpl_stats_get_min(s),
           max = cpl_stats_get_max(s),
           locut = aSigmaLo > 0 ? median - aSigmaLo * mdev : min,
           hicut = aSigmaHi > 0 ? median + aSigmaHi * mdev : max;
    cpl_msg_debug(__func__, "quadrant %d bad pixel limits: %g ... %g +/- %g ... %g",
                  n, locut, median, mdev, hicut);
#if 0
    cpl_stats_dump(s, smode, stdout);
    fflush(stdout);
#endif
    cpl_stats_delete(s);

    int i, nx = cpl_image_get_size_x(aDark->data);
    for (i = w[0] - 1; i < w[1]; i++) {
      int j;
      for (j = w[2] - 1; j < w[3]; j++) {
        cpl_size pos = i + j*nx;
        double value = data[pos];
        if (value < locut) {
          dq[pos] |= EURO3D_DEADPIXEL;
          databpm[pos] = CPL_BINARY_1;
          if (statbpm) {
            statbpm[pos] = CPL_BINARY_1;
          }
          nlo++;
        }
        if (value > hicut) {
          dq[pos] |= EURO3D_HOTPIXEL;
          databpm[pos] = CPL_BINARY_1;
          if (statbpm) {
            statbpm[pos] = CPL_BINARY_1;
          }
          nhi++;
        }
      } /* for j (y direction) */
    } /* for i (x direction) */
    cpl_free(w);
  } /* for n (quadrants) */
  if (nlo || aSigmaLo > 0) {
    cpl_msg_info(__func__, "%d pixel%s lower than %.3f sigma marked as dark",
                 nlo, nlo != 1 ? "s" : "", aSigmaLo);
  }
  if (nhi || aSigmaHi > 0) {
    cpl_msg_info(__func__, "%d pixel%s higher than %.3f sigma marked as hot",
                 nhi, nhi != 1 ? "s" : "", aSigmaHi);
  }
  return nlo + nhi;
} /* muse_quality_dark_badpix() */

/*---------------------------------------------------------------------------*/
/**
  @brief    Find more bad pixels in residuals of a master dark.
  @param    aDarkRes   MUSE dark image to process
  @param    aSigma     sigma for low and high cuts (in terms of median deviation)
  @param    aIter      number of iterations to search for bad neightbors
  @return   number of bad pixels found or a negative value on error

  This function is thought to operate on master dark images which are
  bias-subtracted and where the background was already pre-subtracted using some
  model. This allows this function to use stricter sigma-clipping levels than
  possible without the subtraction, as used in @ref muse_quality_dark_badpix().

  Determine outliers that deviate too more than aSigma * median-deviation across
  the whole image. In a first step, they are only marked as bad, if there is at
  least one neighboring pixel that is above the limit as well.  The low pixels
  are flagged EURO3D_DEADPIXEL, the high (hot) pixels as EURO3D_HOTPIXEL in the
  dq extension of aDarkRes. Any existing bad-pixel mask of the cpl_images and
  the stat extension (containing the data variance) are ignored by this
  function, but pixels that are already flagged in the dq extension are not
  touched again (and not counted).

  If aIter is positive, further bad neightbors (strictly in x and y direction)
  are searched iteratively: if all four neightbors of a pixel are deviant, mark
  it as bad as well.

  Note that incoming bad pixels (marked as bad in the incoming bad-pixel mask of
  the cpl_image of the data component) are excluded from the statistics, but are
  otherwise treated like all other pixels.

  @error{set CPL_ERROR_NULL_INPUT and return -1,
         aDarkRes or its data and/or dq extensions are NULL}
  @error{set CPL_ERROR_ILLEGAL_INPUT and return -2,
         the data buffers of the data or dq components of aDarkRes could not be accessed}
 */
/*---------------------------------------------------------------------------*/
int
muse_quality_dark_refine_badpix(muse_image *aDarkRes, double aSigma,
                                unsigned short aIter)
{
  cpl_ensure(aDarkRes, CPL_ERROR_NULL_INPUT, -1);

  cpl_stats_mode smode = CPL_STATS_MEDIAN | CPL_STATS_MEDIAN_DEV;
  cpl_stats *s = cpl_stats_new_from_image(aDarkRes->data, smode);
  double median = cpl_stats_get_median(s),
         mdev = cpl_stats_get_median_dev(s),
         limit = aSigma * mdev;
  cpl_msg_debug(__func__, "bad pixel limit: %f (%f +/- %f)", limit, median,
                mdev);
#if 0
  cpl_stats_dump(s, smode, stdout);
  fflush(stdout);
#endif
  cpl_stats_delete(s);

  const float *res = cpl_image_get_data_float_const(aDarkRes->data);
  int *dq = cpl_image_get_data_int(aDarkRes->dq);
  cpl_ensure(res && dq, CPL_ERROR_ILLEGAL_INPUT, -2);
  int i, j, n = 0,
      nx = cpl_image_get_size_x(aDarkRes->data),
      ny = cpl_image_get_size_y(aDarkRes->data);
  for (i = 0; i < nx; i++) {
    for (j = 0; j < ny; j++) {
      if (dq[i + j*nx]) {
        /* already a bad pixel, do nothing */
        continue;
      }
      double r = res[i + j*nx];
      if (fabs(r) > limit) {
        /* check neighbors */
        int nhi = 0;
        /* left */
        if (i > 0 && fabs(res[(i-1) + j*nx]) > limit) {
          nhi++;
        }
        /* right */
        if (i < (nx-1) && fabs(res[(i+1) + j*nx]) > limit) {
          nhi++;
        }
        /* bottom */
        if (j > 0 && fabs(res[i + (j-1)*nx]) > limit) {
          nhi++;
        }
        /* top */
        if (j < (ny-1) && fabs(res[i + (j+1)*nx]) > limit) {
          nhi++;
        }
        /* if there are also at least two deviant neighbors, mark it as bad */
        if (nhi > 1) {
          if (r > 0) {
            dq[i + j*nx] |= EURO3D_HOTPIXEL;
          } else {
            dq[i + j*nx] |= EURO3D_DARKPIXEL;
          }
          n++;
        } /* if nhi */
      }/* if r */
    } /* for j (columns) */
  } /* for i (rows) */
  cpl_msg_debug(__func__, "%d new bad pixels after marking dubious cases", n);

  /* next step: for insignificant residuals, check if all neighbors are bad */
  unsigned short ni;
  for (ni = 1; ni <= aIter; ni++) {
    for (i = 0; i < nx; i++) {
      for (j = 0; j < ny; j++) {
        if (dq[i + j*nx]) {
          /* already a bad pixel, do nothing */
          continue;
        }
        /* count neighbors */
        int nhi = 0;
        if (i > 0 && fabs(res[(i-1) + j*nx]) > limit) {
          nhi++;
        }
        if (i < (nx-1) && fabs(res[(i+1) + j*nx]) > limit) {
          nhi++;
        }
        if (j > 0 && fabs(res[i + (j-1)*nx]) > limit) {
          nhi++;
        }
        if (j < (ny-1) && fabs(res[i + (j+1)*nx]) > limit) {
          nhi++;
        }
        /* if there are also deviant neighbors, mark it as bad */
        if (nhi == 4) {
          if (res[i + j*nx] > 0) {
            dq[i + j*nx] |= EURO3D_HOTPIXEL;
          } else {
            dq[i + j*nx] |= EURO3D_DARKPIXEL;
          }
          n++;
        } /* if nhi */
      } /* for j (columns) */
    } /* for i (rows) */
    cpl_msg_debug(__func__, "%d new bad pixels after iteration %hu marking "
                  "cases with bad neighbors", n, ni);
  } /* for ni iterations */

  return n;
} /* muse_quality_dark_refine_badpix() */

/*---------------------------------------------------------------------------*/
/**
  @brief    Find bad columns (in a master bias).
  @param    aBias   MUSE bias image to process
  @param    aLow    sigma for low cut
  @param    aHigh   sigma for high cut
  @return   number of pixels in bad columns or a negative value on error

  Operate per quadrant: determine columns that have lower or higher values than
  others. Within these columns, search for pixels that still have reasonable
  values. Mark all other pixels in these columns as EURO3D_DEADPIXEL.

  @error{return -1 and set CPL_ERROR_NULL_INPUT,
         aBias or its components are NULL}
 */
/*---------------------------------------------------------------------------*/
int
muse_quality_bad_columns(muse_image *aBias, double aLow, double aHigh)
{
  cpl_ensure(aBias && aBias->data && aBias->dq && aBias->stat && aBias->header,
             CPL_ERROR_NULL_INPUT, -1);

  int nx = cpl_image_get_size_x(aBias->data),
      nlo = 0, nhi = 0; /* numbers of low and high bad pixels */
  unsigned char n;
  for (n = 1; n <= 4; n++) {
    /* get the image window for this quadrant */
    cpl_size *w = muse_quadrants_get_window(aBias, n);
    cpl_vector *vmean = cpl_vector_new(w[1] - w[0] + 1),
               *vstdev = cpl_vector_new(w[1] - w[0] + 1);
    int i;
    for (i = w[0]; i <= w[1]; i++) {
      double mean = cpl_image_get_mean_window(aBias->data, i, w[2], i, w[3]),
             stdev = cpl_image_get_stdev_window(aBias->data, i, w[2], i, w[3]);
      cpl_vector_set(vmean, i - w[0], mean);
      cpl_vector_set(vstdev, i - w[0], stdev);
    } /* for i (all columns) */

    /* median and median deviation of the averages of all columns */
    double cmedian = cpl_vector_get_median_const(vmean),
           cmdev = muse_cplvector_get_adev_const(vmean, cmedian),
           hicut = cmedian + aHigh*cmdev,
           locut = cmedian - aLow*cmdev;
    char *keyword = cpl_sprintf(QC_BIAS_MASTER_RON, n);
    double ron = cpl_propertylist_get_double(aBias->header, keyword);
    cpl_free(keyword);
    cpl_msg_debug(__func__, "quadrant %1d: mean %f+/-%f(%f); valid range "
                  "%f...(%f+/-%f)...%f RON=%f", n, cpl_vector_get_mean(vmean),
                  cpl_vector_get_stdev(vmean), cpl_vector_get_mean(vstdev),
                  locut, cmedian, cmdev, hicut, ron);

    int j;
    float *data = cpl_image_get_data_float(aBias->data);
    int *dq = cpl_image_get_data_int(aBias->dq);
    for (i = w[0]; i <= w[1]; i++) {
      double cmean = cpl_vector_get(vmean, i - w[0]),
             cstdev = cpl_vector_get(vstdev, i - w[0]);
      if (cmean > hicut && cstdev > ron) {
        cpl_msg_debug(__func__, "hot column %d (%f+/-%f)", i, cmean, cstdev);
        /* now find first high pixel in this column */
        int jfirst = w[2], jlast = w[3];
        for (j = w[2]; j <= w[3]; j++) {
          if (data[(i-1) + (j-1)*nx] > hicut) {
            jfirst = j;
            break;
          }
        } /* for j */
        /* find last high pixel in this column */
        for (j = w[3]; j >= w[2]; j--) {
          if (data[(i-1) + (j-1)*nx] > hicut) {
            jlast = j;
            break;
          }
        } /* for j */
        /* everything in between is untrustworthy, mark as bad */
        for (j = jfirst; j <= jlast; j++) {
          dq[(i-1) + (j-1)*nx] |= EURO3D_DEADPIXEL;
          nhi++;
        } /* for j */
      } else if (cmean < locut) {
        cpl_msg_debug(__func__, "dark column %d (%f+/-%f)", i, cmean, cstdev);
        /* now find first high pixel in this column */
        int jfirst = w[2], jlast = w[3];
        for (j = w[2]; j <= w[3]; j++) {
          if (data[(i-1) + (j-1)*nx] < locut) {
            jfirst = j;
            break;
          }
        } /* for j */
        /* find last high pixel in this column */
        for (j = w[3]; j >= w[2]; j--) {
          if (data[(i-1) + (j-1)*nx] < locut) {
            jlast = j;
            break;
          }
        } /* for j */
        /* everything in between is untrustworthy, mark as bad */
        for (j = jfirst; j <= jlast; j++) {
          dq[(i-1) + (j-1)*nx] |= EURO3D_DEADPIXEL;
          nhi++;
        } /* for j */
      } /* if < locut */
    } /* for i (all columns) */

    cpl_vector_delete(vmean);
    cpl_vector_delete(vstdev);
    cpl_free(w);
  } /* for n (quadrants) */

  cpl_msg_info(__func__, "%d low and %d high pixels found", nlo, nhi);

  return nlo + nhi;
} /* muse_quality_bad_columns() */

/*---------------------------------------------------------------------------*/
/**
  @brief    Find bad (especially dark) pixels (in a master flat).
  @param    aFlat       MUSE flat image to process
  @param    aTrace      the trace table to determine the slice edges
  @param    aSigmaLo    sigma for low cut (in terms of median deviation)
  @param    aSigmaHi    sigma for high cut (in terms of median deviation)
  @return   number of bad pixels found or a negative value on error

  Operate per slice: loop along all rows in the y-direction. Within these rows,
  search for pixels that have lower or higher values than others.  Mark low
  pixels as EURO3D_DARKPIXEL and high pixels as EURO3D_HOTPIXEL.  Low pixels
  below 20% of the mean flat-field level are also marked as EURO3D_LOWQEPIXEL.
  Pixels near the edge of a slice (as given by the trace function) are excluded
  from the bad pixel search.

  Finally, pixels with non-positive values are marked as EURO3D_BADOTHER, but
  are not added to the total number, since there are so many (at the CCD edges,
  between slices, at the blue end of the CCD).

  @error{set CPL_ERROR_NULL_INPUT and return -1,
         aFlat or its components or aTrace are NULL}
 */
/*---------------------------------------------------------------------------*/
int
muse_quality_flat_badpix(muse_image *aFlat, cpl_table *aTrace,
                         double aSigmaLo, double aSigmaHi)
{
  cpl_ensure(aFlat && aFlat->data && aFlat->dq && aFlat->stat && aTrace,
             CPL_ERROR_NULL_INPUT, -1);

  cpl_msg_info(__func__, "Marking dark/bright pixels using sigmas %.2f/%.2f",
               aSigmaLo, aSigmaHi);
  int ndark = 0, nhot = 0, nlowqe = 0,
      nx = cpl_image_get_size_x(aFlat->data),
      ny = cpl_image_get_size_y(aFlat->data);
  float *data = cpl_image_get_data_float(aFlat->data);
  int *dq = cpl_image_get_data_int(aFlat->dq);

  /* get mean count level of the flat-field, *
   * to decide for or against low QE pixels  */
  double mean = cpl_image_get_mean(aFlat->data);
#if 0
  cpl_msg_debug(__func__, "mean flat value: %f", mean);
#endif
  int nslice;
  for (nslice = 1; nslice <= kMuseSlicesPerCCD; nslice++) {
    /* get the tracing polynomials for this slice */
    cpl_polynomial **ptrace = muse_trace_table_get_polys_for_slice(aTrace,
                                                                   nslice);
    if (!ptrace) {
      cpl_msg_warning(__func__, "slice %2d: tracing polynomials missing!",
                      nslice);
      continue;
    }

    /* within each slice, loop from bottom to top */
    int j;
    for (j = 0; j < ny; j++) {
      /* determine the slice edges for this vertical position so that we  *
       * can loop over those pixels, excluding slice edges for the moment */
      cpl_errorstate prestate = cpl_errorstate_get();
      double x1 = cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_LEFT], j+1, NULL),
             x2 = cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_RIGHT], j+1, NULL);
      if (!cpl_errorstate_is_equal(prestate) || !isnormal(x1) || !isnormal(x2)
          || x1 < 1 || x2 > nx || x1 > x2) {
        cpl_msg_warning(__func__, "slice %2d: faulty polynomial detected at "
                        "y=%d (borders: %f ... %f): %s", nslice, j+1, x1, x2,
                        cpl_error_get_message());
        j = ny; /* skip the rest of this slice */
        continue;
      }
      /* integer limits where we can be sure to get only well- *
       * illuminated pixels within the width of the slice      */
      int mleft = ceil(x1),     /* FITS coords, starting at 1! */
          mright = floor(x2);

#if 0
      double xc = cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_CENTER], j+1, NULL);
      cpl_msg_debug(__func__, "%02d %04d: %04d/%.3f...%.3f...%04d/%.3f "
                    "(%02d/%.3f wide)", nslice, j+1, mleft,x1, xc, mright,x2,
                    mright - mleft, x2 - x1);
#endif
      unsigned statmask = CPL_STATS_MIN | CPL_STATS_MAX
                        | CPL_STATS_MEAN | CPL_STATS_STDEV
                        | CPL_STATS_MEDIAN | CPL_STATS_MEDIAN_DEV;
      cpl_stats *stats = cpl_stats_new_from_image_window(aFlat->data, statmask,
                                                         mleft, j+1,
                                                         mright, j+1);
#if 0
      cpl_stats_dump(stats, statmask, stdout);
      fflush(stdout);
#endif
      double median = cpl_stats_get_median(stats),
             llim = median - aSigmaLo * cpl_stats_get_median_dev(stats),
             hlim = median + aSigmaHi * cpl_stats_get_median_dev(stats);
      cpl_stats_delete(stats);
      /* limit <= 0 doesn't make sense for master flats, set *
       * to value that would also work for normalized image  */
      llim = llim <= 0 ? 1.e-4 : llim;
#if 0
      cpl_msg_debug(__func__, "limits: %f...%f (sigmas %.2f/%.2f)", llim, hlim,
                    aSigmaLo, aSigmaHi);
#endif

      /* except contiguous regions of low flux at the edges, but *
       * at most kMuseSliceMaxEdgeWidth pixels on each side      */
      int i, i1 = mleft - 1, i2 = mright - 1; /* indices, starting at 0! */
      for (i = mleft - 1; i < mleft + kMuseSliceMaxEdgeWidth; i++) {
        if (data[i + j*nx] > llim) {
          i1 = i;
          break;
        }
      } /* for i (horizontal pixels) */
      for (i = mright - 1; i > mright - kMuseSliceMaxEdgeWidth; i--) {
        if (data[i + j*nx] > llim) {
          i2 = i;
          break;
        }
      } /* for i (horizontal pixels) */

      /* now loop over all "normal" pixels of this slice     *
       * horizontally, excluding the pixels at the very edge */
      for (i = i1; i <= i2; i++) {
        if (data[i + j*nx] < llim) {
          dq[i + j*nx] |= EURO3D_DARKPIXEL;
          if (data[i + j*nx] < 0.2 * mean) {
            /* is a low QE pixel according to the Euro3D specifications, *
             * as it has less than 20% of the average flat-field level   */
            dq[i + j*nx] |= EURO3D_LOWQEPIXEL;
            nlowqe++;
          }
#if 0
          cpl_msg_debug(__func__, "%04d,%04d is DARK%s: %f", i+1, j+1,
                        data[i + j*nx] < 0.2 * mean ? " and low QE" : "",
                        data[i + j*nx]);
#endif
          ndark++;
        } else if (data[i + j*nx] > hlim) {
          dq[i + j*nx] |= EURO3D_HOTPIXEL;
#if 0
          cpl_msg_debug(__func__, "%04d,%04d is HOT: %f", i+1, j+1, data[i + j*nx]);
#endif
          nhot++;
        }
      } /* for i (horizontal pixels) */
    } /* for j (vertical direction) */

    muse_trace_polys_delete(ptrace);
  } /* for nslice */

  /* search and mark non-positive pixels */
  int i, nnonpos = 0;
  for (i = 0; i < nx; i++) {
    int j;
    for (j = 0; j < ny; j++) {
      if (data[i + j*nx] <= 0) {
        dq[i + j*nx] |= EURO3D_BADOTHER;
        nnonpos++;
      }
    } /* for j (vertical direction) */
  } /* for i (horizontal pixels) */

  cpl_msg_info(__func__, "Found %d dark (%d of them are also low QE), %d hot, "
               "and %d non-positive pixels", ndark, nlowqe, nhot, nnonpos);
  return ndark + nhot;
} /* muse_quality_flat_badpix() */

/*---------------------------------------------------------------------------*/
/**
  @brief    Set all pixels above the saturation limit in the bad pixel image.
  @param    aImage   MUSE image to process
  @return   number of pixels detected as saturated, or a negative number on
            error

  Saturated pixels are all those with data values higher than
  kMuseSaturationLimit or zero (less than FLT_EPSILON).

  @error{set CPL_ERROR_NULL_INPUT and return -1,
         aImage or its relevant components are NULL}
 */
/*---------------------------------------------------------------------------*/
int
muse_quality_set_saturated(muse_image *aImage)
{
  cpl_ensure(aImage && aImage->data && aImage->dq, CPL_ERROR_NULL_INPUT, -1);

  float *data = cpl_image_get_data_float(aImage->data);
  int *dq = cpl_image_get_data_int(aImage->dq);
  int i, j, nsaturated = 0,
      nx = cpl_image_get_size_x(aImage->data),
      ny = cpl_image_get_size_y(aImage->data);
  for (i = 0; i < nx; i++) {
    for (j = 0; j < ny; j++) {
      if (data[i + j*nx] > kMuseSaturationLimit ||
          data[i + j*nx] < FLT_EPSILON) {
        dq[i + j*nx] |= EURO3D_SATURATED;
        nsaturated++;
      }
    } /* for j */
  } /* for i */

  return nsaturated;
} /* muse_quality_set_saturated() */

/*---------------------------------------------------------------------------*/
/**
  @brief    Convert a data quality (DQ) image extension to a bad pixel table.
  @param    aDQ   the data quality image
  @return   a bad pixel table on success or NULL on error

  Note that the input image is assumed to be trimmed, i.e. overscans and
  prescans are missing. This function adds both back, assuming standard MUSE
  image sizes, so that the output table contains bad pixels in raw image
  coordinates.

  @error{set CPL_ERROR_NULL_INPUT\, return NULL, aDQ is NULL}
  @error{return empty table (zero rows), no bad pixels found in aDQ image}
 */
/*---------------------------------------------------------------------------*/
cpl_table *
muse_quality_convert_dq(cpl_image *aDQ)
{
  cpl_ensure(aDQ, CPL_ERROR_NULL_INPUT, NULL);
  int nx = cpl_image_get_size_x(aDQ),
      ny = cpl_image_get_size_y(aDQ);
  const int *dq = cpl_image_get_data_int_const(aDQ);
  int i, j, nbad = 0;
  for (i = 0; i < nx; i++) {
    for (j = 0; j < ny; j++) {
      if (dq[i + j*nx] != EURO3D_GOODPIXEL) {
        nbad++;
      }
    } /* for j (columns) */
  } /* for i (rows) */
  cpl_table *table = muse_cpltable_new(muse_badpix_table_def, nbad);
  /* if no bad pixels were found, return the empty table */
  if (!nbad) {
    return table;
  }

  nbad = 0;
  for (i = 0; i < nx; i++) {
    for (j = 0; j < ny; j++) {
      if (dq[i + j*nx] == EURO3D_GOODPIXEL) {
        continue;
      }
      int xbad = i + 1,
          ybad = j + 1;
      /* transform to raw positions */
      muse_quadrants_coords_to_raw(NULL, &xbad, &ybad);
      cpl_table_set_int(table, MUSE_BADPIX_X, nbad, xbad);
      cpl_table_set_int(table, MUSE_BADPIX_Y, nbad, ybad);
      cpl_table_set_int(table, MUSE_BADPIX_DQ, nbad, dq[i + j*nx]);
      /* no way to set MUSE_BADPIX_VALUE from the data in the DQ extension */
      nbad++;
    } /* for j (columns) */
  } /* for i (rows) */
  return table;
} /* muse_quality_convert_dq() */

/*---------------------------------------------------------------------------*/
/**
  @brief    Merge two bad pixel tables.
  @param    aTable     the table to merge into
  @param    aToMerge   the table to merge into the first one
  @return   CPL_ERROR_NONE on success, another CPL error code on failure

  The tables are merged in such a way, that in the output table each pixel is
  only listed once, with the bad pixel flag ORed.
  The output table is sorted by the pixel coordinates.

  @error{return CPL_ERROR_NULL_INPUT, aTable or aToMerge are NULL}
  @error{propagate CPL error code, inserting the table fails}
 */
/*---------------------------------------------------------------------------*/
cpl_error_code
muse_quality_merge_badpix(cpl_table *aTable, const cpl_table *aToMerge)
{
  cpl_ensure_code(aTable && aToMerge, CPL_ERROR_NULL_INPUT);
  cpl_error_code rc = cpl_table_insert(aTable, aToMerge,
                                       cpl_table_get_nrow(aTable));
  cpl_ensure_code(rc == CPL_ERROR_NONE, rc);

  /* sort the merged table, to ease search of duplicate pixel entries */
  cpl_propertylist *order = cpl_propertylist_new();
  cpl_propertylist_append_bool(order, MUSE_BADPIX_X, FALSE);
  cpl_propertylist_append_bool(order, MUSE_BADPIX_Y, FALSE);
  cpl_table_sort(aTable, order);
  cpl_propertylist_delete(order);

  /* now select rows for duplicate pixel coordinate entries, merge their DQ */
  cpl_table_unselect_all(aTable);
  const int *x = cpl_table_get_data_int_const(aTable, MUSE_BADPIX_X),
            *y = cpl_table_get_data_int_const(aTable, MUSE_BADPIX_Y);
  int *dq = cpl_table_get_data_int(aTable, MUSE_BADPIX_DQ);
  float *value = cpl_table_get_data_float(aTable, MUSE_BADPIX_VALUE);
  int i, nrow = cpl_table_get_nrow(aTable);
  for (i = 1; i < nrow; i++) {
    if (x[i] == x[i-1] && y[i] == y[i-1]) {
      dq[i-1] |= dq[i]; /* merge Euro3D status flags */
      if (value) {
        /* as we know nothing else, use the maximum for the value, *
         * but only if both values are valid                       */
        cpl_boolean v1 = cpl_table_is_valid(aTable, MUSE_BADPIX_VALUE, i-1),
                    v2 = cpl_table_is_valid(aTable, MUSE_BADPIX_VALUE, i);
        if (v1 && v2) {
          value[i-1] = fmax(value[i-1], value[i]);
        } else if (v2) { /* if v1, then nothing needs to be done */
          value[i-1] = value[i];
        } else if (!v1 && !v2) { /* neither is valid! */
          cpl_table_set_invalid(aTable, MUSE_BADPIX_VALUE, i-1);
        }
      } /* if value */
      cpl_table_select_row(aTable, i);
    } /* if same coordinates */
  } /* for i (all table rows) */
  rc = cpl_table_erase_selected(aTable);

  return rc;
} /* muse_quality_merge_badpix() */

/*---------------------------------------------------------------------------*/
/**
  @brief    Reject pixels of one or two images on a DQ image.
  @param    aData   the data image to process
  @param    aDQ     the image to use
  @param    aStat   the optional variance image to process
  @return   number of rejected pixels on success, a negative value on failure

  Pixels in aData (and, if given, aStat) are set as bad depending on aDQ, using
  cpl_image_reject(). This is to let other CPL functions automatically ignore
  the bad pixels.

  This function can be used instead of muse_image_reject_from_dq(), if one has
  data not in the form of a muse_image, but in separate structures. E.g. when
  processing planes from a muse_datacube. It also returns the count of bad
  pixels, in case that is needed.

  @error{set CPL_ERROR_NULL_INPUT\, return -1, aData or aDQ are NULL}
  @error{set CPL_ERROR_INCOMPATIBLE_INPUT\, return -2,
         aData (and aStat\, if given) and aDQ have different sizes}
  @error{propagate error of cpl_image_get_data_int_const()\, return -3,
         aDQ is not of type CPL_TYPE_INT}
 */
/*---------------------------------------------------------------------------*/
int
muse_quality_image_reject_using_dq(cpl_image *aData, cpl_image *aDQ,
                                   cpl_image *aStat)
{
  cpl_ensure(aData && aDQ, CPL_ERROR_NULL_INPUT, -1);
  int nx = cpl_image_get_size_x(aData),
      ny = cpl_image_get_size_y(aData);
  cpl_ensure(nx == cpl_image_get_size_x(aDQ) &&
             ny == cpl_image_get_size_y(aDQ), CPL_ERROR_INCOMPATIBLE_INPUT, -2);
  if (aStat) {
    cpl_ensure(nx == cpl_image_get_size_x(aStat) &&
               ny == cpl_image_get_size_y(aStat), CPL_ERROR_INCOMPATIBLE_INPUT,
               -2);
  }
  const int *dq = cpl_image_get_data_int_const(aDQ);
  if (!dq) {
    return -3;
  }
#if 0
  double cputime = cpl_test_get_cputime(),
         walltime = cpl_test_get_walltime();
#endif
  cpl_binary *bpm = cpl_mask_get_data(cpl_image_get_bpm(aData)),
             *statbpm = NULL;
  if (aStat) {
    statbpm = cpl_mask_get_data(cpl_image_get_bpm(aStat));
  }
  int i, j, nbad = 0;
  for (i = 0; i < nx; i++) {
    for (j = 0; j < ny; j++) {
      if (dq[i + j*nx] == EURO3D_GOODPIXEL) {
        continue;
      }
      nbad++;
      bpm[i + j*nx] = CPL_BINARY_1;
      if (aStat) {
        statbpm[i + j*nx] = CPL_BINARY_1;
      }
    } /* for j (columns) */
  } /* for i (rows) */
#if 0
  double cputime2 = cpl_test_get_cputime(),
         walltime2 = cpl_test_get_walltime();
  cpl_msg_debug(__func__, "reject_using_dq times new: cpu %f wall %f", cputime2 - cputime,
                walltime2 - walltime);
#endif
  return nbad;
} /* muse_quality_image_reject_using_dq() */

/*---------------------------------------------------------------------------*/
/**
  @brief    Merge bad pixel table in memory with table from file on disk.
  @param    aTable     table to merge
  @param    aInFile    the filename of the file to read the other table from
  @param    aExtname   the FITS extension name for the other table
  @param    aExt       pointer to where to save the extension number from
                       which the other table was read
  @return   the merged table or NULL on error

  Load an existing bad pixel table from file and merge it with the input table,
  using @ref muse_quality_merge_badpix().

  This function is to be used from a muse_badpix tool, it outputs messages using
  stdio functions instead of CPL messages.

  @error{set CPL_ERROR_NULL_INPUT\, return NULL, aTable and/or aInFile are NULL}
  @error{set CPL_ERROR_DATA_NOT_FOUND\, return NULL,
         extension aExtname was not found or table could not be loaded}
 */
/*---------------------------------------------------------------------------*/
cpl_table *
muse_quality_merge_badpix_from_file(const cpl_table *aTable, const char *aInFile,
                                    const char *aExtname, int *aExt)
{
  cpl_ensure(aTable && aInFile, CPL_ERROR_NULL_INPUT, NULL);

  int extension = cpl_fits_find_extension(aInFile, aExtname);
  cpl_table *table = NULL;
  if (extension < 1) {
    if (cpl_msg_get_level() == CPL_MSG_DEBUG) {
      printf("Input table \"%s\" does not contain a table for EXTNAME=\"%s\""
             " yet\n", aInFile, aExtname);
    }
  } else {
    table = cpl_table_load(aInFile, extension, 1);
    if (!table) {
      printf("WARNING: could not load BADPIX_TABLE from EXTNAME=\"%s\" from "
             "table \"%s\" (the headers say that it should be extension %d)!\n",
             aExtname, aInFile, extension);
    }
  } /* else */
  cpl_ensure(table, CPL_ERROR_DATA_NOT_FOUND, NULL);

  if (aExt) {
    *aExt = extension;
  }

  cpl_size nold = cpl_table_get_nrow(table);
  cpl_error_code rc = muse_quality_merge_badpix(table, aTable);
  if (rc != CPL_ERROR_NONE) {
    printf("WARNING: Merging input and new table failed: %s\n",
           cpl_error_get_message());
    printf("Table still has %"CPL_SIZE_FORMAT" bad pixel%s\n", nold,
           nold != 1 ? "s" : "");
  } else {
    cpl_size nbad = cpl_table_get_nrow(table);
    printf("Merged %"CPL_SIZE_FORMAT" of %"CPL_SIZE_FORMAT" bad pixel%s into "
           "the input table (now %"CPL_SIZE_FORMAT" entries)\n", nbad - nold,
           cpl_table_get_nrow(table), (nbad - nold) != 1 ? "s" : "", nbad);
  }

  return table;
} /* muse_quality_merge_badpix_from_file() */

/*---------------------------------------------------------------------------*/
/**
  @brief    Copy bad pixel table on disk, replacing the table in one extension.
  @param    aInFile      filename of the file to read
  @param    aOutFile     filename of the file to write
  @param    aExtension   FITS extension number to replace (has to be > 0)
  @param    aTable       table to save in given extension
  @return   CPL_ERROR_NONE on success, another cpl_error_code on failure

  This function copies the primary FITS header and all binary table extensions
  from input to output, replacing one table in on extension. Extensions that are
  not binary tables are ignored.

  This function is to be used from a muse_badpix tool, it outputs messages using
  stdio functions instead of CPL messages.

  XXX this needs error handling for failure to save files, too...

  @error{return CPL_ERROR_NULL_INPUT, aInFile\, aOutFile\, or aTable are NULL}
  @error{propagate CPL error code, opening aInFile fails}
 */
/*---------------------------------------------------------------------------*/
cpl_error_code
muse_quality_copy_badpix_table(const char *aInFile, const char *aOutFile,
                               int aExtension, const cpl_table *aTable)
{
  cpl_ensure_code(aInFile && aOutFile && aTable, CPL_ERROR_NULL_INPUT);

  cpl_errorstate state = cpl_errorstate_get();
  cpl_error_code rc = CPL_ERROR_NONE;
  cpl_size nextensions = cpl_fits_count_extensions(aInFile);
  if (!cpl_errorstate_is_equal(state)) {
    rc = cpl_error_get_code();
  }
  if (nextensions > 0) {
    printf("Saving primary header and %"CPL_SIZE_FORMAT" extensions to \"%s\"\n",
           nextensions, aOutFile);
  }
  cpl_size i;
  for (i = 0; i <= nextensions; i++) {
    cpl_propertylist *extheader = cpl_propertylist_load(aInFile, i);
    if (i == 0) {
      /* just save the header for the primary data unit but update    *
       * the pipeline filename in the header to the one we write here */
      cpl_propertylist_update_string(extheader, "PIPEFILE", aOutFile);
      cpl_propertylist_set_comment(extheader, "PIPEFILE",
                                   "pretend to be a pipeline output file");
      cpl_propertylist_save(extheader, aOutFile, CPL_IO_CREATE);
      if (cpl_msg_get_level() == CPL_MSG_DEBUG) {
        printf("Saved primary header to \"%s\"\n", aOutFile);
      }
      cpl_propertylist_delete(extheader);
      continue;
    }
    if (aTable && i == aExtension) {
      unsigned char nifu = muse_utils_get_ifu(extheader);
      printf("Saving merged table of IFU %2hhu to extension %"CPL_SIZE_FORMAT
             "\n", nifu, i);
      cpl_table_save(aTable, NULL, extheader, aOutFile, CPL_IO_EXTEND);
      cpl_propertylist_delete(extheader);
      continue;
    }

    cpl_table *tab = NULL;
    const char *xtension = cpl_propertylist_get_string(extheader, "XTENSION");
    if (xtension && strncmp(xtension, "BINTABLE", 8)) {
      if (cpl_msg_get_level() == CPL_MSG_DEBUG) {
        printf("WARNING: Not a binary table extension, skipping data section "
               "(extension %"CPL_SIZE_FORMAT")", i);
      }
      cpl_propertylist_save(extheader, aOutFile, CPL_IO_EXTEND);
    } else {
      tab = cpl_table_load(aInFile, i, 1);
      cpl_table_save(tab, NULL, extheader, aOutFile, CPL_IO_EXTEND);
      if (cpl_msg_get_level() == CPL_MSG_DEBUG) {
        printf("Saved table extension %"CPL_SIZE_FORMAT" to \"%s\"\n", i,
               aOutFile);
      }
    }
    cpl_table_delete(tab);
    cpl_propertylist_delete(extheader);
  } /* for i (all FITS extensions) */

  return rc;
} /* muse_quality_copy_badpix_table() */

/**@}*/
