/* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set sw=2 sts=2 et cin: */
/*
 * This file is part of the MUSE Instrument Pipeline
 * Copyright (C) 2005-2014 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

/*----------------------------------------------------------------------------*
 *                             Includes                                       *
 *----------------------------------------------------------------------------*/
#include <cpl.h>
#include <math.h>

#include "muse_artifacts.h"

#include "muse_quality.h"

/*----------------------------------------------------------------------------*/
/**
 * @defgroup muse_artifacts    Artifact handling
 *
 * This group implements a few functions to be used to remove artifacts from
 * MUSE data at the level of single CCDs.
 */
/*----------------------------------------------------------------------------*/

/**@{*/

/*----------------------------------------------------------------------------*/
/**
  @private
  @brief  Compute the subframe statistics for the DCR algorithm and mark cosmic
          ray hits.
  @param  aImage   image on which to replace cosmic rays
  @param  aX1      left x position of subframe (starting at 1)
  @param  aX2      right x position of subframe (starting at 1)
  @param  aY1      lower y position of subframe (starting at 1)
  @param  aY2      upper y position of subframe (starting at 1)
  @param  aThres   threshold in factors of standard deviation
  @param  aDebug   output extra debug messages if positive
  @return The number of pixels affected by cosmic rays.
  @remark This function does no error checking of the input parameters, it
          assumes that the caller has validated them.

  This function carries out steps 2 to 8 of the DCR algorithm (see
  muse_cosmics_dcr()).
 */
/*----------------------------------------------------------------------------*/
static int
muse_cosmics_dcr_subframe(muse_image *aImage,
                          int aX1, int aX2, int aY1, int aY2,
                          float aThres, unsigned short aDebug)
{
  float *data = cpl_image_get_data_float(aImage->data);
  int *dq = cpl_image_get_data_int(aImage->dq);

  /* compute mean and standard deviation for good pixels, */
  double dsum = 0., dsq = 0.,
         min = FLT_MAX, max = -FLT_MAX;
  int i, ngood = 0,
      nx = cpl_image_get_size_x(aImage->data),
      ny = cpl_image_get_size_y(aImage->data);
  for (i = aX1 - 1; i < aX2 && i < nx; i++) {
    int j;
    for (j = aY1 - 1; j < aY2 && j < ny; j++) {
      if (dq[i + j*nx]) { /* exclude bad pixels */
        continue;
      }
      dsum += data[i + j*nx];
      dsq += data[i + j*nx]*data[i + j*nx];
      ngood++;
      if (data[i + j*nx] < min) {
        min = data[i + j*nx];
      }
      if (data[i + j*nx] > max) {
        max = data[i + j*nx];
      }
    } /* for j */
  } /* for i */
  /* sigma = sqrt((sum(ci^2) - (sum(ci))^2 / n) / n) as defined by Pych */
  double mean = dsum / ngood, /* simple average */
         sigma = sqrt((dsq - dsum*dsum / ngood) / ngood),
         lo = mean - sigma * aThres,
         hi = mean + sigma * aThres;
  if (aDebug > 2) {
    printf("         stats(1): %f+/-%f -> limits=%f...%f, extremes=%f...%f\n",
            mean, sigma, lo, hi, min, max);
    fflush(stdout);
  }
  if (!isfinite(mean) || !isfinite(sigma) || !isfinite(lo) || !isfinite(hi) ||
      min == FLT_MAX || max == -FLT_MAX) {
    if (aDebug > 0) {
      printf("No good pixels found or statistics are infinite!\n"
             "  stats: %f+/-%f -> limits=%f...%f, extremes=%f...%f\n",
             mean, sigma, lo, hi, min, max);
      fflush(stdout);
    }
    return 0;
  }

  /* compute mean and standard deviation for good pixels between the limits    *
   * at the same time create the histogram of good points, even outside limits */
#define HIST_BIN_WIDTH 1. /* which of each histogram bin in counts */
  long hlength = lround((max - min) / HIST_BIN_WIDTH + 1);
  int *hist = cpl_calloc(hlength, sizeof(int));
  dsum = 0.;
  dsq = 0.;
  ngood = 0;
  for (i = aX1 - 1; i < aX2 && i < nx; i++) {
    int j;
    for (j = aY1 - 1; j < aY2 && j < ny; j++) {
      if (dq[i + j*nx]) { /* exclude bad pixels */
        continue;
      }
      /* only good pixels into histogram, add to the appropriate bin */
      hist[lround((data[i + j*nx] - min) / HIST_BIN_WIDTH)] += 1;
      if (data[i + j*nx] > lo && data[i + j*nx] < hi) {
        /* add only good pixels that are between thresholds into statistics */
        dsum += data[i + j*nx];
        dsq += data[i + j*nx]*data[i + j*nx];
        ngood++;
      } /* if between limits */
    } /* for j */
  } /* for i */
  mean = dsum / ngood;
  sigma = sqrt((dsq - dsum*dsum / ngood) / ngood);

  /* Find the mode of the distribution of counts *
   * (i.e., the peak of the histogram).          */
  cpl_array *histogram = cpl_array_wrap_int(hist, hlength);
  cpl_size nmaxpos; /* the mode of the histogram distribution */
  cpl_array_get_maxpos(histogram, &nmaxpos);
  if (aDebug > 2) {
    printf("         stats(2): %f+/-%f, histogram: length=%ld, peak=%.0f at %f "
           "(%"CPL_SIZE_FORMAT")\n", mean, sigma, hlength,
           cpl_array_get_max(histogram), (double)nmaxpos * HIST_BIN_WIDTH + min,
           nmaxpos);
    if (aDebug > 3) {
      printf("         histogram:\n           entry    value     number\n");
      for (i = 0; i < hlength; i++) {
        printf("         %7d %10.3f %8d\n", i, i * HIST_BIN_WIDTH + min, hist[i]);
      }
    }
    fflush(stdout);
  }

  /* Find the first gap that is wider than a threshold, which is the standard *
     deviation multiplied by an arbitrary number (usually 3.0).               */
  double gapwidth = sigma * aThres;

  for (i = nmaxpos ; i < hlength; i++) {
    if (hist[i] == 0) {
      /* we found the start of a gap, search the end of it */
      int j = i + 1;
      while (j < hlength && !hist[j]) {
        j++;
      }
      if (j >= hlength) {
        /* there is no gap (left) in this histogram, we are done */
        cpl_array_delete(histogram); /* clean up array and the data buffer */
        return 0;
      }
      double gap = (double)(j - i) * HIST_BIN_WIDTH;
      if (gap > gapwidth) {
        /* found the right gap, starting at i */
        break;
      }
      i = j; /* else jump to the end of the gap */
    } /* if gap */
  } /* for i */
  cpl_array_delete(histogram); /* clean up array and the data buffer */

  /* if we are at the end of the histogram, we *
   * haven't found a gap of the necessary size */
  if (i >= hlength) {
    return 0;
  }

  /* If such a gap exists, flag pixels with counts lying *
   * above the gap as affected by cosmic rays.           */
  hi = i * HIST_BIN_WIDTH + min;
  if (aDebug > 2) {
    printf("         flagging pixels above %.3f counts (%d of %ld in histogram)\n",
           hi, i+1, hlength);
    fflush(stdout);
  }
  int ncr = 0;
  for (i = aX1 - 1; i < aX2 && i < nx; i++) {
    int j;
    for (j = aY1 - 1; j < aY2 && j < ny; j++) {
      if (!dq[i + j*nx] && data[i + j*nx] > hi) { /* flag only good pixels */
        dq[i + j*nx] |= EURO3D_COSMICRAY;
        ncr++;
      } /* if good pixel above gap */
    } /* for j */
  } /* for i */

  return ncr;
} /* muse_cosmics_dcr_subframe() */

/*----------------------------------------------------------------------------*/
/**
  @brief  Quickly mark cosmic rays in an image using the DCR algorithm.
  @param  aImage   image on which to replace cosmic rays
  @param  aXBox    search box size in x
  @param  aYBox    search box size in y
  @param  aPasses  maximum number of cleaning passes
  @param  aThres   detection gap threshold in factors of standard deviation
  @return the integer number of cosmic rays found or a negative value on error
  @remark The result is directly applied to the input image: the cosmic
          rays are marked in the dq extension of the input image. The data and
          stat components of the input image are not changed.
  @remark This routine should only be used for quick reduction or in the
          case that only a single exposure of one target is available.

  This uses the DCR algorithm of Pych, 2004, PASP, 116, 148. From Sect. 2 ("The
  Algorithm") of the paper:

  1. Select small sized subframes that cover the whole frame, with substantial
     overlap. (This implementation overlaps 50% of each subframe with the next
     subframe, more around the top and right edges of the image. Pych recommends
     to have at least 100 pixels within each subframe for good statistics.)

  In each subframe:

  2. Calculate the standard deviation of the distribution of counts:
     sigma = sqrt((sum(ci^2) - (sum(ci))^2 / n) / n)

  3. Apply a single sigma-clipping step to correct the estimate of standard
     deviation for outlying pixels.

  4. Construct a histogram of the distribution of counts.

  5. Find the mode of the distribution of counts (i.e., the peak of the
     histogram).

  6. In the interval of counts higher than the mode, find gaps in the histogram
     (i.e., bins with zero data points).

  7. Find the first gap that is wider than a threshold, which is the standard
     deviation multiplied by an arbitrary number (usually 3.0).

  8. If such a gap exists, flag pixels with counts lying above the gap as
     affected by cosmic rays.

  @error{set CPL_ERROR_NULL_INPUT\, return -1, input image is NULL}
  @error{set CPL_ERROR_ILLEGAL_INPUT\, return -2, aThres is not positive}
  @error{set CPL_ERROR_ILLEGAL_INPUT\, return -3, aPasses is not positive}
  @error{set CPL_ERROR_ILLEGAL_INPUT\, return -4,
         aXBox is larger than the horizontal image size}
  @error{set CPL_ERROR_ILLEGAL_INPUT\, return -5,
         aYBox is larger than the horizontal image size}
  @error{print warning, size of a subframe is less than 100pix}
 */
/*----------------------------------------------------------------------------*/
int
muse_cosmics_dcr(muse_image *aImage, unsigned int aXBox, unsigned int aYBox,
                 unsigned int aPasses, float aThres)
{
  cpl_ensure(aImage, CPL_ERROR_NULL_INPUT, -1);
  cpl_ensure(aThres > 0, CPL_ERROR_ILLEGAL_INPUT, -2);
  cpl_ensure(aPasses > 0, CPL_ERROR_ILLEGAL_INPUT, -3);
  int nx = cpl_image_get_size_x(aImage->data),
      ny = cpl_image_get_size_y(aImage->data);
  cpl_ensure(aXBox <= (unsigned int)nx, CPL_ERROR_ILLEGAL_INPUT, -4);
  cpl_ensure(aYBox <= (unsigned int)ny, CPL_ERROR_ILLEGAL_INPUT, -5);

  if (aXBox * aYBox < 100) {
    cpl_msg_warning(__func__, "Boxes containing more than 100 pixels are "
                    "recommended for DCR!");
  }
  /* init for extra debug output based on environment variable */
  char *dodebug = getenv("MUSE_DEBUG_DCR");
  unsigned short debug = dodebug ? atoi(dodebug) : 0,
                 sfdebug = debug > 1;
  if (debug) {
    cpl_msg_debug(__func__, "Cosmic ray rejection using DCR: subframe %dx%d "
                  "(%d pixels/subframe), %d passes, threshold %.3f sigma)",
                  aXBox, aYBox, aXBox * aYBox, aPasses, aThres);
  }

  unsigned npass, ncr = 0;
  for (npass = 1; npass <= aPasses; npass++) {
    /* keep track of top/rightmost pixel covered by standard boxes */
    int imax = 0, jmax = 0;
    /* check detection in this pass */
    unsigned ncrpass = 0;

    /* run CR operation in standard boxes, offsetting by half a box size */
    unsigned i;
    for (i = 1; i <= nx - aXBox + 1; i+=aXBox/2) {
      unsigned j;
      for (j = 1; j <= ny - aYBox + 1; j+=aYBox/2) {
        if (i + aXBox > (unsigned)imax) {
          imax = i + aXBox;
        }
        if (j + aYBox > (unsigned)jmax) {
          jmax = j + aYBox;
        }
        if (debug > 1) {
          printf("subframe [%u:%u,%u:%u] (standard)\n",
                 i, i + aXBox, j, j + aYBox);
          fflush(stdout);
        }
        int npx = muse_cosmics_dcr_subframe(aImage, i, i + aXBox, j, j + aYBox,
                                            aThres, debug);
        ncrpass += npx;
        if (debug > 1 && npx) {
          printf("%8d affected pixels\n", npx);
          fflush(stdout);
        }
      } /* for j (box positions in y) */

      if (jmax < ny) {
        /* run CR operation in boxes at upper edge of the image */
        if (debug > 1) {
          printf("subframe [%u:%u,%u:%d] (upper)\n",
                 i, i + aXBox, ny - aYBox + 1, ny);
          fflush(stdout);
        }
        int npx = muse_cosmics_dcr_subframe(aImage, i, i + aXBox, ny - aYBox + 1, ny,
                                            aThres, debug);
        ncrpass += npx;
        if (debug > 1 && npx) {
          printf("%8d affected pixels\n", npx);
          fflush(stdout);
        }
      }
    } /* for i (box positions in x) */
    if (sfdebug) {
      printf("standard subframe coverage to [%d,%d] (image has %dx%d)\n",
             imax, jmax, nx, ny);
      fflush(stdout);
      sfdebug = 0; /* only need this output once */
    }

    if (imax < nx) {
      /* run CR operation in boxes at right edge of the image */
      unsigned j;
      for (j = 1; j <= ny - aYBox + 1; j+=aYBox/2) {
        if (debug > 1) {
          printf("subframe [%u:%d,%u:%u] (right)\n",
                 nx - aXBox + 1, nx, j, j + aYBox);
          fflush(stdout);
        }
        int npx = muse_cosmics_dcr_subframe(aImage, nx - aXBox + 1, nx, j, j + aYBox,
                                            aThres, debug);
        ncrpass += npx;
        if (debug > 1 && npx) {
          printf("%8d affected pixels\n", npx);
          fflush(stdout);
        }
      } /* for j (box positions in y) */
    } /* if imax < nx */

    if (imax < nx && jmax < ny) {
      /* run CR operation in box at upper right corner of the image */
      if (debug > 1) {
        printf("subframe [%u:%d,%u:%d] (corner)\n",
               nx - aXBox + 1, nx, ny - aXBox + 1, ny);
        fflush(stdout);
      }
      int npx = muse_cosmics_dcr_subframe(aImage, nx - aXBox + 1, nx, ny - aXBox + 1, ny,
                                          aThres, debug);
      ncrpass += npx;
      if (debug > 1 && npx) {
        printf("%8d affected pixels\n", npx);
        fflush(stdout);
      }
    } /* if imax < nx and jmax < ny */

    ncr += ncrpass;
    if (debug) {
      cpl_msg_debug(__func__, "%d (%d new) pixels found after pass %d", ncr,
                  ncrpass, npass);
    } /* if debug */

    if (!ncrpass) {
      break; /* stop early, if already the current pass didn't find new CRs */
    } /* if */
  } /* for npass (passes) */

  return ncr;
} /* muse_cosmics_dcr() */

/**@}*/
