/* $Id: casu_match.c,v 1.2 2015/08/07 13:06:54 jim Exp $
 *
 * This file is part of the CASU Pipeline utilities
 * Copyright (C) 2015 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

/*
 * $Author: jim $
 * $Date: 2015/08/07 13:06:54 $
 * $Revision: 1.2 $
 * $Name:  $
 */

/* Includes */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <math.h>

#include <cpl.h>
#include <string.h>
#include <strings.h>

#include "casu_mods.h"
#include "casu_stats.h"
#include "catalogue/casu_utils.h"

#define NX 2048
#define NY 2048
#define NGRIDMAX_XY 61
#define NGRIDMAX_STD 31

static cpl_table *casu_mkmstd_table(cpl_table *objtab, cpl_table *stdstab);

/**@{*/

/*---------------------------------------------------------------------------*/
/**
    \ingroup casu_modules
    \brief Match two lists of x,y coordinates from two tables to find the
    cartesian offset between them

    \par Name:
        casu_matchxy
    \par Purpose:
        Match two lists of x,y coordinates from two tables to find the
        cartesian offset between them
    \par Description:
        A program and a template table are given. The x,y coordinates of each
        are searched to find the offsets that maximises the number of matched
        objects. These offsets are returned along with the number of matches.
        The offsets are in the sense: xoffset = x_template - x_programme
    \par Language:
        C
    \param progtab
        The input table with the programme objects. Must have columns called
        X_coordinate and Y_coordinate.
    \param template
        The input table with the template objects. Must have columns called
        X_coordinate and Y_coordinate.
    \param srad
        A search radius in pixels. This helps to define the number of points
        that are used in the grid search
    \param xoffset
        The returned x offset
    \param yoffset
        The returned y offset
    \param nm
        The returned number of matches
    \param outtab
        The returned table of matched objects.
    \param status
        An input/output status that is the same as the returned values below.
    \retval CASU_OK
        If everything succeeded.
    \retval CASU_WARN
        If either table has no rows.
    \retval CASU_FATAL
        If either table is missing one of the required columns
    \par QC headers:
        None
    \par DRS headers:
        None
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

extern int casu_matchxy(cpl_table *progtab, cpl_table *template, float srad,
                        float *xoffset, float *yoffset, int *nm, 
                        cpl_table **outtab, int *status) {
    cpl_propertylist *p;
    float *xprog,*yprog,*xtemp,*ytemp,aveden,errlim,xoffbest,yoffbest,xoff;
    float yoff,x,y,*xoffs,*yoffs;
    const char *fctid = "casu_matchxy";
    int nprog,ntemp,ngrid,ngrid2,ibest,ig,jg,nmatch,k,jm;

    /* Inherited status */

    *xoffset = 0.0;
    *yoffset = 0.0;
    *nm = 0;
    *outtab = NULL;
    if (*status != CASU_OK)
        return(*status);

    /* Check the size of each of the tables */

    nprog = (int)cpl_table_get_nrow(progtab);
    ntemp = (int)cpl_table_get_nrow(template);
    if (nprog == 0) {
        cpl_msg_warning(fctid,"Program table has no rows");
        WARN_RETURN
    } else if (ntemp == 0) {
        cpl_msg_warning(fctid,"Template table has no rows");
        WARN_RETURN
    }

    /* First, sort the two tables by the Y coordinate */

    p = cpl_propertylist_new();
    cpl_propertylist_append_bool(p,"Y_coordinate",0);
    if (cpl_table_sort(progtab,p) != CPL_ERROR_NONE) {
        cpl_propertylist_delete(p);
        FATAL_ERROR
    }
    if (cpl_table_sort(template,p) != CPL_ERROR_NONE) {
        cpl_propertylist_delete(p);
        FATAL_ERROR
    }
    cpl_propertylist_delete(p);

    /* Get the x,y coordinates for each table */

    xprog = cpl_table_get_data_float(progtab,"X_coordinate");
    yprog = cpl_table_get_data_float(progtab,"Y_coordinate");
    xtemp = cpl_table_get_data_float(template,"X_coordinate");
    ytemp = cpl_table_get_data_float(template,"Y_coordinate");
    if (xprog == NULL || yprog == NULL || xtemp == NULL || ytemp == NULL)
        FATAL_ERROR

    /* Calculate the error limit and the number of grid points */

    aveden = (float)ntemp/(float)(NX*NY);
    errlim = 1.0/sqrt(4.0*CPL_MATH_PI*aveden);
    errlim = min(errlim,15.0);
    ngrid = (int)(srad/errlim);
    ngrid = (ngrid/2)*2 + 1;
    ngrid = max(5,min(NGRIDMAX_XY,ngrid));
    ngrid2 = ngrid/2 + 1;

    /* Now search for the best solution */

    ibest = 0;
    xoffbest = 0.0;
    yoffbest = 0.0;
    for (ig = -ngrid2; ig <= ngrid2; ig++) {
        xoff = (float)ig*errlim*CPL_MATH_SQRT2;
        for (jg = -ngrid2; jg <= ngrid2; jg++) {
            yoff = (float)jg*errlim*CPL_MATH_SQRT2;
            nmatch = 0;
            for (k = 0; k < nprog; k++) {
                x = xprog[k] + xoff;
                y = yprog[k] + yoff;
                if (casu_fndmatch(x,y,xtemp,ytemp,ntemp,errlim) > -1) 
                    nmatch++;
            }
            if (nmatch > ibest) {
                ibest = nmatch;
                xoffbest = xoff;
                yoffbest = yoff;
            }
        }
    }

    /* Now allocate some workspace so that you can calculate a good median
       coordinate difference */

    xoffs = cpl_malloc(nprog*sizeof(*xoffs));
    yoffs = cpl_malloc(nprog*sizeof(*yoffs));

    /* Now get the coordinate differences and find the medians */

    nmatch = 0;
    for (k = 0; k < nprog; k++) {
        x = xprog[k] + xoffbest;
        y = yprog[k] + yoffbest;
        jm = casu_fndmatch(x,y,xtemp,ytemp,ntemp,errlim);
        if (jm > -1) {
            xoffs[nmatch] = xtemp[jm] - xprog[k];
            yoffs[nmatch] = ytemp[jm] - yprog[k];
            nmatch++;
        }
    }
    if (nmatch > 0) {
        *xoffset = casu_med(xoffs,NULL,nmatch);
        *yoffset = casu_med(yoffs,NULL,nmatch);
    } else {
        *xoffset = 0.0;
        *yoffset = 0.0;
    }
    *nm = nmatch;

    /* Create the output table */

    *outtab = cpl_table_new((cpl_size)nprog);
    cpl_table_new_column(*outtab,"X_coordinate_1",CPL_TYPE_FLOAT);
    cpl_table_new_column(*outtab,"Y_coordinate_1",CPL_TYPE_FLOAT);
    cpl_table_new_column(*outtab,"X_coordinate_2",CPL_TYPE_FLOAT);
    cpl_table_new_column(*outtab,"Y_coordinate_2",CPL_TYPE_FLOAT);
    nmatch = 0;
    for (k = 0; k < nprog; k++) {
        x = xprog[k] + *xoffset;
        y = yprog[k] + *yoffset;
        jm = casu_fndmatch(x,y,xtemp,ytemp,ntemp,1.0);
        if (jm > -1) {
            cpl_table_set_float(*outtab,"X_coordinate_1",(cpl_size)nmatch,
                                xtemp[jm]);
            cpl_table_set_float(*outtab,"Y_coordinate_1",(cpl_size)nmatch,
                                ytemp[jm]);
            cpl_table_set_float(*outtab,"X_coordinate_2",(cpl_size)nmatch,
                                xprog[k]);
            cpl_table_set_float(*outtab,"Y_coordinate_2",(cpl_size)nmatch,
                                yprog[k]);
            nmatch++;
        }
    }
    cpl_table_set_size(*outtab,(cpl_size)nmatch);

    /* Tidy and exit */

    freespace(xoffs);
    freespace(yoffs);
    GOOD_STATUS
}

/*---------------------------------------------------------------------------*/
/**
    \ingroup casu_modules
    \brief Match object and standard star tables by their xy coordinates

    \par Name:
        casu_matchstds
    \par Purpose:
        Match object and standard star tables by their xy coordinates
    \par Description:
        An object table (nominally from casu_imcore) and a standard star
        table (nominally from casu_getstds) are given. The x,y coordinates 
        of each are searched to find the offsets that maximises the number 
        of matched objects. Objects that match are written to an output table
        with both sets of cartesian coordinates plus any ancillary information
        that might be in the standards table.
    \par Language:
        C
    \param objtab
        The input table with the programme objects. Must have columns called
        X_coordinate and Y_coordinate (as one gets from casu_imcore).
    \param stdstab
        The input table with the template objects. Must have columns called
        xpredict and ypredict (as one gets from casu_getstds).
    \param srad
        A search radius in pixels. This helps to define the number of points
        that are used in the grid search
    \param outtab
        The returned output table with both sets of cartesian coordinates plus
        any extra information that appears in the standards table.
    \param status
        An input/output status that is the same as the returned values below.
    \retval CASU_OK
        If everything succeeded.
    \retval CASU_WARN
        If either table has no rows.
    \retval CASU_FATAL
        If either table is missing one of the required columns
    \par QC headers:
        None
    \par DRS headers:
        None
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

extern int casu_matchstds(cpl_table *objtab, cpl_table *stdstab, float srad,
                          cpl_table **outtab, int *status) {
    const char *fctid = "casu_matchstds";
    char *colname;
    int nobj,nstd,ngrid,ngrid2,ibest,ig,jg,nmatch,k,*matches,jm,l,dont,null,k2;
    float *xstd,*ystd,*xobj,*yobj,aveden,errlim,xoffbest,yoffbest,*xoffs;
    float *yoffs,x,y,xx2,yy2,r2,xx1,yy1,r1,xoffmed,sigx,yoffmed,sigy,xoff,yoff;
    cpl_propertylist *p;
    cpl_table *mstds;
    cpl_array *colnames;

    /* Inherited status */

    *outtab = NULL;
    if (*status != CASU_OK)
        return(*status);

    /* Check the size of each of the tables */

    nobj = (int)cpl_table_get_nrow(objtab);
    nstd = (int)cpl_table_get_nrow(stdstab);
    if (nobj == 0) {
        cpl_msg_warning(fctid,"Object table has no rows");
        mstds = casu_mkmstd_table(objtab,stdstab);
        *outtab = cpl_table_extract_selected(mstds);
        cpl_table_delete(mstds);
        WARN_RETURN
    } else if (nstd == 0) {
        cpl_msg_warning(fctid,"Standards RA/DEC table has no rows");
        mstds = casu_mkmstd_table(objtab,stdstab);
        *outtab = cpl_table_extract_selected(mstds);
        cpl_table_delete(mstds);
        WARN_RETURN
    }

    /* First, sort the two tables by the Y coordinate */

    p = cpl_propertylist_new();
    cpl_propertylist_append_bool(p,"Y_coordinate",0);
    if (cpl_table_sort(objtab,p) != CPL_ERROR_NONE) {
        cpl_propertylist_delete(p);
        FATAL_ERROR
    }
    cpl_propertylist_erase(p,"Y_coordinate");
    cpl_propertylist_append_bool(p,"ypredict",0);
    if (cpl_table_sort(stdstab,p) != CPL_ERROR_NONE) {
        cpl_propertylist_delete(p);
        FATAL_ERROR
    }
    cpl_propertylist_delete(p);

    /* Get the x,y coordinates for each table */

    xobj = cpl_table_get_data_float(objtab,"X_coordinate");
    yobj = cpl_table_get_data_float(objtab,"Y_coordinate");
    xstd = cpl_table_get_data_float(stdstab,"xpredict");
    ystd = cpl_table_get_data_float(stdstab,"ypredict");
    if (xstd == NULL || ystd == NULL || xobj == NULL || yobj == NULL)
        FATAL_ERROR

    /* Calculate the error limit and the number of grid points */

    // This is the average density of stars in the image
    // FIXME it is incorrect to assume NX and NY are fixed
    aveden = (float)max(nstd,nobj)/(float)(NX*NY);
    // An initial estimate of a sensible error limit
    errlim = 1.0/sqrt(4.0*CPL_MATH_PI*aveden);
    errlim = min(errlim,15.0);
    // The grid size to achieve this
    ngrid = (int)(srad/errlim);
    ngrid = (ngrid/2)*2 + 1;
    // Constrain the number of grid points
    ngrid = max(5,min(NGRIDMAX_STD,ngrid));
    ngrid2 = ngrid/2 + 1;

    cpl_msg_info(fctid, "Search radius is %f pixels. Using a %d x %d grid, with error limit of %f", srad, ngrid, ngrid, errlim);

    /* Now search for the best solution */

    ibest = 0;
    xoffbest = 0.0;
    yoffbest = 0.0;
    for (ig = -ngrid2; ig <= ngrid2; ig++) {
        xoff = (float)ig*errlim*CPL_MATH_SQRT2;
        for (jg = -ngrid2; jg <= ngrid2; jg++) {
            yoff = (float)jg*errlim*CPL_MATH_SQRT2;
            nmatch = 0;
            for (k = 0; k < nobj; k++) {
                x = xobj[k] + xoff;
                y = yobj[k] + yoff;
                if (casu_fndmatch(x,y,xstd,ystd,nstd,errlim) > -1) 
                    nmatch++;
            }
            if (nmatch > ibest) {
                ibest = nmatch;
                xoffbest = xoff;
                yoffbest = yoff;
            }
        }
    }

    /* Now allocate some workspace so that you can calculate a good median
       coordinate difference */

    xoffs = cpl_malloc(nstd*sizeof(*xoffs));
    yoffs = cpl_malloc(nstd*sizeof(*yoffs));
    matches = cpl_malloc(nstd*sizeof(*matches));
    for (k = 0; k < nstd; k++)
        matches[k] = -1;

    /* Now get the best matches */

    nmatch = 0;
    for (k = 0; k < nstd; k++) {
        x = xstd[k] - xoffbest;
        y = ystd[k] - yoffbest;
        jm = casu_fndmatch(x,y,xobj,yobj,nobj,errlim);
        if (jm > -1) {
            dont = 0;
            xx2 = xobj[jm] - x;
            yy2 = yobj[jm] - y;
            r2 = sqrt(xx2*xx2 + yy2*yy2);
            for (l = 0; l < nstd; l++) {
                if (matches[l] == jm) {
                    xx1 = xobj[jm] - (xstd[l] - xoffbest);
                    yy1 = yobj[jm] - (ystd[l] - yoffbest);
                    r1 = sqrt(xx1*xx1 + yy1*yy1);
                    if (r2 < r1) 
                        matches[l] = -1;
                    else
                        dont = 1;
                    break;
                }
            }
            if (dont == 0)
                matches[k] = jm;
        }
    }

    /* Now get the coordinate difference for the best matches */

    for (k = 0; k < nstd; k++) {
        jm = matches[k];
        if (jm != -1) {
            xoffs[nmatch] = xobj[jm] - xstd[k];
            yoffs[nmatch] = yobj[jm] - ystd[k];
            nmatch++;
        }
    }
    if (nmatch == 0) {
        xoffmed = 0.0;
        sigx = 1.0;
        yoffmed = 0.0;
        sigy = 1.0;
    } else {
        casu_medmad(xoffs,NULL,nmatch,&xoffmed,&sigx);
        sigx *= 1.48;
        casu_medmad(yoffs,NULL,nmatch,&yoffmed,&sigy);
        sigy *= 1.48;
    }

    cpl_msg_info(fctid, "Median offset x=%f y=%f, sigx=%f sigy=%f", xoffmed, yoffmed, sigx, sigy);

    /* Now go through one final time with a reduced error box and get
       the final matches */

    errlim = 3.0*max(sigx,sigy);
    for (k = 0; k < nstd; k++)
        matches[k] = -1;
    for (k = 0; k < nstd; k++) {
        x = xstd[k] + xoffmed;
        y = ystd[k] + yoffmed;
        jm = casu_fndmatch(x,y,xobj,yobj,nobj,errlim);
        if (jm > -1) {
            dont = 0;
            xx2 = xobj[jm] - x;
            yy2 = yobj[jm] - y;
            r2 = sqrt(xx2*xx2 + yy2*yy2);
            for (l = 0; l < nstd; l++) {
                if (matches[l] == jm) {
                    xx1 = xobj[jm] - (xstd[l] + xoffmed);
                    yy1 = yobj[jm] - (ystd[l] + yoffmed);
                    r1 = sqrt(xx1*xx1 + yy1*yy1);
                    if (r2 < r1) 
                        matches[l] = -1;
                    else
                        dont = 1;
/*                  break; */
                }
            }
            if (dont == 0)
                matches[k] = jm;
        }
    }
    jm = matches[1];

    /* Make a copy of the standards table and add all the columns from the
       object catalogue to it. Ignore the RA and DEC columns in the catalogue
       as the standards table will already have these.*/

    mstds = cpl_table_duplicate(stdstab);
    colnames = cpl_table_get_column_names(objtab);
    ig = (int)cpl_array_get_size(colnames);
    for (k = 0; k < ig; k++) {
        colname = (char *)cpl_array_get_string(colnames,(cpl_size)k);
        if (strcasecmp(colname,"RA") && strcasecmp(colname,"DEC"))
            cpl_table_new_column(mstds,colname,
                                 cpl_table_get_column_type(objtab,colname));
    }
    cpl_table_unselect_all(mstds);
    cpl_array_delete(colnames);

    /* Now go through and find the matches. NB: the columns in objtab
       are created by casu_imcore which only produces integer, float and
       double columns, so we don't need to worry about any other type 
       when copying the columns to the matched standards catalogue. */

    colnames = cpl_table_get_column_names(objtab);
    ig = (int)cpl_array_get_size(colnames);
    for (k = 0; k < nstd; k++) {
        jm = matches[k];
        if (jm != -1) {
            for (k2 = 0; k2 < ig; k2++) {
                colname = (char *)cpl_array_get_string(colnames,
                                                             (cpl_size)k2);
                if (!strcasecmp(colname,"RA") || !strcasecmp(colname,"DEC")) 
                    continue;
                null = 0;
                switch (cpl_table_get_column_type(objtab,colname)) {
                case CPL_TYPE_INT:
                    cpl_table_set_int(mstds,colname,(cpl_size)k,
                                      cpl_table_get_int(objtab,colname,
                                                        (cpl_size)jm,&null));
                    break;
                case CPL_TYPE_FLOAT:
                    cpl_table_set_float(mstds,colname,(cpl_size)k,
                                        cpl_table_get_float(objtab,colname,
                                                            (cpl_size)jm,&null));
                    break;
                case CPL_TYPE_DOUBLE:
                    cpl_table_set_double(mstds,colname,(cpl_size)k,
                                         cpl_table_get_double(objtab,colname,
                                                              (cpl_size)jm,&null));
                    break;
                case CPL_TYPE_STRING:
                    cpl_table_set_string(mstds,colname,(cpl_size)k,
                                         cpl_table_get_string(objtab,colname,
                                                              (cpl_size)jm));
                    break;
                default:
                    cpl_msg_info(fctid, "Ignoring column %s of type %d", colname, cpl_table_get_column_type(objtab,colname));
                    break;
                }
            }
            cpl_table_select_row(mstds,(cpl_size)k);
        }
    }
    cpl_array_delete(colnames);

    /* Now extract the selected rows into the output table */

    *outtab = cpl_table_extract_selected(mstds);
    cpl_table_delete(mstds);

    /* Tidy up */

    freespace(matches);
    freespace(xoffs);
    freespace(yoffs);
    GOOD_STATUS
}

/*---------------------------------------------------------------------------*/
/**
    \par Name:
        casu_mkmstds_table
    \par Purpose:
        Make a matched standards table
    \par Description:
        A matched standards table is created by copying a standards table and
        then adding all the columns from an object table into the copy.
    \par Language:
        C
    \param objtab
        The input object table
    \param stdstab
        The input standards table
    \return 
        The output matched standards table
    \author
        Jim Lewis, CASU
*/
/*---------------------------------------------------------------------------*/
    
static cpl_table *casu_mkmstd_table(cpl_table *objtab, cpl_table *stdstab) {
    cpl_table *mstds;
    char *colname;
    cpl_array *colnames;
    cpl_size ncol,i;

    /* Copy the input standards table */

    mstds = cpl_table_duplicate(stdstab);

    /* Loop throught the object table columns and copy all of them over, 
       except for the RA and DEC tables */

    colnames = cpl_table_get_column_names((const cpl_table *)objtab);
    ncol = cpl_array_get_size(colnames);
    for (i = 0; i < ncol; i++) {
        colname = (char *)cpl_array_get_string(colnames,i);
        if (strcmp(colname,"RA") && strcmp(colname,"DEC"))
            cpl_table_new_column(mstds,colname,
                                 cpl_table_get_column_type(objtab,colname));
    }
    cpl_array_delete(colnames);
    cpl_table_unselect_all(mstds);

    /* Get out of here */

    return(mstds);
}
        
/**@}*/


/*

$Log: casu_match.c,v $
Revision 1.2  2015/08/07 13:06:54  jim
Fixed copyright to ESO

Revision 1.1.1.1  2015/06/12 10:44:32  jim
Initial import

Revision 1.5  2015/05/20 11:59:48  jim
Modified to remove calls to deprecated cpl_table_get_column_name

Revision 1.4  2015/02/14 12:32:06  jim
fixed typo

Revision 1.3  2015/01/29 11:51:56  jim
modified comments

Revision 1.2  2013/11/21 09:38:14  jim
detabbed

Revision 1.1.1.1  2013-08-27 12:07:48  jim
Imported


*/
