/***************************** LICENSE START ***********************************

 Copyright 2021 ECMWF and INPE. This software is distributed under the terms
 of the Apache License version 2.0. In applying this license, ECMWF does not
 waive the privileges and immunities granted to it by virtue of its status as
 an Intergovernmental Organization or submit itself to any jurisdiction.

 ***************************** LICENSE END *************************************/

#include <string>
#include <vector>

#include <math.h>
#include <ctype.h>
#include <float.h>
#include <string.h>
#include "macro.h"
#include "arith.h"
#include "cbufr.h"
#include "methods.h"
#include "MvFieldSet.h"
#include "MvPath.hpp"
#include "MvException.h"

// should include MvFieldSet.h, but it requires other header files...
//double interpolateValue( field* Grib, double lat, double lon );


// To do : filter(geo,area)
// Rms = sqrt(geo*geo/2)
// Mean
// distance(ab) = R*acos(sin(latA)*sin(latB)+cos(latA)*cos(latB)*cos(lonA-LonB))

//===========================================================================


// used in function GeoColumnsFunction
enum eGeoColumnsRequestType
{
    GCRT_ALL = 0,
    GCRT_VALUES
};


// helper function - if the Value is a vector, then set its indexed value; otherwise make it Nil
void setIndexedValueToNull(Value& val, int index)
{
    if (val.GetType() == tvector) {
        CVector* vec;
        val.GetValue(vec);
        vec->setIndexedValueToMissing(index);
    }
    else {
        val = Value();
    }
}

// helper function - if the Value is a vector, then set its indexed value; otherwise make it a Number
void setIndexedValueToNumber(Value& val, int index, double number)
{
    if (val.GetType() == tvector) {
        CVector* vec;
        val.GetValue(vec);
        vec->setIndexedValue(index, number);
    }
    else {
        val = Value(number);
    }
}


Value valueIndexFromValueArg(Value& arg, CGeopts* g, const char* fnName)
{
    int resultIndex = 0;

    if (arg.GetType() == tstring) {
        const char* valueName_c;
        arg.GetValue(valueName_c);
        std::string valueName(valueName_c);
        int vi = g->gpts().indexOfNamedValue(valueName);
        if (vi == -1) {
            // it's ok to pass this temporary string to CError because it caches it
            char msg[1024];
            sprintf(msg, "The %s function was supplied with a value column name of %s, but this does not exist.", fnName, valueName_c);
            return new CError(1, msg);
        }
        resultIndex = vi;
    }
    else {
        int indexOffset = 1 - Context::BaseIndex();  // = 0(Macro) or 1(Python)
        arg.GetValue(resultIndex);
        resultIndex += indexOffset;  // turn it into a 1-based offset
        if (resultIndex < 1 || resultIndex > (long)g->gpts().nValCols()) {
            // it's ok to pass this temporary string to CError because it caches it
            char msg[1024];
            int largestIndex = g->gpts().nValCols() - indexOffset;
            sprintf(msg, "set_values got a bad value column index (%d). Index for this data should be from %d to %d",
                    resultIndex - indexOffset, 1 - indexOffset, largestIndex);
            return new CError(1, msg);
        }
        resultIndex--;  // turn it into a 0-based offset
    }
    return Value(resultIndex);
}

CBufr::CBufr(request* s) :
    InPool(tbufr, s)
{
    r = clone_all_requests(s);
}

CBufr::~CBufr()
{
    const char* t = get_value(r, "TEMPORARY", 0);
    const char* p = get_value(r, "PATH", 0);
    if (t && p) {
        if (atoi(t)) {
            unlink(p);
        }
    }
    free_all_requests(r);
}

const char* CBufr::GetFileName()
{
    return get_value(r, "PATH", 0);
}

void CBufr::ToRequest(request*& s)
{
    // Initialise hidden parameters
    if (!get_value(r, "_CLASS", 0))
        set_value(r, "_CLASS", "BUFR");

    const char* path = get_value(r, "PATH", 0);
    if (!get_value(r, "_NAME", 0)) {
        if (path) {
            const char* name = mbasename(path);
            set_value(r, "_NAME", name);
        }
        else
            set_value(r, "_NAME", "bufr_data");
    }

    if (!get_value(r, "_PATH", 0)) {
        if (path) {
            const char* path1 = mdirname(path);
            set_value(r, "_PATH", path1);
        }
        else
            set_value(r, "_PATH", ".");
    }

    // Copy request
    s = r;
}

int CBufr::Write(FILE* f)
{
    return CopyFile(get_value(r, "PATH", 0), f);
}

CBufr::CBufr(const char* p, int temp) :
    InPool(tbufr)
{
    r = empty_request("BUFR");
    //set_value(r,"PATH","%s",p);
    set_value(r, "PATH", "%s", FullPathName(p).c_str());
    set_value(r, "TEMPORARY", "%d", temp);
}

//===========================================================================
class MergeBufrFunction : public Function
{
public:
    MergeBufrFunction(const char* n) :
        Function(n, 2, tbufr, tbufr) {}
    virtual Value Execute(int arity, Value* arg);
};

Value MergeBufrFunction::Execute(int, Value* arg)
{
    CBufr* a;
    CBufr* b;

    arg[0].GetValue(a);
    arg[1].GetValue(b);

    char* tmp = marstmp();

    char buf[2048];
    sprintf(buf, "cat %s %s > %s", a->GetFileName(),
            b->GetFileName(), tmp);
    system(buf);

    return Value(new CBufr(tmp, 1));
}

//===========================================================================
//===========================================================================


eGeoFormat geoTypeEnum(const char* type)
{
    if (type[0] == '\0') {
        return eGeoTraditional;
    }
    else {
        if (strcmp(type, "polar_vector") == 0)
            return eGeoVectorPolar;
        else if (strcmp(type, "xy_vector") == 0)
            return eGeoVectorXY;
        else if (strcmp(type, "xyv") == 0)
            return eGeoXYV;
        else if (strcmp(type, "ncols") == 0)
            return eGeoNCols;
        else if (strcmp(type, "standard") == 0)
            return eGeoTraditional;
        else
            return eGeoError;
    }
}


const char* CGeopts::GetFileName()
{
    return get_value(r, "PATH", 0);
}


void CGeopts::load(void)
{
    if (gpts_.count() > 0)
        return;

    if (r) {
        const char* path = get_value(r, "PATH", 0);
        gpts_.load(path);
    }
    //else
    //  is probably a geopoints structure in memory, or with zero count!
}

void CGeopts::unload(bool doFree)
{
    //	if( gpts.count() > 0 || (r == nullptr))
    //	  {
    if (doFree) {
        if (r == nullptr) {
            char* path = marstmp();
            gpts_.write(path);

            r = empty_request("GEOPOINTS");
            set_value(r, "TEMPORARY", "1");
            set_value(r, "PATH", "%s", path);
        }

        gpts_.unload();
    }
    //	  }
}

int CGeopts::Write(FILE* f)
{
    unload(true);
    return CopyFile(get_value(r, "PATH", 0), f);
}

CGeopts::CGeopts(request* s) :
    InPool(tgeopts, s)
{
    r = clone_all_requests(s);
}

CGeopts::CGeopts(const char* p, int temp) :
    InPool(tgeopts)
{
    r = empty_request("GEOPOINTS");
    //set_value(r,"PATH","%s",p);
    set_value(r, "PATH", "%s", FullPathName(p).c_str());
    set_value(r, "TEMPORARY", "%d", temp);
}

CGeopts::CGeopts(long count, int nvals, eGeoFormat efmt, bool init) :
    InPool(tgeopts),
    gpts_(count, nvals, efmt, init)
{
    r = nullptr;
}

CGeopts::CGeopts(long count, const MvGeoPointColumnInfo& colInfo, eGeoFormat efmt, bool init) :
    InPool(tgeopts),
    gpts_(count, colInfo, efmt, init)
{
    r = nullptr;
}


CGeopts::CGeopts(CGeopts* p) :
    InPool(tgeopts)
{
    r = nullptr;
    p->load();
    gpts_ = p->gpts_;
}

CGeopts::CGeopts(MvGeoPoints& gptsIn) :
    InPool(tgeopts)
{
    r     = nullptr;
    gpts_ = gptsIn;  // makes a copy, we should make this more efficient
}


CGeopts::CGeopts(CGeopts* p, fieldset* v, int n, bool nearest, bool nearestValid, bool storeLocs) :
    InPool(tgeopts)
{
    r = nullptr;
    p->load();
    eGeoFormat fmt        = (storeLocs)
                                ? eGeoNCols
                                : p->gpts_.format();
    eGeoCopyMode copyMode = (fmt == eGeoNCols)
                                ? eGeoCopyCoordinatesOnly
                                : eGeoCopyAll;


    gpts_.copy(p->gpts_, copyMode);  // make a copy of the input geopoints

    if (storeLocs) {
        gpts_.set_format(eGeoNCols, 3);
        gpts_.addColName("value");
        gpts_.addColName("nearest_latitude");
        gpts_.addColName("nearest_longitude");
    }


    field* g = get_field(v, n, expand_mem);

    // It needs to be dynamically allocated because it needs to
    // be destroyed before releasing the field (field* g). This is
    // because the destructor of MvField will try to restore the
    // previous memory status of the field, which is expand memory.
    auto* fld = new MvField(g);

    //-- get date/time/level metadata from the GRIB field
    MvDate base(fld->yyyymmddFoh());  //-- base (analysis) date
    double step  = fld->stepFoh();    //-- forecast step
    MvDate valid = base + step;       //-- valid date
    long dat     = valid.YyyyMmDd();
    long tim     = valid.Hour() * 100 + valid.Minute();
    double level = fld->level();

    n = 0;
    for (size_t i = 0; i < p->gpts_.count(); i++) {
        double x        = DBL_MAX;
        double near_lat = DBL_MAX;
        double near_lon = DBL_MAX;
        gpts_.set_rowIndex(i);  // all row-based ops will operate on this row
        // if lat/lon are missing, then it will be DBL_MAX
        if (!gpts_.latlon_missing() && gpts_.isLocationValid()) {  // if lat/lon are missing, then it will be DBL_MAX
            if (nearest) {
                if (storeLocs) {
                    MvGridPoint gridp = fld->nearestGridpointInfo(gpts_.lon_x(), gpts_.lat_y(), nearestValid);
                    near_lat          = gridp.loc_.latitude();
                    near_lon          = gridp.loc_.longitude();
                    x                 = gridp.value_;
                }
                else
                    x = fld->nearestGridpoint(gpts_.lon_x(), gpts_.lat_y(), nearestValid);
            }
            else {
                x = fld->interpolateAt(gpts_.lon_x(), gpts_.lat_y());
            }
        }

        if (x != DBL_MAX && x != mars.grib_missing_value) {
            gpts_.set_value(x);
        }
        else {
            // x will be DBL_MAX if there is no valid corresponding grid value.
            gpts_.set_value_missing();
        }

        gpts_.set_date(dat);
        gpts_.set_time(tim);
        gpts_.set_height(level);

        if (storeLocs) {
            gpts_.set_value(x);
            gpts_.set_ivalue(near_lat, 1);
            gpts_.set_ivalue(near_lon, 2);
        }

        n++;
    }

    delete fld;
    fld = 0;
    release_field(g);
}


CGeopts::~CGeopts()
{
    if (r) {
        const char* t = get_value(r, "TEMPORARY", 0);
        const char* p = get_value(r, "PATH", 0);
        if (t && p) {
            if (atoi(t)) {
                unlink(p);
            }
        }
    }
    //if(count) delete[] pts;
    free_all_requests(r);
}

void CGeopts::ToRequest(request*& s)
{
    unload(true);

    // Initialise hidden parameters
    if (!get_value(r, "_CLASS", 0))
        set_value(r, "_CLASS", "GEOPOINTS");

    const char* path = get_value(r, "PATH", 0);
    if (!get_value(r, "_NAME", 0)) {
        if (path) {
            const char* name = mbasename(path);
            set_value(r, "_NAME", name);
        }
        else
            set_value(r, "_NAME", "geopoints_data");
    }

    if (!get_value(r, "_PATH", 0)) {
        if (path) {
            const char* path1 = mdirname(path);
            set_value(r, "_PATH", path1);
        }
        else
            set_value(r, "_PATH", ".");
    }

    s = r;
}


// NOTE: this function could be put into the MvGeoPoints class if needed
bool CGeopts::doesMetadataMatch(const request* r)
{
    // for each parameter in the request, check that it exists in the metadata
    // and its value matches

    MvGeoPoints::metadata_t& md = gpts_.metadata();
    parameter* p                = r->params;
    while (p) {
        const char* key = p->name;
        value* v        = p->values;

        if (key) {
            // loop through all the given values for this key - succeed if any match
            bool doesThisKeyMatch = false;
            auto it               = md.find(key);

            if (it == md.end())  // key not found
                return false;

            while (v) {
                MvVariant val(v->name);
                if (it->second == val)  // key found and matches
                {
                    doesThisKeyMatch = true;
                    break;
                }

                v = v->next;
            }

            if (!doesThisKeyMatch)
                return false;
        }

        p = p->next;
    }

    return true;  // if we got to here, then everything matched
}


std::string CGeopts::SetColumnValues(eGeoColType col, Value& newVals, bool isList, bool isVector, bool valueIndexProvided, Value* valueIndex)
{
    CList* lst;
    CVector* vector;
    int inputCount;
    int valColIndex = 0;
    double val      = GEOPOINTS_MISSING_VALUE;  //-- used if list contains non-numbers

    if (col == eGeoColStnId && (!isList)) {
        return std::string("set_stnids: must supply a list of strings");
    }

    if ((col == eGeoColValue2) && (this->gpts().nValCols() < 2)) {
        return std::string("set_value2s: this geopoints only has one value column");
    }


    // value index provided? check for correctness
    if (valueIndexProvided) {
        Value indexVal = valueIndexFromValueArg(*valueIndex, this, "set_values");
        if (indexVal.GetType() == tnumber)
            indexVal.GetValue(valColIndex);
        else {
            CError* err;
            indexVal.GetValue(err);
            return std::string(err->Message());  // if not a number, then it is an Error object
        }
    }

    if (isList) {
        newVals.GetValue(lst);
        inputCount = lst->Count();
    }

    else if (isVector) {
        newVals.GetValue(vector);
        inputCount = vector->Count();
    }
    else {
        newVals.GetValue(val);
    }

    this->load();

    int cnt = this->Count();  //-- count is the count of geopoints
                              //-- unless a list/vector and list/vector is shorter
    if ((isList || isVector) && inputCount < cnt)
        cnt = inputCount;


    for (int i = 0; i < cnt; ++i) {
        double cur     = val;  //-- if list then cur is initialised as missing_value
        const char* id = "";
        Date d;
        bool isDate = false;
        gpts_.set_rowIndex(i);
        if (isList) {
            vtype listElementType = (*lst)[i].GetType();
            if (listElementType == tnumber) {
                (*lst)[i].GetValue(cur);  //-- if list and current list element is number
                if (is_number_nan_d(cur))  // nans can come from Python
                    cur = val;  // missing
            }
            else if (listElementType == tnil) {
            }  // id remains ""
            else if (listElementType == tdate) {
                (*lst)[i].GetValue(d);
                isDate = true;
            }
            else
                (*lst)[i].GetValue(id);
        }
        else if (isVector) {
            cur = (*vector)[i];  //-- if vector

            if (cur == VECTOR_MISSING_VALUE)  //-- convert missing value indicators
                cur = GEOPOINTS_MISSING_VALUE;
        }

        switch (col) {
            case eGeoColLat:
                gpts_.set_lat_y(cur);
                break;

            case eGeoColLon:
                gpts_.set_lon_x(cur);
                break;

            case eGeoColLevel:
                gpts_.set_height(cur);
                break;

            case eGeoColElevation:
                gpts_.set_elevation(cur);
                gpts_.hasElevations(true);  // record that we have a 'real' elevation
                break;

            case eGeoColDate:
                if (isDate)
                    gpts_.set_date(d.YyyyMmDd());
                else
                    gpts_.set_date((long)cur);
                break;

            case eGeoColTime:
                gpts_.set_time((long)cur);
                break;

            case eGeoColValue:
                gpts_.set_ivalue(cur, valColIndex);
                break;

            case eGeoColValue2:
                gpts_.set_value2(cur);
                break;

            case eGeoColStnId:
                gpts_.set_stnid(id);
                gpts_.hasStnIds(true);  // record that we have a 'real' stnid
                break;

            case eGeoColError:
                break;
        }
    }

    return std::string("");
}

//===========================================================================
void CGeopts::SetSubValue(Value& v, int arity, Value* arg)
{
    if (!Check(1, v, arity, arg, tany, 1, tstring))
        return;

    const char* indexName;
    arg[0].GetValue(indexName);
    eGeoColType coltype     = eGeoColValue;
    bool isList             = v.GetType() == tlist;
    bool isVector           = v.GetType() == tvector;
    bool valueIndexProvided = false;
    Value* valueIndex       = nullptr;


    // standard column names
    if (!strcmp(indexName, "latitude"))
        coltype = eGeoColLat;
    else if (!strcmp(indexName, "longitude"))
        coltype = eGeoColLon;
    else if (!strcmp(indexName, "level"))
        coltype = eGeoColLevel;
    else if (!strcmp(indexName, "time"))
        coltype = eGeoColTime;
    else if (!strcmp(indexName, "date"))
        coltype = eGeoColDate;
    else if (!strcmp(indexName, "value"))
        coltype = eGeoColValue;
    else if (!strcmp(indexName, "value2"))
        coltype = eGeoColValue2;
    else if (!strcmp(indexName, "stnid"))
        coltype = eGeoColStnId;
    else if (!strcmp(indexName, "elevation"))
        coltype = eGeoColElevation;
    else {
        // e.g. 't2' or some other value heading in the NCOLS format
        coltype            = eGeoColValue;
        valueIndexProvided = true;
        valueIndex         = new Value(strcache(indexName));
    }

    std::string errmsg = SetColumnValues(coltype, v, isList, isVector, valueIndexProvided, valueIndex);
    if (!errmsg.empty()) {
        Error(errmsg.c_str());
        return;
    }
}


//===========================================================================
class CountGeoFunction : public Function
{
public:
    CountGeoFunction(const char* n) :
        Function(n, 1, tgeopts) {}
    virtual Value Execute(int arity, Value* arg);
};

Value CountGeoFunction::Execute(int, Value* arg)
{
    CGeopts* g;
    arg[0].GetValue(g);
    g->load();
    return Value(g->Count());
}

//===========================================================================
class MaxGeoFunction : public Function
{
    int usemax;

public:
    MaxGeoFunction(const char* n, int m) :
        Function(n, 1, tgeopts),
        usemax(m) {}
    virtual Value Execute(int arity, Value* arg);
};

Value MaxGeoFunction::Execute(int, Value* arg)
{
    size_t i;
    size_t nFirstValid;
    double x;
    Value returnValue;
    CGeopts* g;
    arg[0].GetValue(g);
    g->load();
    if (g->Count() == 0)
        return Value();


    size_t ncols = g->gpts().nValColsForCompute();
    bool multi   = (ncols > 1);
    if (multi) {
        returnValue = Value(new CVector(ncols));
    }


    for (size_t c = 0; c < ncols; c++) {
        // Get the index of the first valid point. If -1 is returned,
        // then there are no valid points.

        nFirstValid = g->gpts().indexOfFirstValidPoint(c);

        if (nFirstValid == MvGeoPoints::INVALID_INDEX) {
            setIndexedValueToNull(returnValue, c);
            continue;
        }


        // store the first valid value, then replace it with each successive
        // greater or smaller value

        x = g->gpts().ivalue(nFirstValid, c);

        if (usemax) {
            for (i = nFirstValid + 1; i < g->Count(); i++) {
                double val = g->gpts().ivalue(i, c);
                if (!g->gpts().isMissingValue(val))  // only consider non-missing values
                    if (val > x)
                        x = val;
            }
        }
        else {
            for (i = nFirstValid + 1; i < g->Count(); i++) {
                double val = g->gpts().ivalue(i, c);
                if (!g->gpts().isMissingValue(val))  // only consider non-missing values
                    if (val < x)
                        x = val;
            }
        }
        setIndexedValueToNumber(returnValue, c, x);
    }

    return returnValue;
}
//-------------------------------------------------------------------
//-------------------------------------------------------------------
class FilterBoxFunction : public Function
{
public:
    FilterBoxFunction(const char* n) :
        Function(n) {}
    virtual Value Execute(int arity, Value* arg);
    virtual int ValidArguments(int arity, Value* arg);
};
Value FilterBoxFunction::Execute(int arity, Value* arg)
{
    CGeopts* g;
    arg[0].GetValue(g);

    double d[4];  //-- get area and normalize it with MvGeoBox
    if (arity == 2) {
        CList* l;
        arg[1].GetValue(l);
        for (int i = 0; i < 4; i++)
            (*l)[i].GetValue(d[i]);
    }
    else {
        for (int i = 0; i < 4; i++)
            arg[i + 1].GetValue(d[i]);
    }
    MvGeoBox myArea(d[0], d[1], d[2], d[3]);

    g->load();
    auto* x = new CGeopts(g);

    int n = 0;
    for (size_t i = 0; i < g->Count(); i++) {
        g->gpts().set_rowIndex(i);
        if (myArea.isInside(g->gpts().lat_y(), g->gpts().lon_x())) {
            x->gpts().copyRow(g->gpts(), i, n++);
        }
    }

    x->SetSize(n);

    g->unload();
    x->unload();

    return Value(x);
}
int FilterBoxFunction::ValidArguments(int arity, Value* arg)
{
    int i;
    CList* l;

    if (arity < 1)
        return false;
    if (arg[0].GetType() != tgeopts)
        return false;

    switch (arity) {
        case 5:
            for (i = 1; i < 5; i++)
                if (arg[i].GetType() != tnumber)
                    return false;
            break;
        case 2:
            switch (arg[1].GetType()) {
                case tlist:
                    arg[1].GetValue(l);
                    if (l->Count() != 4)
                        return false;
                    for (i = 0; i < 4; i++)
                        if ((*l)[i].GetType() != tnumber)
                            return false;
                    return true;

                default:
                    return false;
            }

            /* break; */

        case 1:
            break;

        default:
            return false;
    }
    return true;
}
//-------------------------------------------------------------------
//-------------------------------------------------------------------
class FilterLevelFunction : public Function
{
public:
    FilterLevelFunction(const char* n) :
        Function(n) {}
    virtual Value Execute(int arity, Value* arg);
    virtual int ValidArguments(int arity, Value* arg);
};
Value FilterLevelFunction::Execute(int arity, Value* arg)
{
    CGeopts* g;
    arg[0].GetValue(g);

    int i;
    double f[2];
    double& l1 = f[0];
    double& l2 = f[1];

    if (arity == 2) {
        if (arg[1].GetType() == tnumber) {
            double dd;
            arg[1].GetValue(dd);
            l2 = l1 = dd;
        }
        else {
            CList* l;
            arg[1].GetValue(l);
            double dd;
            for (i = 0; i < 2; i++) {
                (*l)[i].GetValue(dd);
                f[i] = dd;
            }
        }
    }
    else {
        for (i = 0; i < 2; i++) {
            double dd;
            arg[i + 1].GetValue(dd);
            f[i] = dd;
        }
    }

    g->load();

    auto* x = new CGeopts(g);

    int n = 0;

    for (size_t r = 0; r < g->Count(); r++) {
        if (l1 <= g->gpts().height(r) && g->gpts().height(r) <= l2) {
            x->gpts().copyRow(g->gpts(), r, n++);
        }
    }

    x->SetSize(n);

    g->unload();
    x->unload();

    return Value(x);
}
int FilterLevelFunction::ValidArguments(int arity, Value* arg)
{
    int i;
    CList* l;

    if (arity < 1)
        return false;
    if (arg[0].GetType() != tgeopts)
        return false;

    switch (arity) {
        case 3:
            for (i = 1; i < 3; i++)
                if (arg[i].GetType() != tnumber)
                    return false;
            break;
        case 2:
            switch (arg[1].GetType()) {
                case tlist:
                    arg[1].GetValue(l);
                    if (l->Count() != 2)
                        return false;
                    for (i = 0; i < 2; i++)
                        if ((*l)[i].GetType() != tnumber)
                            return false;
                    return true;
                case tnumber:
                    return true;

                default:
                    return false;
            }

            /* break; */

        case 1:
            break;

        default:
            return false;
    }
    return true;
}
//-------------------------------------------------------------------
//-------------------------------------------------------------------
class FilterDateFunction : public Function
{
public:
    FilterDateFunction(const char* n) :
        Function(n) {}
    virtual Value Execute(int arity, Value* arg);
    virtual int ValidArguments(int arity, Value* arg);
};
Value FilterDateFunction::Execute(int arity, Value* arg)
{
    CGeopts* g;
    arg[0].GetValue(g);

    Date d[4];

    Date& d1 = d[0];
    Date& d2 = d[1];

    int i;

    if (arity == 2) {
        if (arg[1].GetType() == tdate) {
            arg[1].GetValue(d1);
            d2 = d1;
        }
        else {
            CList* l;
            arg[1].GetValue(l);
            for (i = 0; i < 2; i++)
                (*l)[i].GetValue(d[i]);
        }
    }
    else {
        for (i = 0; i < 2; i++)
            arg[i + 1].GetValue(d[i]);
    }

    g->load();

    auto* x = new CGeopts(g);

    int n = 0;

    for (size_t r = 0; r < g->Count(); r++) {
        Date date(g->gpts().date(r), g->gpts().time(r));
        if (d1 <= date && date <= d2)
            x->gpts().copyRow(g->gpts(), r, n++);
    }
    x->SetSize(n);

    g->unload();
    x->unload();

    return Value(x);
}

int FilterDateFunction::ValidArguments(int arity, Value* arg)
{
    int i;
    CList* l;

    if (arity < 1)
        return false;

    if (arg[0].GetType() != tgeopts)
        return false;

    switch (arity) {
        case 3:
            for (i = 1; i < 3; i++)
                if (arg[i].GetType() != tdate)
                    return false;

            break;

        case 2:
            switch (arg[1].GetType()) {
                case tlist:
                    arg[1].GetValue(l);

                    if (l->Count() != 2)
                        return false;

                    for (i = 0; i < 2; i++)
                        if ((*l)[i].GetType() != tdate)
                            return false;

                    return true;

                case tdate:
                    return true;

                default:
                    return false;
            }

            /* break; */

        case 1:
            break;

        default:
            return false;
    }
    return true;
}

//-------------------------------------------------------------------

class GeoMaskFunction : public Function
{
public:
    GeoMaskFunction(const char* n) :
        Function(n, 2, tgeopts, tlist) { info = "Generates masks for geopoints"; }
    virtual Value Execute(int arity, Value* arg);
};

Value GeoMaskFunction::Execute(int /*arity*/, Value* arg)
{
    CGeopts* g;
    int i;

    double d[4];

    double& n = d[0];
    double& w = d[1];
    double& s = d[2];
    double& e = d[3];

    arg[0].GetValue(g);

    CList* l;
    arg[1].GetValue(l);
    for (i = 0; i < 4; i++)
        (*l)[i].GetValue(d[i]);

    while (w > e)
        w -= 360.0;
    MvGeoBox geoArea(n, w, s, e);

    g->load();
    auto* x = new CGeopts(g);

    size_t m = 0;
    for (size_t i = 0; i < g->Count(); i++) {
        g->gpts().set_rowIndex(i);
        bool inside = geoArea.isInside(g->gpts().lat_y(), g->gpts().lon_x());
        x->gpts().set_ivalue((inside) ? 1 : 0, m++, 0);
    }

    return Value(x);
}

//-------------------------------------------------------------------

class GeoPolyMaskFunction : public Function
{
public:
    GeoPolyMaskFunction(const char* n) :
        Function(n) { info = "Generates polygon masks for geopoints"; }
    int ValidArguments(int arity, Value* arg) override;
    Value Execute(int arity, Value* arg) override;

private:
    bool setMissing_{false};
    int adjustedArity_{0};
    std::string incorrectOption_;
};

int GeoPolyMaskFunction::ValidArguments(int arity, Value* arg)
{
    metview::checkStringOption("missing", arity, arg, setMissing_, incorrectOption_);
    adjustedArity_ = arity;

    if (arity == 3) {
        if (arg[0].GetType() != tgeopts) {
            return false;
        }
        if (arg[1].GetType() == tvector && arg[2].GetType() == tvector) {
            return true;
        }
        if (arg[1].GetType() == tlist && arg[2].GetType() == tlist) {
            return true;
        }
    }
    return false;
}

Value GeoPolyMaskFunction::Execute(int arity, Value* arg)
{
    CGeopts* g;
    std::vector<eckit::geometry::polygon::LonLatPolygon> polygons;

    arity = adjustedArity_;
    if (!incorrectOption_.empty()) {
        return Error("%s: if supplied, the option parameter must be 'missing'; it is '%s'", Name(), incorrectOption_.c_str());
    }

    arg[0].GetValue(g);

    // build polygons
    try {
        metview::buildPolygons(&arg[1], &arg[2], polygons);
    } catch (MvException &e) {
        return Error("%s: %s", Name(), e.what());
    }

    g->load();
    auto* x = new CGeopts(g);

    std::size_t m = 0;
    for (std::size_t i = 0; i < g->Count(); i++) {
        g->gpts().set_rowIndex(i);

        eckit::geometry::Point2 p(g->gpts().lon_x(), g->gpts().lat_y());
        bool inside = false;
        for (auto const& poly: polygons) {
            if (poly.contains(p)) {
                inside = true;
                break;
            }
        }
        if (setMissing_) {
            if (!inside) {
                x->gpts().set_value_missing(m++, 0);
            } else {
                x->gpts().set_ivalue(g->gpts().value(), m++, 0);
            }
        } else {
            x->gpts().set_ivalue((inside) ? 1 : 0, m++, 0);
        }
    }

    return Value(x);
}

//-------------------------------------------------------------------


/*******************************************************************************
 *
 * Class        : RemoveMissingValuesFunction : Function
 *
 * Description  : Macro function that creates a new set of geopoints based on
 *                an input set of geopoints but with all the missing values
 *                removed. Both value and value2 are considered.
 *
 ******************************************************************************/


class RemoveMissingValuesFunction : public Function
{
public:
    RemoveMissingValuesFunction(const char* n) :
        Function(n, 1, tgeopts)
    {
        info = "Copies a set of geopoints, removing missing values";
    }
    virtual Value Execute(int arity, Value* arg);
};


Value RemoveMissingValuesFunction::Execute(int, Value* arg)
{
    CGeopts* g;
    arg[0].GetValue(g);
    int n = 0;


    // load the input geopoints and create a new set based on those

    g->load();

    auto* x = new CGeopts(g);


    // for each input geopoint, only copy it over if it is not missing

    x->SetSize(g->Count());
    for (size_t i = 0; i < g->Count(); i++) {
        if (!g->gpts().any_missing(i)) {
            x->gpts().copyRow(g->gpts(), i, n++);
        }
    }
    x->SetSize(n);


    // unload the data and return the result

    g->unload();
    x->unload();

    return Value(x);
}


/*******************************************************************************
 * Class        : RemoveMissingLatLonsFunction : Function
 *
 * Description  : Macro function that creates a new set of geopoints based on
 *                an input set of geopoints but with all the missing lat/lons
 *                removed.
 ******************************************************************************/


class RemoveMissingLatLonsFunction : public Function
{
public:
    RemoveMissingLatLonsFunction(const char* n) :
        Function(n, 1, tgeopts)
    {
        info = "Copies a set of geopoints, removing missing lat/lons";
    }
    virtual Value Execute(int arity, Value* arg);
};


Value RemoveMissingLatLonsFunction::Execute(int, Value* arg)
{
    CGeopts* g;
    arg[0].GetValue(g);


    // load the input geopoints and create a new set based on those

    g->load();

    auto* x = new CGeopts(g);


    // for each input geopoint, only copy it over if it is not missing

    size_t n = 0;
    for (size_t i = 0; i < g->Count(); i++) {
        if (!(g->gpts().latlon_missing(i))) {
            x->gpts().copyRow(g->gpts(), i, n++);
        }
    }

    x->SetSize(n);

    // unload the data and return the result

    g->unload();
    x->unload();

    return Value(x);
}


//-------------------------------------------------------------------
//-------------------------------------------------------------------

/*******************************************************************************
 *
 * Function      : GeoIntBits
 *
 * Description   : For each geopoint, converts the value into an integer and then returns
 *                 the value of a given bit or bits, starting from the least
 *                 significant. For example, intbits (4, 1) = 0,
 *                 intbits (4, 2) = 0, intbits (4, 3) = 1, intbits (4, 4) = 0,
 *                 ... The user can also specify the number of bits to extract,
 *                 for example intbits (3, 1, 2): this extracts the first
 *                 2 bits from the number 3, returning the result of 3.
 *                 As another example, intbits (12, 3, 2) = 3.
 *                 This function was originally written to help with ODB
 *                 access.
 *
 ******************************************************************************/

class GeoIntBits : public Function
{
public:
    GeoIntBits(const char* n) :
        Function(n)
    {
        info = "Returns ranges of bits in a geopoints variable";
    }
    virtual Value Execute(int arity, Value* arg);
    virtual int ValidArguments(int arity, Value* arg);
};

int GeoIntBits::ValidArguments(int arity, Value* arg)
{
    if (arity != 2 && arity != 3)  // only accept 2- and 3-argument calls
        return false;

    if (arg[0].GetType() != tgeopts)  // first argument must be a geopoints
        return false;

    if (arg[1].GetType() != tnumber)  // second argument must be a number
        return false;

    if ((arity == 3) && (arg[2].GetType() != tnumber))
        return false;  // argument 3, if supplied, must be a number

    return true;
}

Value GeoIntBits::Execute(int arity, Value* arg)
{
    int value, bitindex;
    //int    integervalue;
    int bitmask = 0;
    int bits;
    int num_bits = 1;
    int max_bits_in_long;
    CGeopts *geo, *geonew;
    arg[0].GetValue(geo);
    arg[1].GetValue(bitindex);


    // how many bits can we hold in a 'long'?

    max_bits_in_long = sizeof(long) * 8;


    // get the number of bits we want to retrieve

    if (arity == 3) {
        arg[2].GetValue(num_bits);
    }


    // sensible numbers of bits only, please

    if (num_bits < 1) {
        return Error("The number of bits must be between 1 and %d inclusive.", max_bits_in_long);
    }


    // we cannot handle bit indexes outside of a sensible range

    if (bitindex < 1 || (bitindex + num_bits - 1) > max_bits_in_long) {
        return Error("The bit indexes must be between 1 and %d inclusive.", max_bits_in_long);
    }


    // load the input geopoints and create a new set based on those

    geo->load();

    geonew = new CGeopts(geo);


    // compute the bitmask we will use to isolate the desired bit(s)

    for (int i = 0; i < num_bits; i++) {
        bitmask = bitmask | (int)pow(2.0, (bitindex - 1 + i));
    }


    // isolate the desired bits using the bitmask

    for (size_t r = 0; r < geo->Count(); r++) {
        geo->gpts().set_rowIndex(r);
        geonew->gpts().set_rowIndex(r);
        value = (int)geo->gpts().value();  // take input value as an integer
        bits  = value & bitmask;           // mask off the bits we want

        // shift the result down. For instance, if the 5th bit is set, we
        // just want the result to be 1 when checking for that bit. If
        // we are checking bits 10 and 11, then we want a result between
        // 0 and 3 inclusive. So what we want is the position-independent
        // value of the bit(s).

        geonew->gpts().value(bits >> (bitindex - 1));
    }


    // unload the data and return the result

    geo->unload();
    geonew->unload();

    return Value(geonew);
}


//-------------------------------------------------------------------
//-------------------------------------------------------------------

class GeoGetFieldFunction : public Function
{
    eGeoColType fieldtype;
    bool deprecated;
    const char* fieldName;
    const char* newName;
    char expandedInfo[100];

public:
    GeoGetFieldFunction(const char* n, eGeoColType field, bool d, const char* fieldn, const char* nn = nullptr) :
        Function(n),
        fieldtype(field),
        deprecated(d),
        fieldName(fieldn),
        newName(nn)
    {
        sprintf(expandedInfo, "Returns a list/vector of %ss from the given geopoints.", fieldName);
        info = expandedInfo;  //"Returns a list of heights from the given geopoints.";
    }
    virtual int ValidArguments(int arity, Value* arg);
    virtual Value Execute(int arity, Value* arg);
};

int GeoGetFieldFunction::ValidArguments(int arity, Value* arg)
{
    if (arity != 1 && arity != 2)  // only accept 1- and 2-argument calls
        return false;

    if (arg[0].GetType() != tgeopts)  // first argument must be a geopoints
        return false;

    if (arity == 2 && (arg[1].GetType() != tnumber && arg[1].GetType() != tstring))  // second argument must be a number or a string
        return false;

    return true;
}

Value GeoGetFieldFunction::Execute(int arity, Value* arg)
{
    DeprecatedMessage(deprecated, "geopoints", newName);

    CGeopts* g;
    arg[0].GetValue(g);
    int vit = 0;  // 0-based index into the values array

    g->load();

    if (fieldtype == eGeoColValue) {
        if (arity == 2)  // value index/name supplied
        {
            Value indexVal = valueIndexFromValueArg(arg[1], g, "values");
            if (indexVal.GetType() == tnumber)
                indexVal.GetValue(vit);
            else
                return indexVal;  // if not a number, then it is an Error object
        }
    }
    else if (fieldtype == eGeoColValue2) {
        vit = 1;
    }

    if (fieldtype == eGeoColValue || fieldtype == eGeoColValue2) {
        if (g->Count() > 0) {
            size_t nvals = g->gpts().nValCols();
            if ((vit + 1) > (int)nvals) {
                return Error("The values function was supplied with an index of %d, but there are only %d values per point", vit + 1, nvals);
            }
        }
    }


    size_t i;
    switch (fieldtype)  // which column do we wish to extract?
    {
        case eGeoColLevel: {
            auto* v = new CVector(g->Count());

            for (i = 0; i < g->Count(); i++)
                v->setIndexedValue(i, g->gpts().height(i));

            return Value(v);
            break;
        }

        case eGeoColLat: {
            auto* v = new CVector(g->Count());

            for (i = 0; i < g->Count(); i++) {
                double value = g->gpts().lat_y(i);
                if (value != GEOPOINTS_MISSING_VALUE)
                    v->setIndexedValue(i, value);
                else
                    v->setIndexedValueToMissing(i);
            }

            return Value(v);
            break;
        }

        case eGeoColLon: {
            auto* v = new CVector(g->Count());

            for (i = 0; i < g->Count(); i++) {
                double value = g->gpts().lon_x(i);
                if (value != GEOPOINTS_MISSING_VALUE)
                    v->setIndexedValue(i, value);
                else
                    v->setIndexedValueToMissing(i);
            }

            return Value(v);
            break;
        }

        case eGeoColDate: {
            auto* v = new CList(g->Count());

            for (i = 0; i < g->Count(); i++) {
                Date d(g->gpts().date(i), g->gpts().time(i));
                (*v)[i] = Value(d);
            }

            return Value(v);
            break;
        }

        case eGeoColTime: {
            auto* v = new CVector(g->Count());

            for (i = 0; i < g->Count(); i++)
                v->setIndexedValue(i, g->gpts().time(i));

            return Value(v);
            break;
        }

        case eGeoColValue: {
            // strings are put into a list
            if (g->gpts().format() == eGeoString) {
                auto* v = new CList(g->Count());
                for (i = 0; i < g->Count(); i++) {
                    (*v)[i] = g->gpts().strValue(i).c_str();
                }
                return Value(v);
            }
            // numbers are put into a vector
            else {
                auto* v = new CVector(g->Count());
                for (i = 0; i < g->Count(); i++) {
                    double value = g->gpts().ivalue(i, vit);

                    if (value != GEOPOINTS_MISSING_VALUE)
                        v->setIndexedValue(i, value);
                    else
                        v->setIndexedValueToMissing(i);
                }
                return Value(v);
            }
            break;
        }

        case eGeoColValue2: {
            auto* v = new CVector(g->Count());

            for (i = 0; i < g->Count(); i++) {
                double value = g->gpts().direc(i);

                if (value != GEOPOINTS_MISSING_VALUE)
                    v->setIndexedValue(i, value);
                else
                    v->setIndexedValueToMissing(i);
            }

            return Value(v);
        }

        case eGeoColStnId: {
            auto* v = new CList(g->Count());
            for (i = 0; i < g->Count(); i++) {
                if (g->gpts().isStnIdEmpty(i))
                    (*v)[i] = Value();  // nil
                else
                    (*v)[i] = g->gpts().strValue(i).c_str();  // stnid is stored in strValue()
            }
            return Value(v);
            break;
        }

        case eGeoColElevation: {
            auto* v = new CVector(g->Count());

            for (i = 0; i < g->Count(); i++) {
                double value = g->gpts().elevation(i);

                if (value != GEOPOINTS_MISSING_VALUE)
                    v->setIndexedValue(i, value);
                else
                    v->setIndexedValueToMissing(i);
            }

            return Value(v);
            break;
        }

        default: {
            // should never get here
            return Error("Cannot extract this field  (%d) from geopoints", fieldtype);
        }
    }
}


//===========================================================================
class GeoFilterFunction : public Function
{
public:
    GeoFilterFunction(const char* n) :
        Function(n) {}
    virtual Value Execute(int arity, Value* arg);
    virtual int ValidArguments(int arity, Value* arg);
};

int GeoFilterFunction::ValidArguments(int arity, Value* arg)
{
    // geopts,geopts|tvector
    if (arity != 2)
        return false;
    if (arg[0].GetType() != tgeopts)
        return false;
    if (arg[1].GetType() != tgeopts && arg[1].GetType() != tvector)
        return false;
    return true;
}


Value GeoFilterFunction::Execute(int, Value* arg)
{
    CGeopts* x = nullptr;
    CGeopts* g;
    arg[0].GetValue(g);
    g->load();

    if (arg[1].GetType() == tgeopts) {
        CGeopts* h;
        arg[1].GetValue(h);

        h->load();
        if (g->Count() != h->Count())
            return Error("filter: the two geopoints have different sizes");

        x = new CGeopts(g);

        size_t n = 0;

        for (size_t i = 0; i < g->Count(); i++)
            if (h->gpts().value(i))
                x->gpts().copyRow(g->gpts(), i, n++);

        x->SetSize(n);
        h->unload();
    }
    else  // condition arg is tvector
    {
        CVector* v;
        arg[1].GetValue(v);

        if (g->Count() != (size_t)v->Count())
            return Error("filter: the geopoints and vector have different sizes");

        x = new CGeopts(g);

        size_t n = 0;

        for (size_t i = 0; i < g->Count(); i++)
            if ((*v)[i])
                x->gpts().copyRow(g->gpts(), i, n++);

        x->SetSize(n);
    }

    g->unload();
    x->unload();
    return Value(x);
}

//===========================================================================
class GeoVectorFunction : public Function
{
public:
    GeoVectorFunction(const char* n) :
        Function(n, 2, tgeopts, tgeopts) {}
    virtual Value Execute(int arity, Value* arg);
    virtual eGeoFormat vectorType() = 0;
};

class GeoPolarVectorFunction : public GeoVectorFunction
{
public:
    GeoPolarVectorFunction(const char* n) :
        GeoVectorFunction(n)
    {
        info = "Combines two 1-parameter geopoints variables into polar vector style";
    }
    virtual eGeoFormat vectorType() { return eGeoVectorPolar; }
};

class GeoXYVectorFunction : public GeoVectorFunction
{
public:
    GeoXYVectorFunction(const char* n) :
        GeoVectorFunction(n)
    {
        info = "Combines two 1-parameter geopoints variables into u/v vector style";
    }
    virtual eGeoFormat vectorType() { return eGeoVectorXY; }
};

Value GeoVectorFunction::Execute(int, Value* arg)
{
    CGeopts* g;
    CGeopts* h;
    arg[0].GetValue(g);
    arg[1].GetValue(h);

    g->load();
    h->load();
    if (g->Count() != h->Count())
        return Error("The two geopoints have different sizes");

    auto* cgpts = new CGeopts(g);
    cgpts->gpts().set_format(vectorType());
    cgpts->gpts().setColumnsForFormat();
    cgpts->gpts().valuesColumn(0) = g->gpts().valuesColumn(0);
    cgpts->gpts().valuesColumn(1) = h->gpts().valuesColumn(0);

    g->unload();
    h->unload();
    cgpts->unload();

    return Value(cgpts);
}

//===========================================================================
//                                                      this one interpolates
class GeoFromGribFunction1 : public Function
{
public:
    GeoFromGribFunction1(const char* n) :
        Function(n, 2, tgrib, tgeopts) {}
    virtual Value Execute(int arity, Value* arg);
};

Value GeoFromGribFunction1::Execute(int, Value* arg)
{
    fieldset* v;
    CGeopts* g;

    arg[0].GetValue(v);
    arg[1].GetValue(g);

    return Value(new CGeopts(g, v, 0, false, false, false));
}

//===========================================================================
//                                          this one looks for nearest points
// takes (fieldset, geopoints) as args
class GeoFromGribFunction2 : public Function
{
public:
    GeoFromGribFunction2(const char* n) :
        Function(n, 2, tgrib, tgeopts) {}
    virtual Value Execute(int arity, Value* arg);
};

Value GeoFromGribFunction2::Execute(int /*arity*/, Value* arg)
{
    fieldset* v;
    CGeopts* g;

    arg[0].GetValue(v);
    arg[1].GetValue(g);

    return Value(new CGeopts(g, v, 0, true, false, false));
}

// takes (fieldset, geopoints, string) as args
class GeoFromGribFunction3 : public Function
{
public:
    GeoFromGribFunction3(const char* n) :
        Function(n),
        validOptionString_("valid"),
        storeLocsOptionString_("store_locs"),
        valid_(false),
        storeLocs_(false) {}
    int ValidArguments(int arity, Value* arg);
    virtual Value Execute(int arity, Value* arg);

protected:
    std::string validOptionString_;
    std::string storeLocsOptionString_;
    bool valid_;
    bool storeLocs_;
};

int GeoFromGribFunction3::ValidArguments(int arity, Value* arg)
{
    if (arity < 3)
        return false;
    if (arg[0].GetType() != tgrib || arg[1].GetType() != tgeopts)
        return false;

    // the 3rd and optional 4th args are string parameters
    for (int a = 2; a <= arity - 1; a++) {
        if (arg[a].GetType() == tstring) {
            const char* tmpCh;
            arg[a].GetValue(tmpCh);
            if (tmpCh) {
                if (strcmp(tmpCh, validOptionString_.c_str()) == 0)
                    valid_ = true;
                if (strcmp(tmpCh, storeLocsOptionString_.c_str()) == 0)
                    storeLocs_ = true;
            }
            if (!valid_ && !storeLocs_)
                return false;
        }
        else {
            return false;
        }
    }
    return true;
}

Value GeoFromGribFunction3::Execute(int /*arity*/, Value* arg)
{
    fieldset* v;
    CGeopts* g;
    arg[0].GetValue(v);
    arg[1].GetValue(g);

    return Value(new CGeopts(g, v, 0, true, valid_, storeLocs_));
}


#if 0

//===========================================================================
//                                               use grib-to-geo app instead!
class GeoFromGribFunction3 : public Function {
public:
	GeoFromGribFunction3(char *n) : Function(n,1,tgrib) { };
	virtual Value Execute(int arity,Value *arg);
};

Value GeoFromGribFunction3::Execute(int,Value *arg)
{
	fieldset *v;

	arg[0].GetValue(v);

	return Value(new CGeopts(v,0));
}
#endif

//===========================================================================
//===========================================================================
class SubGeoFunction : public Function
{
    bool indexByNumber_;

public:
    SubGeoFunction(const char* n) :
        Function(n),
        indexByNumber_(true) {}
    virtual Value Execute(int arity, Value* arg);
    virtual int ValidArguments(int arity, Value* arg);
};

int SubGeoFunction::ValidArguments(int arity, Value* arg)
{
    if (arity != 2)
        return false;
    if (arg[0].GetType() == tgeopts) {
        if (arg[1].GetType() == tnumber) {
            indexByNumber_ = true;
            return true;
        }
        if (arg[1].GetType() == tstring) {
            indexByNumber_ = false;
            return true;
        }
    }
    return false;
}


Value SubGeoFunction::Execute(int, Value* arg)
{
    CGeopts* p;
    long nn;
    size_t n;

    arg[0].GetValue(p);
    p->load();

    request* r = empty_request(nullptr);

    // index by number? then return a definition with that single indexed geopoint
    if (indexByNumber_) {
        arg[1].GetValue(nn);
        n = nn;
        if (n < 0 || n > p->Count() - 1)
            return Error("Geopoints index is %ld, but should be from 0 to %ld", n, p->Count() - 1);


        MvGeoPoints& g = p->gpts();
        g.set_rowIndex(n);

        set_value(r, "latitude", "%g", g.lat_y());
        set_value(r, "longitude", "%g", g.lon_x());
        set_value(r, "height", "%g", g.height());
        set_value(r, "date", "%ld", g.date());
        set_value(r, "time", "%ld", g.time());
        set_value(r, "value", "%g", g.value());
        set_value(r, "value_missing", "%d", g.value_missing() ? 1 : 0);

        //-- valid only for geovectors
        double direc       = (g.nValCols() > 1) ? g.direc() : 0;
        bool direc_missing = (g.nValCols() > 1) ? g.direc_missing() : false;
        set_value(r, "value2", "%g", direc);
        set_value(r, "value2_missing", "%d", direc_missing ? 1 : 0);

        // for NCOLS, we add entries for all value columns
        if (g.format() == eGeoNCols) {
            set_value(r, "elevation", "%g", g.elevation());
            std::string stnid(g.strValue());
            if (g.isStnIdEmpty())
                unset_value(r, "stnid");
            else
                set_value(r, "stnid", "%s", g.strValue().c_str());

            size_t numValueCols = g.nValCols();
            for (size_t i = 0; i < numValueCols; i++) {
                std::string colName(g.valueColName(i));
                set_value(r, colName.c_str(), "%g", g.ivalue(i));
            }

            if (!g.isStnIdEmpty())
                set_value(r, "__strings", "%s", "stnid");
        }


        return Value(r);
    }
    // index by string? then return a vector/list of values for that named column
    // - technique is to call the existing function that will return that column
    else {
        const char* indexName;
        arg[1].GetValue(indexName);
        // standard column names
        if (!strcmp(indexName, "latitude") ||
            !strcmp(indexName, "longitude") ||
            !strcmp(indexName, "level") ||
            !strcmp(indexName, "elevation") ||
            !strcmp(indexName, "time") ||
            !strcmp(indexName, "date") ||
            !strcmp(indexName, "stnid") ||
            !strcmp(indexName, "value") ||
            !strcmp(indexName, "value2")) {
            char funcName[64];
            sprintf(funcName, "%ss", indexName);  // e.g. "level" -> "levels"
            char* fname = strcache(funcName);
            Owner()->Push(Value(p));
            Owner()->CallFunction(fname, 1);
            Value y = Owner()->Pop();
            return y;
        }
        // non-standard column, e.g. named parameter in NCOLS format, e.g. 'temperature'
        else {
            char* fname = strcache("values");
            Owner()->Push(Value(p));
            Owner()->Push(Value(indexName));
            Owner()->CallFunction(fname, 2);
            Value y = Owner()->Pop();
            return y;
        }
    }
}

//===========================================================================
// These are single-argument functions such as sin, cos, abs...

class GeoUnOp : public Function
{
    uniproc F_;

public:
    GeoUnOp(const char* n, uniproc f) :
        Function(n, 1, tgeopts) { F_ = f; }
    virtual Value Execute(int arity, Value* arg);
};

Value GeoUnOp::Execute(int, Value* arg)
{
    CGeopts* g;  // original  geopoints
    CGeopts* p;  // resultant geopoints

    arg[0].GetValue(g);
    g->load();

    p = new CGeopts(g);


    size_t ncols = g->gpts().nValColsForCompute();
    for (size_t c = 0; c < ncols; c++) {
        for (size_t r = 0; r < g->Count(); r++) {
            g->gpts().set_rowIndex(r);
            p->gpts().set_rowIndex(r);
            if (g->gpts().value_missing(c))      // if the input value is missing
                p->gpts().set_value_missing(c);  // set the output value to missing
            else                                 // otherwise, compute with (first) value
                p->gpts().set_ivalue(F_(g->gpts().ivalue(c)), r, c);
        }
    }


    return Value(p);
}

//===========================================================================
// Grib + Geopoint

class GribGeoBinOp : public Function
{
    binproc F_;

public:
    GribGeoBinOp(const char* n, binproc f) :
        Function(n) { F_ = f; }
    virtual Value Execute(int arity, Value* arg);
    virtual int ValidArguments(int arity, Value* arg);
};

Value GribGeoBinOp::Execute(int, Value* arg)
{
    fieldset* v;
    CGeopts* g;

    if (arg[0].GetType() == tgrib) {
        arg[0].GetValue(v);
        arg[1].GetValue(g);

        auto* p = new CGeopts(g, v, 0, false, false, false);


        // for each point,
        //   if either input value is missing
        //     set the output value to missing
        //   otherwise, compute

        for (size_t i = 0; i < g->Count(); i++) {
            g->gpts().set_rowIndex(i);
            p->gpts().set_rowIndex(i);
            if (g->gpts().value_missing() || p->gpts().value_missing())
                p->gpts().set_value_missing();
            else
                p->gpts().set_value(F_(p->gpts().value(), g->gpts().value()));
        }

        return Value(p);
    }
    else {
        arg[0].GetValue(g);
        arg[1].GetValue(v);

        auto* p = new CGeopts(g, v, 0, false, false, false);


        // for each point,
        //   if either input value is missing
        //     set the output value to missing
        //   otherwise, compute

        for (size_t i = 0; i < g->Count(); i++) {
            g->gpts().set_rowIndex(i);
            p->gpts().set_rowIndex(i);
            if (g->gpts().value_missing() || p->gpts().value_missing())
                p->gpts().set_value_missing();
            else
                p->gpts().set_value(F_(g->gpts().value(), p->gpts().value()));
        }

        return Value(p);
    }
}

int GribGeoBinOp::ValidArguments(int arity, Value* arg)
{
    if (arity != 2)
        return false;
    if (arg[0].GetType() == tgrib && arg[1].GetType() == tgeopts)
        return true;
    if (arg[1].GetType() == tgrib && arg[0].GetType() == tgeopts)
        return true;
    return false;
}

//===========================================================================
// Number + Geopoint

class NumGeoBinOp : public Function
{
    binproc F_;

public:
    NumGeoBinOp(const char* n, binproc f) :
        Function(n) { F_ = f; }
    virtual Value Execute(int arity, Value* arg);
    virtual int ValidArguments(int arity, Value* arg);
};

Value NumGeoBinOp::Execute(int, Value* arg)
{
    if (arg[0].GetType() == tnumber) {
        double d;
        CGeopts* g;

        arg[0].GetValue(d);
        arg[1].GetValue(g);

        auto* p = new CGeopts(g);

        // for each point,
        //   if the geopoint value is missing
        //     set the output value to missing
        //   otherwise, compute
        //   if XY geovector then compute also with the second value

        size_t ncols = g->gpts().nValColsForCompute();
        for (size_t c = 0; c < ncols; c++) {
            for (size_t r = 0; r < g->Count(); r++) {
                g->gpts().set_rowIndex(r);
                p->gpts().set_rowIndex(r);
                if (g->gpts().value_missing(c))
                    p->gpts().set_value_missing(c);
                else
                    p->gpts().set_ivalue(F_(d, g->gpts().ivalue(c)), r, c);
            }
        }

        return Value(p);
    }
    else {
        double d;
        CGeopts* g;

        arg[0].GetValue(g);
        arg[1].GetValue(d);

        auto* p = new CGeopts(g);


        // for each point,
        //   if the geopoint value is missing
        //     set the output value to missing
        //   otherwise, compute

        size_t ncols = g->gpts().nValColsForCompute();
        for (size_t c = 0; c < ncols; c++) {
            for (size_t r = 0; r < g->Count(); r++) {
                g->gpts().set_rowIndex(r);
                p->gpts().set_rowIndex(r);
                if (g->gpts().value_missing(c))
                    p->gpts().set_value_missing(c);
                else
                    p->gpts().set_ivalue(F_(g->gpts().ivalue(c), d), r, c);
            }
        }

        return Value(p);
    }
}

int NumGeoBinOp::ValidArguments(int arity, Value* arg)
{
    if (arity != 2)
        return false;
    if (arg[0].GetType() == tnumber && arg[1].GetType() == tgeopts)
        return true;
    if (arg[1].GetType() == tnumber && arg[0].GetType() == tgeopts)
        return true;
    return false;
}

//===========================================================================
// Geopoint + Geopoint

class GeoGeoBinOp : public Function
{
    binproc F_;

public:
    GeoGeoBinOp(const char* n, binproc f) :
        Function(n, 2, tgeopts, tgeopts) { F_ = f; }
    virtual Value Execute(int arity, Value* arg);
};

Value GeoGeoBinOp::Execute(int, Value* arg)
{
    CGeopts* g;
    CGeopts* h;

    arg[0].GetValue(g);
    arg[1].GetValue(h);

    g->load();
    h->load();

    if (g->Count() != h->Count())
        return Error("geopoints do not have the same number of points");

    if (g->gpts().nValColsForCompute() != h->gpts().nValColsForCompute())
        return Error("geopoints do not have the same number of value columns");


    auto* p = new CGeopts(g);

    size_t ncols = g->gpts().nValColsForCompute();
    for (size_t c = 0; c < ncols; c++) {
        for (size_t r = 0; r < g->Count(); r++) {
#if 0
			if(h->Lat(i) != g->Lat(r)
			|| h->Lon(i) != h->Lon(r))
				// Not yet, || h->Height() != g->Height())
				return Error("geopoints are not identical");
#endif
            g->gpts().set_rowIndex(r);
            h->gpts().set_rowIndex(r);
            p->gpts().set_rowIndex(r);
            if (g->gpts().value_missing(c) || h->gpts().value_missing(c))
                p->gpts().set_value_missing(c);
            else
                p->gpts().set_ivalue(F_(g->gpts().ivalue(c), h->gpts().ivalue(c)), r, c);
        }
    }

    return Value(p);
}

//===========================================================================
//===========================================================================

/* Commented out by Iain Russell, 17/12/2004.
   This class/function appears to not be called by anything, so it has been
   commented out to avoid confusion.

class GeoMulOp : public Function {

	typedef double (*binproc)(double,double);
	binproc F_;

public:
	GeoMulOp(char *n,binproc f) : Function(n) { F_ = f; }
	virtual Value Execute(int arity,Value *arg);
	virtual int  ValidArguments(int arity,Value *arg);
};

int GeoMulOp::ValidArguments(int arity,Value *arg)
{
	if(arity < 1) return false;
	for(int i = 0;i<arity;i++)
		if(arg[i].GetType() != tnumber)
			return false;
	return true;
}

Value GeoMulOp::Execute(int arity,Value *arg)
{
	double d;
	arg[0].GetValue(d);

	for(int i = 1 ; i < arity; i++)
	{
		double x;
		arg[i].GetValue(x);
		d = F_(d,x);
	}

	return Value(d);
}
 */


class MeanGeoFunction : public Function
{
    int compute_mean;

public:
    MeanGeoFunction(const char* n, int m) :
        Function(n, 1, tgeopts),
        compute_mean(m)
    {
        info = "Returns the sum or mean of the values in a geopoints variable";
    }
    virtual Value Execute(int arity, Value* arg);
};

Value MeanGeoFunction::Execute(int, Value* arg)
{
    CGeopts* g;
    double dSum   = 0.0;
    int nNumValid = 0;
    Value returnValue;

    arg[0].GetValue(g);

    g->load();

    size_t ncols = g->gpts().nValColsForCompute();
    bool multi   = (ncols > 1);
    if (multi) {
        returnValue = Value(new CVector(ncols));
    }

    for (size_t c = 0; c < ncols; c++) {
        dSum      = 0.0;
        nNumValid = 0;
        for (size_t i = 0; i < g->Count(); i++) {
            g->gpts().set_rowIndex(i);
            if (!(g->gpts().value_missing(c))) {
                dSum += g->gpts().ivalue(c);
                nNumValid++;
            }
        }

        if (nNumValid > 0) {
            double result;
            if (compute_mean)
                result = dSum / nNumValid;
            else
                result = dSum;
            setIndexedValueToNumber(returnValue, c, result);
        }
        else
            setIndexedValueToNull(returnValue, c);
    }

    g->unload();

    return returnValue;
}


//-------------------------------------------------------------------
class GeoDistanceFunction : public Function
{
public:
    GeoDistanceFunction(const char* n) :
        Function(n) {}
    virtual Value Execute(int arity, Value* arg);
    virtual int ValidArguments(int arity, Value* arg);
};

Value GeoDistanceFunction::Execute(int arity, Value* arg)
{
    CGeopts* g;
    arg[0].GetValue(g);

    double d[2];
    double lat, lon;
    int j;

    if (arity == 2) {
        CList* l;
        arg[1].GetValue(l);
        for (j = 0; j < 2; ++j)
            (*l)[j].GetValue(d[j]);
    }
    else {
        for (j = 0; j < 2; ++j)
            arg[j + 1].GetValue(d[j]);
    }

    MvLocation centre(d[0], d[1]);
    MvLocation point;

    g->load();

    auto* x = new CGeopts(g);

    for (size_t i = 0; i < g->Count(); i++) {
        lat         = g->gpts().lat_y(i);
        lon         = g->gpts().lon_x(i);
        double dist = GEOPOINTS_MISSING_VALUE;
        if (lat != GEOPOINTS_MISSING_VALUE && lon != GEOPOINTS_MISSING_VALUE) {
            point.set(lat, lon);
            dist = point.distanceInMeters(centre);
        }
        x->gpts().set_ivalue(dist, i, 0);
    }

    g->unload();
    x->unload();

    return Value(x);
}

int GeoDistanceFunction::ValidArguments(int arity, Value* arg)
{
    int i;
    CList* l;

    if (arity < 2)
        return false;

    if (arg[0].GetType() != tgeopts)
        return false;

    switch (arity) {
        case 3:
            for (i = 1; i < 3; ++i)
                if (arg[i].GetType() != tnumber)
                    return false;
            break;

        case 2:
            switch (arg[1].GetType()) {
                case tlist:
                    arg[1].GetValue(l);
                    if (l->Count() != 2)
                        return false;

                    for (i = 0; i < 2; ++i)
                        if ((*l)[i].GetType() != tnumber)
                            return false;

                    return true;

                default:
                    return false;
            }

        default:
            return false;
    }
    return true;
}


//-------------------------------------------------------------------

/*******************************************************************************
 *
 * Class        : GeoMergeFunction : Function
 *
 * Description  : Macro function that merges two geopoints variables. The second
 *                set is simply appended to the first.
 *
 ******************************************************************************/


class GeoMergeFunction : public Function
{
public:
    GeoMergeFunction(const char* n) :
        Function(n, 2, tgeopts, tgeopts)
    {
        info = "Merges 2 sets of geopoints";
    }
    virtual Value Execute(int arity, Value* arg);
};

Value GeoMergeFunction::Execute(int, Value* arg)
{
    CGeopts* g1;
    CGeopts* g2;
    CGeopts* gmerged;

    // load up our input geopoints

    arg[0].GetValue(g1);
    arg[1].GetValue(g2);

    g1->load();
    g2->load();


    // we can only merge two geopoints if they are in the same format

    if (g1->gpts().format() != g2->gpts().format())
        return Error("The two geopoints have different formats");


    // stricter test if NCOLS format - all the column information needs to be the same
    if (g1->gpts().format() == eGeoNCols && !g1->gpts().isCompatibleForMerging(g2->gpts()))
        return Error("Cannot merge two NCOLS geopoints that have different columns - please check carefully");


    // create a new set of geopoints large enough to hold both sets
    if (g1->gpts().format() == eGeoNCols)
        gmerged = new CGeopts(g1->Count() + g2->Count(), g1->gpts().columnInfo(), g1->gpts().format(), true);
    else
        gmerged = new CGeopts(g1->Count() + g2->Count(), g1->gpts().nValCols(), g1->gpts().format(), true);


    // copy the first set, followed by the second set

    gmerged->gpts().copyRows(g1->gpts(), 0, g1->Count(), 0);
    gmerged->gpts().copyRows(g2->gpts(), 0, g2->Count(), g1->Count());


    // unload the data and return the result
    g1->unload();
    g2->unload();
    gmerged->unload();

    return Value(gmerged);
}

//-------------------------------------------------------------------

/*******************************************************************************
 *
 * Class        : GeoCreateFunction : Function
 *
 * Description  : Macro function that creates a new geopoints variable
 *
 ******************************************************************************/


class GeoCreateFunction : public Function
{
public:
    GeoCreateFunction(const char* n) :
        Function(n)
    {
        info = "Creates a new set of geopoints";
    }
    virtual Value Execute(int arity, Value* arg);
    std::string removeTrailingS(const std::string& str);
};


// remove trailing 's' from string if there is one
std::string GeoCreateFunction::removeTrailingS(const std::string& str)
{
    if (str.size() > 1 && str[str.size() - 1] == 's') {
        std::string removed(str);
        removed.erase(removed.size() - 1);
        return removed;
    }
    return str;  // no trailing 's' - just return the original
}


Value GeoCreateFunction::Execute(int arity, Value* arg)
{
    CGeopts* gnew = nullptr;
    int n;
    int nv           = 1;
    int numUserCols  = 0;
    const char* type = "";
    eGeoFormat efmt  = eGeoTraditional;
    bool setNames    = false;


    // function signature: create_geo(nrows [,type [,list-of-colnames]])
    if (arg[0].GetType() == tnumber) {
        arg[0].GetValue(n);

        if (arity > 1) {
            if (arg[1].GetType() != tstring)
                return Error("create_geo: 2nd argument must be the type format name");
            arg[1].GetValue(type);
        }

        if (strcmp(type, "ncols") == 0 && arity >= 3)
            arg[2].GetValue(nv);

        if (strcmp(type, "ncols") == 0 && arity == 4)
            setNames = true;

        // set the format according to the supplied string
        efmt = geoTypeEnum(type);
        if (efmt == eGeoError)
            return Error("create_geo: format %s not recognised", type);

        // create a new set of geopoints
        gnew = new CGeopts(n, nv, efmt, true);

        if (setNames) {
            CList* varnames;
            arg[3].GetValue(varnames);
            gnew->gpts().clearValueColNames();
            for (int c = 0; c < varnames->Count(); c++) {
                Value& name = (*varnames)[c];
                if (name.GetType() == tstring) {
                    const char* cname;
                    name.GetValue(cname);
                    gnew->gpts().addColName(cname, true, true);
                }
                else {
                    return Error("create_geo: all columns names must be strings - number %d is not", c + 1);
                }
            }
        }
        else {
            // at least the first value column should have a name by default
            if (nv >= 1 && (efmt == eGeoNCols)) {
                gnew->gpts().addColName("value");
                gnew->gpts().fillValueColumnNames();
            }
        }


        return Value(gnew);
    }
    // function signature: create_geo(type: 'ncols', latitudes: lats, longitudes: lons, temperature: t, precip: p, ozone: oz)
    else {
        // first sanity check: ensure that there are an even number of arguments
        // - this is probably impossible, but just in case...
        if (arity % 2 != 0)
            return Error("create_geo: if no number given as first argument, must supply pairs of parameters - param:val");

        // tackle the params in 2 passes. 1st pass: check 'type'. 2nd pass: check everything else
        for (int pass = 1; pass <= 2; pass++) {
            // get each param/value pair
            for (int i = 0; i < arity; i += 2) {
                const char* param;

                // get the parameter name
                if (arg[0].GetType() != tstring)
                    return Error("create_geo: if no number given as first argument, must supply pairs of parameters - param:val");

                arg[i].GetValue(param);

                // get the parameter value
                Value& value = arg[i + 1];

                // special case: 'type', which we check for on the first pass so we set it first
                if (pass == 1) {
                    if (!strcmp(param, "type")) {
                        const char* typeName;
                        value.GetValue(typeName);
                        efmt = geoTypeEnum(typeName);
                        if (efmt == eGeoError)
                            return Error("create_geo: unrecognised type: %s", typeName);
                        continue;
                    }

                    // also on the first pass: count the number of user-defined value columns
                    else {
                        if (efmt == eGeoNCols &&
                            (MvGeoPoints::colTypeFromName(removeTrailingS(param), true) == eGeoColError || MvGeoPoints::colTypeFromName(removeTrailingS(param), true) == eGeoColValue)) {
                            numUserCols++;
                        }
                    }
                }
                else {
                    if (!strcmp(param, "type"))  // already handled in pass 1
                        continue;

                    // we accept things like latitude:nil - we just skip them
                    if (value.GetType() == tnil)
                        continue;

                    // first column to set? if so, create the CGeopts with that many values
                    if (gnew == nullptr) {
                        CVector* vec;
                        value.GetValue(vec);
                        if (numUserCols == 0)  // always have at least one value column
                            numUserCols = 1;
                        gnew = new CGeopts(vec->Count(), numUserCols, efmt, true);
                    }

                    // check the column name and prepare to set its values
                    std::string colName(param);
                    eGeoColType colType      = MvGeoPoints::colTypeFromName(removeTrailingS(colName), true);
                    bool isList              = (value.GetType() == tlist);
                    bool isVector            = (value.GetType() == tvector);
                    bool valueIndexProvided  = false;
                    Value* indexValue        = nullptr;
                    eGeoColType colTypeToSet = colType;

                    // column name not one of our standard ones, e.g. latitude?
                    if (colType == eGeoColError || colType == eGeoColValue) {
                        // user-defined column?
                        if (efmt == eGeoNCols) {
                            valueIndexProvided = true;
                            indexValue         = &arg[i];
                            gnew->gpts().addColName(colName);
                            colTypeToSet = eGeoColValue;
                        }
                        else if (colType == eGeoColError) {
                            return Error("create_geo: can only set standard column names, not %s unless creating ncols format", param);
                        }
                    }

                    std::string errmsg = gnew->SetColumnValues(colTypeToSet, value, isList, isVector, valueIndexProvided, indexValue);
                    if (!errmsg.empty())
                        return Error(errmsg.c_str());
                }
            }
        }
        if (gnew)
            return Value(gnew);
        else
            return Error("create_geo: must supply at least one column of data");
    }
}


//===========================================================================
class GeoTypeFunction : public Function
{
public:
    GeoTypeFunction(const char* n) :
        Function(n, 1, tgeopts)
    {
        info = "Returns the subtype of a geopoints variable";
    }
    virtual Value Execute(int arity, Value* arg);
};

Value GeoTypeFunction::Execute(int, Value* arg)
{
    CGeopts* g;
    arg[0].GetValue(g);

    g->load();

    switch (g->gpts().format()) {
        case eGeoTraditional:
            return Value("standard");
        case eGeoXYV:
            return Value("xyv");
        case eGeoVectorXY:
            return Value("xy_vector");
        case eGeoVectorPolar:
            return Value("polar_vector");
        case eGeoString:
            return Value("standard_string");
        case eGeoNCols:
            return Value("ncols");
        default:
            return Value("unknown");
    }
}

//===========================================================================
class GeoColumnsFunction : public Function
{
    eGeoColumnsRequestType type;

public:
    GeoColumnsFunction(const char* n, eGeoColumnsRequestType t) :
        Function(n, 1, tgeopts),
        type(t)
    {
        info = "Returns a list of column names for a geopoints variable";
    }
    virtual Value Execute(int arity, Value* arg);
};

Value GeoColumnsFunction::Execute(int, Value* arg)
{
    CGeopts* g;
    arg[0].GetValue(g);

    g->load();

    const std::vector<std::string> colNames = (type == GCRT_ALL)
                                                  ? g->gpts().usedColNames()
                                                  : g->gpts().valueColNames();

    size_t ncols = colNames.size();
    if (ncols > 0) {
        auto* l = new CList(ncols);
        for (size_t i = 0; i < ncols; i++) {
            (*l)[i] = colNames[i].c_str();
        }
        return Value(l);
    }
    else {
        return Value(new CList(0));  // empty list
    }
}

//===========================================================================
class GeoSortFunction : public Function
{
public:
    GeoSortFunction(const char* n) :
        Function(n, 1, tgeopts)
    {
        info = "Sorts the geopoints North to South and West to East";
    }
    virtual Value Execute(int arity, Value* arg);
};

Value GeoSortFunction::Execute(int, Value* arg)
{
    CGeopts* g;
    arg[0].GetValue(g);

    g->load();
    g->sort();

    auto* x = new CGeopts(g);


    // XXX it looks as though this function modifies the input geopoints
    // - is this intentional?

    g->unload();
    x->unload();

    return Value(x);
}

//===========================================================================
class GeoSubsampleFunction : public Function
{
public:
    GeoSubsampleFunction(const char* n) :
        Function(n, 2, tgeopts, tgeopts)
    {
        info = "Filters from the first geopoints variable points that exist in the second";
    }
    virtual Value Execute(int arity, Value* arg);
};

Value GeoSubsampleFunction::Execute(int, Value* arg)
{
    CGeopts* g1;
    arg[0].GetValue(g1);  //-- geopoints to filter

    CGeopts* g2;
    arg[1].GetValue(g2);  //-- geopoints to define subsample

    g1->load();
    g2->load();

    g1->sort();
    g2->sort();

    auto* gx = new CGeopts(g2);

    MvGeoPoints& gp1 = g1->gpts();
    MvGeoPoints& gp2 = g2->gpts();
    MvGeoPoints& gpx = gx->gpts();
    size_t count_g1  = g1->Count();
    size_t count_g2  = g2->Count();
    size_t rx        = 0;
    size_t rg1       = 0;
    for (size_t rg2 = 0; rg2 < count_g2; ++rg2) {
        // both sets of gpts are sorted geographically, so we can immediately
        // discard any points that are outside the bounds of our source points
        while (rg1 < count_g1 && gp1.latLonHeightBefore(gp2, rg1, rg2))
            ++rg1;

        if (rg1 < count_g1 && gp1.sameLocation(gp2, rg1, rg2)) {
            gpx.copyRow(gp1, rg1++, rx);  //-- g2 location exists in g1
        }
        else {
            gpx.copyRow(gp2, rg2, rx);
            gpx.set_value_missing(rx, 0);  //-- g2 location missing in g1
            if (gpx.have_direc())
                gpx.set_direc_missing(rx);
        }

        ++rx;
    }

    g1->unload();
    g2->unload();
    gx->unload();

    return Value(gx);
}

//===========================================================================
// Maybe this could/should be a functionality in visualisation...
//
class GeoOffsetFunction : public Function
{
    double dlat, dlon;

public:
    GeoOffsetFunction(const char* n) :
        Function(n)
    {
        info = "Offsets the locations of geopoints";
    }
    virtual int ValidArguments(int arity, Value* arg);
    virtual Value Execute(int arity, Value* arg);
};

int GeoOffsetFunction::ValidArguments(int arity, Value* arg)
{
    int i;
    //CList *l;

    if (arity < 2)
        return false;

    if (arg[0].GetType() != tgeopts)
        return false;

    switch (arity) {
        case 3:  //-- lat/lon given as two numbers
            if (arg[1].GetType() != tnumber)
                return false;
            if (arg[2].GetType() != tnumber)
                return false;

            arg[1].GetValue(dlat);
            arg[2].GetValue(dlon);

            break;

        case 2:  //-- lat/lon given as a list of 2 numbers
            if (arg[1].GetType() == tlist) {
                CList* lst;
                arg[1].GetValue(lst);
                if (lst->Count() != 2)
                    return false;

                for (i = 0; i < 2; ++i)
                    if ((*lst)[i].GetType() != tnumber)
                        return false;

                (*lst)[0].GetValue(dlat);
                (*lst)[1].GetValue(dlon);
            }
            else
                return false;

            break;

        default:
            return false;
    }

    return true;
}

Value GeoOffsetFunction::Execute(int, Value* arg)
{
    CGeopts* g;
    arg[0].GetValue(g);
    g->load();

    auto* x = new CGeopts(g);

    g->unload();

    x->offset(dlat, dlon);

    x->unload();

    return Value(x);
}

//===========================================================================
class GeoRemoveDuplicatesFunction : public Function
{
public:
    GeoRemoveDuplicatesFunction(const char* n) :
        Function(n, 1, tgeopts)
    {
        info = "Removes geopoint duplicates";
    }
    virtual Value Execute(int arity, Value* arg);
};

Value GeoRemoveDuplicatesFunction::Execute(int, Value* arg)
{
    CGeopts* g;
    arg[0].GetValue(g);

    g->load();
    g->removeDuplicates();

    auto* x = new CGeopts(g);

    // XXX it looks as though this function modifies the input geopoints
    // - is this intentional?


    g->unload();
    x->unload();

    return Value(x);
}

//===========================================================================
class GeoSetFunction : public Function
{
    eGeoColType col;
    bool isList;
    bool isVector;
    bool deprecated;
    bool valueIndexProvided;
    bool valueIndexIsString;
    int indexOfValueArg;
    const char* fieldName;
    const char* newName;
    char expandedInfo[100];

public:
    GeoSetFunction(const char* n, eGeoColType c, bool d, const char* fieldn, const char* nn = nullptr) :
        Function(n),
        col(c),
        deprecated(d),
        fieldName(fieldn),
        newName(nn)
    {
        sprintf(expandedInfo, "Sets the %s column in the geopoints variable.", fieldName);
        info = expandedInfo;
    }
    virtual int ValidArguments(int arity, Value* arg);
    virtual Value Execute(int arity, Value* arg);
};

int GeoSetFunction::ValidArguments(int arity, Value* arg)
{
    if (arity != 2 && arity != 3)  // only accept 2- or 3-argument calls
        return false;

    if (arg[0].GetType() != tgeopts)  // first argument must be a geopoints
        return false;

    // check for syntax  set_values(gpt, |5, 6, 8, 8, 1|)  vs set_values(gpt, index, |5, 6, 8, 8, 1|)
    indexOfValueArg    = 1;
    valueIndexProvided = false;
    valueIndexIsString = false;
    if (arity == 3) {
        valueIndexProvided = true;
        indexOfValueArg    = 2;
        if (arg[1].GetType() == tstring)
            valueIndexIsString = true;
        else if (arg[1].GetType() != tnumber)
            return false;
    }

    isList   = false;
    isVector = false;

    if (arg[indexOfValueArg].GetType() == tnumber)  // second argument can be a number, a list or a vector
    {
        return true;
    }

    if (arg[indexOfValueArg].GetType() == tlist)  // second argument can be a number, a list or a vector
    {
        isList = true;
        return true;
    }

    if (arg[indexOfValueArg].GetType() == tvector)  // second argument can be a number, a list or a vector
    {
        isVector = true;
        return true;
    }

    return false;
}

Value GeoSetFunction::Execute(int, Value* arg)
{
    DeprecatedMessage(deprecated, "geopoints", newName);

    CGeopts* g;
    arg[0].GetValue(g);  //-- get geopoints

    if (col == eGeoColStnId && (!isList)) {
        return Error("set_stnids: must supply a list of strings");
    }

    if ((col == eGeoColValue2) && (g->gpts().nValCols() < 2)) {
        return Error("set_value2s: this geopoints only has one value column");
    }


    // value index provided? check for correctness
    Value* indexValPtr = nullptr;
    Value indexVal;
    if (valueIndexProvided) {
        indexValPtr = &(arg[1]);
    }

    g->load();
    auto* x = new CGeopts(g);  //-- create a copy of input geopoints


    std::string errmsg = x->SetColumnValues(col, arg[indexOfValueArg], isList, isVector, valueIndexProvided, indexValPtr);

    if (!errmsg.empty())
        return Error(errmsg.c_str());

    g->unload();
    x->unload();

    return Value(x);
}


//===========================================================================
class GeoMetadataFunction : public Function
{
public:
    GeoMetadataFunction(const char* n) :
        Function(n, 1, tgeopts)
    {
        info = "Returns a metadata definition from the given geopoints.";
    }
    virtual Value Execute(int arity, Value* arg);
};

Value GeoMetadataFunction::Execute(int, Value* arg)
{
    request* r = empty_request(nullptr);

    CGeopts* g;
    arg[0].GetValue(g);
    g->load();

    MvGeoPoints::metadata_t& md = g->gpts().metadata();
    if (md.size() > 0) {
        for (auto& it : md)
            set_value(r, (it.first).c_str(), "%s", (it.second).toString().c_str());

        g->unload();
        return Value(r);
    }
    else {
        g->unload();
        request* r = empty_request(nullptr);  // return an empty request if no metadata
        return Value(r);
    }
}

//===========================================================================
class GeoSetMetadataFunction : public Function
{
public:
    GeoSetMetadataFunction(const char* n) :
        Function(n, 2, tgeopts, trequest)
    {
        info = "Sets a geopoints metadata from a definition";
    }
    virtual Value Execute(int arity, Value* arg);
};

Value GeoSetMetadataFunction::Execute(int, Value* arg)
{
    request* r = empty_request(nullptr);
    CGeopts* g;
    arg[0].GetValue(g);
    arg[1].GetValue(r);
    g->load();
    auto* newg = new CGeopts(g);  // create a clone of the input geopoints

    auto& md = newg->gpts().metadata();
    md.clear();

    // loop through the request/definition and add each element to the metadata map
    parameter* p = r->params;
    while (p) {
        const char* key = p->name;
        value* v        = p->values;
        md[key]         = MvVariant(v->name);
        p               = p->next;
    }

    g->unload();
    newg->unload();
    return Value(newg);
}

//===========================================================================
class GeoDbFunction : public Function
{
public:
    GeoDbFunction(const char* n) :
        Function(n)
    {
        info = "Returns a string of the database from the given geopoints.";
    }
    virtual int ValidArguments(int arity, Value* arg);
    virtual Value Execute(int arity, Value* arg);
};

int GeoDbFunction::ValidArguments(int arity, Value* arg)
{
    if (arity != 2 && arity != 3)  // only accept 2-argument calls
        return false;

    if (arg[0].GetType() != tgeopts)  // first argument must be a geopoints
        return false;

    if (arg[1].GetType() == tstring)  // second argument must be a string
    {
        if (arity == 3) {
            const char* cval;
            arg[1].GetValue(cval);
            std::string db_item = cval;

            if (arg[2].GetType() == tstring &&
                (db_item == "column" || db_item == "alias")) {
                return true;
            }
            return false;
        }

        return true;
    }

    return false;
}

Value GeoDbFunction::Execute(int, Value* arg)
{
    CGeopts* g;
    arg[0].GetValue(g);

    g->load();

    const char* cval;
    std::string db_item;
    arg[1].GetValue(cval);

    db_item = cval;

    if (db_item == "name") {
        return Value(g->gpts().dbSystem().c_str());
    }
    else if (db_item == "column") {
        const char* col;
        arg[2].GetValue(col);

        return Value(g->gpts().dbColumn(col).c_str());
    }
    else if (db_item == "alias") {
        const char* col;
        arg[2].GetValue(col);

        return Value(g->gpts().dbColumnAlias(col).c_str());
    }
    else if (db_item == "path") {
        return Value(g->gpts().dbPath().c_str());
    }
    else if (db_item == "query") {
        int n = g->gpts().dbQuery().size();

        auto* v = new CList(n);

        for (unsigned int i = 0; i < g->gpts().dbQuery().size(); i++) {
            (*v)[i] = g->gpts().dbQuery().at(i).c_str();
        }

        return Value(v);
    }

    return Value(" ");
}


//-------------------------------------------------------------------
//===========================================================================

static void install(Context* c)
{
    c->AddGlobal(new Variable("geo_missing_value", Value(GEOPOINTS_MISSING_VALUE)));

    c->AddFunction(new MergeBufrFunction("&"));
    c->AddFunction(new MergeBufrFunction("merge"));

    c->AddFunction(new CountGeoFunction("count"));
    c->AddFunction(new SubGeoFunction("[]"));

    c->AddFunction(new GeoMergeFunction("&"));
    c->AddFunction(new GeoMergeFunction("merge"));

    int i;

    // Binary op

    for (i = 0; BinOps[i].symb; i++)
        c->AddFunction(new GeoGeoBinOp(BinOps[i].symb, BinOps[i].proc));

    for (i = 0; BinOps[i].symb; i++)
        c->AddFunction(new NumGeoBinOp(BinOps[i].symb, BinOps[i].proc));

    for (i = 0; BinOps[i].symb; i++)
        c->AddFunction(new GribGeoBinOp(BinOps[i].symb, BinOps[i].proc));

    // Mult op as Binary op

    for (i = 0; MulOps[i].symb; i++)
        c->AddFunction(new GeoGeoBinOp(MulOps[i].symb, MulOps[i].proc));

    for (i = 0; MulOps[i].symb; i++)
        c->AddFunction(new NumGeoBinOp(MulOps[i].symb, MulOps[i].proc));

    for (i = 0; MulOps[i].symb; i++)
        c->AddFunction(new GribGeoBinOp(MulOps[i].symb, MulOps[i].proc));

    // Unary op

    for (i = 0; UniOps[i].symb; i++)
        c->AddFunction(new GeoUnOp(UniOps[i].symb, UniOps[i].proc));

    c->AddFunction(new GeoFromGribFunction1("interpolate"));
    c->AddFunction(new GeoFromGribFunction2("nearest_gridpoint"));
    c->AddFunction(new GeoFromGribFunction3("nearest_gridpoint"));
    //	c->AddFunction(new GeoFromGribFunction3("geopoints"));
    c->AddFunction(new GeoFilterFunction("filter"));
    c->AddFunction(new GeoSortFunction("geosort"));
    c->AddFunction(new GeoSubsampleFunction("subsample"));
    c->AddFunction(new GeoPolarVectorFunction("polar_vector"));
    c->AddFunction(new GeoXYVectorFunction("xy_vector"));
    c->AddFunction(new MaxGeoFunction("maxvalue", 1));
    c->AddFunction(new MaxGeoFunction("minvalue", 0));
    c->AddFunction(new MeanGeoFunction("mean", 1));
    c->AddFunction(new MeanGeoFunction("sum", 0));


    // 'new' versions of geopoints column extraction functions
    //                                       fn name      field    deprecated  name-of-field
    c->AddFunction(new GeoGetFieldFunction("dates", eGeoColDate, false, "date"));
    c->AddFunction(new GeoGetFieldFunction("times", eGeoColTime, false, "time"));
    c->AddFunction(new GeoGetFieldFunction("levels", eGeoColLevel, false, "height"));
    c->AddFunction(new GeoGetFieldFunction("elevations", eGeoColElevation, false, "elevation"));
    c->AddFunction(new GeoGetFieldFunction("latitudes", eGeoColLat, false, "latitude"));
    c->AddFunction(new GeoGetFieldFunction("longitudes", eGeoColLon, false, "longitude"));
    c->AddFunction(new GeoGetFieldFunction("values", eGeoColValue, false, "value"));
    c->AddFunction(new GeoGetFieldFunction("value2s", eGeoColValue2, false, "2nd value"));
    c->AddFunction(new GeoGetFieldFunction("stnids", eGeoColStnId, false, "station id"));

    // deprecated versions of the above
    //                                      fn name       field  deprecated  name-of-field, name-of-new-version
    c->AddFunction(new GeoGetFieldFunction("date", eGeoColDate, true, "date", "dates"));
    c->AddFunction(new GeoGetFieldFunction("level", eGeoColLevel, true, "height", "levels"));
    c->AddFunction(new GeoGetFieldFunction("latitude", eGeoColLat, true, "latitude", "latitudes"));
    c->AddFunction(new GeoGetFieldFunction("longitude", eGeoColLon, true, "longitude", "longitudes"));
    c->AddFunction(new GeoGetFieldFunction("value", eGeoColValue, true, "value", "values"));
    c->AddFunction(new GeoGetFieldFunction("value2", eGeoColValue2, true, "2nd value", "value2s"));


    // 'new' versions of geopoints column extraction functions
    //                                     fn name      field  deprecated  name-of-field
    c->AddFunction(new GeoSetFunction("set_latitudes", eGeoColLat, false, "latitude"));
    c->AddFunction(new GeoSetFunction("set_longitudes", eGeoColLon, false, "longitude"));
    c->AddFunction(new GeoSetFunction("set_levels", eGeoColLevel, false, "level"));
    c->AddFunction(new GeoSetFunction("set_elevations", eGeoColElevation, false, "elevation"));
    c->AddFunction(new GeoSetFunction("set_dates", eGeoColDate, false, "date"));
    c->AddFunction(new GeoSetFunction("set_times", eGeoColTime, false, "time"));
    c->AddFunction(new GeoSetFunction("set_values", eGeoColValue, false, "value"));
    c->AddFunction(new GeoSetFunction("set_value2s", eGeoColValue2, false, "value2"));
    c->AddFunction(new GeoSetFunction("set_stnids", eGeoColStnId, false, "stnid"));

    // deprecated versions of the above
    //                                    fn name       field  deprecated  name-of-field, name-of-new-version
    c->AddFunction(new GeoSetFunction("set_latitude", eGeoColLat, true, "latitude", "set_latitudes"));
    c->AddFunction(new GeoSetFunction("set_longitude", eGeoColLon, true, "longitude", "set_longitudes"));
    c->AddFunction(new GeoSetFunction("set_level", eGeoColLevel, true, "level", "set_levels"));
    c->AddFunction(new GeoSetFunction("set_date", eGeoColDate, true, "date", "set_dates"));
    c->AddFunction(new GeoSetFunction("set_time", eGeoColTime, true, "time", "set_times"));
    c->AddFunction(new GeoSetFunction("set_value", eGeoColValue, true, "value", "set_values"));
    c->AddFunction(new GeoSetFunction("set_value2", eGeoColValue2, true, "value2", "set_value2s"));


    c->AddFunction(new GeoDistanceFunction("distance"));
    c->AddFunction(new GeoCreateFunction("create_geo"));
    c->AddFunction(new GeoTypeFunction("dtype"));
    c->AddFunction(new GeoColumnsFunction("columns", GCRT_ALL));
    c->AddFunction(new GeoColumnsFunction("value_columns", GCRT_VALUES));


    c->AddFunction(new FilterBoxFunction("filter"));
    c->AddFunction(new FilterLevelFunction("filter"));
    c->AddFunction(new FilterDateFunction("filter"));
    c->AddFunction(new GeoMaskFunction("mask"));
    c->AddFunction(new GeoPolyMaskFunction("poly_mask"));

    c->AddFunction(new GeoOffsetFunction("offset"));
    c->AddFunction(new GeoRemoveDuplicatesFunction("remove_duplicates"));
    c->AddFunction(new RemoveMissingValuesFunction("remove_missing_values"));
    c->AddFunction(new RemoveMissingLatLonsFunction("remove_missing_latlons"));
    c->AddFunction(new GeoIntBits("intbits"));

    c->AddFunction(new GeoMetadataFunction("metadata"));
    c->AddFunction(new GeoSetMetadataFunction("set_metadata"));
    c->AddFunction(new GeoDbFunction("db_info"));
}

static Linkage linkage(install);
