/***************************** LICENSE START ***********************************

 Copyright 2012 ECMWF and INPE. This software is distributed under the terms
 of the Apache License version 2.0. In applying this license, ECMWF does not
 waive the privileges and immunities granted to it by virtue of its status as
 an Intergovernmental Organization or submit itself to any jurisdiction.

 ***************************** LICENSE END *************************************/

#include <assert.h>

#include <algorithm>
#include <cstring>
#include <map>
#include <string>
#include <utility>
#include <vector>

#include "macro.h"
#include "arith.h"

/*

  To do:  (o = to do, * = done)
  
    * allow nil & vector
    * allow vector & vector
    * vector op vector
    * vector op number
    * fun(vector)  , e.g. abs()
    * cater for missing values - vector_missing_value?
    * custom function: gridsetvals()
    * allow nil & number  -> vector
      - NO, nil & number -> number already
      - BUT nil & vector -> vector
    * allow vector & number
    * vector->list-of-numbers  ; tolist(vector)
    * bitmap(), nobitmap() functions
    * loop on vector elements (already available)
    * vector subsets, e.g. vector[1, 10]
    * list-of-numbers -> vector
    o set a grib array, e.g. grib_set_double_array
    o parse a string , e.g. '1,2,3' into a vector?
    o consider CSV format - list of vectors, or matrix?
    o consider row(matrix,1), col(matrix,1) to return vector

*/


/*
    A note about missing values in vectors.
    We use mars.grib_missing_value as the missing value indicator.
    This is because we will often be populating vectors with GRIB values,
    and if we use the same missing value indicator, then we don't need to
    perform a conversion from mars.grib_missing_value to vector_missing_value.
    VECTOR_MISSING_VALUE is defined to be mars.grib_missing_value, but changing
    this would not be sufficient, as some code will assume that it is the same as
    mars.grib_missing_value (for efficiency).
*/

enum eVectorSortType
{
    VSORT_VALUES,
    VSORT_INDICES
};


enum eVectorIndexType
{
    VINDEX_NUMBERS,
    VINDEX_VECTOR
};


// by default, vectors are doubles, ie 64-bit floats. This can be changed
// by the user via the vector_set_default_type() function.
CArray::ValuesType CVector::defaultValtype = CArray::VALUES_F64;
std::map<std::string, CArray::ValuesType> CArray::valtypes;


// helper function to allow multiple ways for the user to input numbers, but
// to always receive them as a vector

argtype numberListOrVectorArgAsVector(Context* Owner, Value& arg, CVector** result)
{
    // get the list of percentiles and put them into a vector if not already one
    vtype type = arg.GetType();
    switch (type) {
        case tvector: {
            arg.GetValue(*result);
            return tvector;
            break;
        }
        case tlist: {
            // call a Macro function to convert from a list to a vector
            char* vectorConstructorFn = strcache("vector");
            Owner->Push(arg);
            Owner->CallFunction(vectorConstructorFn, 1);
            Value y = Owner->Pop();
            y.GetValue(*result);
            if (*result) {
                (*result)->Attach();  // to avoid deletion when 'y' is deleted
            }
            return tlist;
            break;
        }
        case tnumber: {
            double d;
            arg.GetValue(d);
            (*result) = new CVector(1);
            (*result)->setIndexedValue(0, d);
            return tnumber;
            break;
        }
        default: {
            return terror;
        }
    }
}


CArray::ValuesType CArray::valuesTypeFromString(const std::string& s)
{
    auto it = CArray::valtypes.find(s);
    if (it != valtypes.end()) {
        return it->second;
    }
    else
        return CArray::VALUES_INVALID;
}

std::string CArray::stringFromValuesType(CArray::ValuesType type)
{
    auto it = CArray::valtypes.begin();
    while (it != CArray::valtypes.end()) {
        if (it->second == type)
            return it->first;
        it++;
    }
    return "NOT_FOUND";
}


/*******************************************************************************
 *
 * Series of functions that handle the float32/64 issues
 *
 ******************************************************************************/

void CArrayF64::allocateMemory(int numElements)
{
    size   = numElements;
    values = new double[numElements];
}

void CArrayF32::allocateMemory(int numElements)
{
    size   = numElements;
    values = new float[numElements];
}


void CArrayF64::setValuesToConstant(double c)
{
    for (int i = 0; i < size; i++)
        values[i] = c;
}

void CArrayF32::setValuesToConstant(double c)
{
    for (int i = 0; i < size; i++)
        values[i] = c;
}


// copying doubles into CVector
void CArrayF64::memorycopy(int targetIndex, const double* srcVals, int numVals)
{
    // trivial case - just copy
    std::memcpy(&values[targetIndex], srcVals, numVals * sizeof(double));
}

void CArrayF32::memorycopy(int targetIndex, const double* srcVals, int numVals)
{
    // need to iterate to copy array of doubles into array of floats
    int endIndex = targetIndex + numVals;
    for (int i = targetIndex; i < endIndex; i++)
        values[i] = *srcVals++;
}

// copying floats into CVector
void CArrayF64::memorycopy(int targetIndex, const float* srcVals, int numVals)
{
    // need to iterate to copy array of doubles into array of floats
    int endIndex = targetIndex + numVals;
    for (int i = targetIndex; i < endIndex; i++)
        values[i] = *srcVals++;
}

void CArrayF32::memorycopy(int targetIndex, const float* srcVals, int numVals)
{
    // trivial case - just copy
    std::memcpy(&values[targetIndex], srcVals, numVals * sizeof(float));
}

// disk I/O
size_t CArrayF64::writeValuesToFile(FILE* fp)
{
    size_t written = fwrite(values, sizeof(double), size, fp);
    return written;
}

size_t CArrayF32::writeValuesToFile(FILE* fp)
{
    size_t written = fwrite(values, sizeof(float), size, fp);
    return written;
}

size_t CArrayF64::readValuesFromFile(FILE* fp, int n)
{
    assert(n <= size);
    return fread(values, sizeof(double), n, fp);
}

size_t CArrayF32::readValuesFromFile(FILE* fp, int n)
{
    assert(n <= size);
    return fread(values, sizeof(float), n, fp);
}


void CArrayF64::clean()
{
    if (size > 0) {
        delete[] values;
        values = 0;
        size   = 0;
    }
}

void CArrayF32::clean()
{
    if (size > 0) {
        delete[] values;
        values = 0;
        size   = 0;
    }
}

void CArrayF64::resize(int newSize)
{
    auto* newValues = new double[newSize];
    std::memcpy(newValues, values, newSize * sizeof(double));
    delete[] values;
    values = newValues;
    size   = newSize;
    for (int i = size; i < newSize; i++)  // pad extra elements with zeros
        values[i] = 0.0;
}

void CArrayF32::resize(int newSize)
{
    auto* newValues = new float[newSize];
    std::memcpy(newValues, values, newSize * sizeof(float));
    delete[] values;
    values = newValues;
    size   = newSize;
    for (int i = size; i < newSize; i++)  // pad extra elements with zeros
        values[i] = 0.0;
}

void CArrayF64::applyNumBinProc(binproc f, double other, bool otherFirst)
{
    for (int i = 0; i < size; i++) {
        if (values[i] != VECTOR_MISSING_VALUE) {
            if (otherFirst)
                values[i] = f(values[i], other);
            else
                values[i] = f(other, values[i]);
        }
    }
    // no need to do something if missing value because we are working on
    // a copy of the original data (so the result will already be missing)
}

void CArrayF32::applyNumBinProc(binproc f, double other, bool otherFirst)
{
    for (int i = 0; i < size; i++) {
        if (values[i] != VECTOR_F32_MISSING_VALUE) {
            if (otherFirst)
                values[i] = f(values[i], other);
            else
                values[i] = f(other, values[i]);
        }
    }
    // no need to do something if missing value because we are working on
    // a copy of the original data (so the result will already be missing)
}


// writes the result into this
void CArrayF64::applyVectorBinProc(binproc f, CArray* v1, CArray* v2)
{
    assert(v1);
    assert(v2);
    assert(size == v1->count());
    assert(size == v2->count());

    if (v1->asDoubles(0) && v2->asDoubles(0))  // can we can an array of doubles from the other vectors?
    {
        auto* f1 = (CArrayF64*)v1;
        auto* f2 = (CArrayF64*)v2;
        for (int i = 0; i < size; i++) {
            if (f1->values[i] != VECTOR_MISSING_VALUE && f2->values[i] != VECTOR_MISSING_VALUE) {
                values[i] = f(f1->values[i], f2->values[i]);
            }
            else {
                values[i] = VECTOR_MISSING_VALUE;
            }
        }
    }
    else  // more generic case
    {
        for (int i = 0; i < size; i++) {
            if (!v1->isIndexedValueMissing(i) && !v2->isIndexedValueMissing(i)) {
                values[i] = f(v1->getIndexedValue(i), v2->getIndexedValue(i));
            }
            else {
                values[i] = VECTOR_MISSING_VALUE;  // we know the result is F64 so no virtual fn required
            }
        }
    }
}


// writes the result into this
void CArrayF32::applyVectorBinProc(binproc f, CArray* v1, CArray* v2)
{
    assert(v1);
    assert(v2);
    assert(size == v1->count());
    assert(size == v2->count());

    if (v1->asFloat32s(0) && v2->asFloat32s(0))  // can we can an array of float32s from the other vectors?
    {
        auto* f1 = (CArrayF32*)v1;
        auto* f2 = (CArrayF32*)v2;
        for (int i = 0; i < size; i++) {
            if (f1->values[i] != VECTOR_F32_MISSING_VALUE && f2->values[i] != VECTOR_F32_MISSING_VALUE) {
                values[i] = f(f1->values[i], f2->values[i]);
            }
            else {
                values[i] = VECTOR_F32_MISSING_VALUE;
            }
        }
    }
    else  // more generic case
    {
        for (int i = 0; i < size; i++) {
            if (!v2->isIndexedValueMissing(i) && !v2->isIndexedValueMissing(i)) {
                values[i] = f(v1->getIndexedValue(i), v2->getIndexedValue(i));
            }
            else {
                values[i] = VECTOR_MISSING_VALUE;  // we know the result is F32 so no virtual fn required
            }
        }
    }
}


void CArrayF64::applyVectorUniProc(uniproc f, CArray* v)
{
    assert(v);

    if (v->asDoubles(0))  // can we can an array of doubles from the other vector?
    {
        auto* f1 = (CArrayF64*)v;
        for (int i = 0; i < size; i++) {
            if (f1->values[i] != VECTOR_MISSING_VALUE)
                values[i] = f(f1->values[i]);
            else
                values[i] = VECTOR_MISSING_VALUE;
        }
    }
    else  // more generic case
    {
        for (int i = 0; i < size; i++) {
            if (!v->isIndexedValueMissing(i)) {
                values[i] = f(v->getIndexedValue(i));
            }
            else {
                values[i] = VECTOR_MISSING_VALUE;  // we know the result is F64 so no virtual fn required
            }
        }
    }
}

void CArrayF32::applyVectorUniProc(uniproc f, CArray* v)
{
    assert(v);

    if (v->asFloat32s(0))  // can we can an array of float32s from the other vector?
    {
        auto* f1 = (CArrayF32*)v;
        for (int i = 0; i < size; i++) {
            if (f1->values[i] != VECTOR_F32_MISSING_VALUE)
                values[i] = f(f1->values[i]);
            else
                values[i] = VECTOR_F32_MISSING_VALUE;
        }
    }
    else  // more generic case
    {
        for (int i = 0; i < size; i++) {
            if (!v->isIndexedValueMissing(i)) {
                values[i] = f(v->getIndexedValue(i));
            }
            else {
                values[i] = VECTOR_F32_MISSING_VALUE;  // we know the result is F32 so no virtual fn required
            }
        }
    }
}


void CArray::copyValues(int targetIndex, CArray* source, int sourceIndex, int numValues)
{
    switch (source->type()) {
        case CArray::VALUES_F64: {
            double* dp = source->asDoubles(sourceIndex);
            memorycopy(targetIndex, dp, numValues);
            break;
        }
        case CArray::VALUES_F32: {
            float* fp = source->asFloat32s(sourceIndex);
            memorycopy(targetIndex, fp, numValues);
            break;
        }
        default: {
            assert(false);
            break;
        }
    }
}


void CVector::init(CArray::ValuesType type)
{
    switch (type) {
        case CArray::VALUES_F64: {
            values = new CArrayF64();
            break;
        }
        case CArray::VALUES_F32: {
            values = new CArrayF32();
            break;
        }
        default: {
            assert("Bad valtype in CVector::init");
        }
    }
}

/*******************************************************************************
 *
 * Function      : CVector::CVector(int)
 *
 * Description   : Creates a new vector of given size and sets all elements
 *                 to 0.0 if zeroValues is true.
 *
 ******************************************************************************/

CVector::CVector(int n, bool zeroValues) :
    Content(tvector)
{
    init(CVector::defaultValtype);

    if (n < 0)
        Error("Vector size cannot be negative: %d", n);

    if (n)
        values->allocateMemory(n);

    if (zeroValues)
        values->setValuesToConstant(0.0);
}

/*******************************************************************************
 *
 * Function      : CVector::CVector(int)
 *
 * Description   : Creates a new vector of given size and sets all elements
 *                 to 0.0 if zeroValues is true.
 *
 ******************************************************************************/

CVector::CVector(std::vector<double>& v) :
    Content(tvector)
{
    init(CVector::defaultValtype);
    size_t n = v.size();

    if (n < 0)
        Error("Vector size cannot be negative: %d", n);

    if (n)
        values->allocateMemory(n);

    values->memorycopy(0, &v[0], n);
}


/*******************************************************************************
 *
 * Function      : CVector::CVector(double *, int)
 *
 * Description   : Creates a new vector of given size copies all elements
 *                 from the given buffer.
 *
 ******************************************************************************/

CVector::CVector(double* v, int n, CArray::ValuesType dtype) :
    Content(tvector)
{
    if (dtype == CArray::VALUES_INVALID)
        init(CVector::defaultValtype);
    else
        init(dtype);

    if (n < 0)
        Error("Vector size cannot be negative: %d", n);

    if (n) {
        values->allocateMemory(n);
        values->memorycopy(0, v, n);
    }
}

/*******************************************************************************
 *
 * Function      : CVector::CVector(float *, int)
 *
 * Description   : Creates a new vector of given size copies all elements
 *                 from the given buffer.
 *
 ******************************************************************************/

CVector::CVector(float* v, int n, CArray::ValuesType dtype) :
    Content(tvector)
{
    if (dtype == CArray::VALUES_INVALID)
        init(CVector::defaultValtype);
    else
        init(dtype);

    if (n < 0)
        Error("Vector size cannot be negative: %d", n);

    if (n) {
        values->allocateMemory(n);
        values->memorycopy(0, v, n);
    }
}

/*******************************************************************************
 *
 * Function      : CVector::CVector(request*)
 *
 * Description   : Creates a new vector from the given request, with size and
 *                 elements taken from the SIZE and VALUES parameters respectively.
 *
 ******************************************************************************/

CVector::CVector(request* r) :
    Content(tvector)
{
    const char* path;
    path = get_value(r, "PATH", 0); /* get the path to the storage file */
    Read(path);
}

CVector::CVector(const char* filename) :
    Content(tvector)
{
    Read(filename);
}

bool CVector::Read(const char* filename)
{
    char buf[20] = "";
    FILE* f;
    int read;

    f = fopen(filename, "r");

    if (!f) {
        Error("CVector::Read: unable to load file %s", filename);
    }
    else {
        fread(buf, sizeof(char), 14, f);
        buf[14] = '\0';

        if (strcmp(buf, "METVIEW_VECTOR")) {
            Error("CVector::Read: start of vector file should be METVIEW_VECTOR. Is: %s", buf);
        }

        // read the data type
        fread(buf, sizeof(char), 10, f);
        buf[7] = '\0';
        std::string typeString(buf);
        CArray::ValuesType valType = CArray::valuesTypeFromString(typeString);
        if (valType == CArray::VALUES_INVALID) {
            Error("read_vector_from_request: unsupported value type: '%s'", buf);
        }

        init(valType);


        /* read the actual array of values */
        int size;
        fread(&size, sizeof(int), 1, f); /* read the number of values */

        if (size < 0)
            Error("Vector size cannot be negative: %d", size);

        if (size) {
            values->allocateMemory(size);
            if (!values)
                Error("CVector::CVector: unable to get memory for %d elements", size);

            read = values->readValuesFromFile(f, size); /* read the values */

            if (read != size)
                Error("CVector::CVector: tried to write %d elements - managed %d.", size, read);
        }

        fread(buf, sizeof(char), 18, f);
        buf[18] = '\0';

        if (strcmp(buf, "METVIEW_VECTOR_END")) {
            Error("CVector::Read: end of vector file should be METVIEW_VECTOR_END. Is: %s", buf);
        }

        fclose(f);
    }

    return true;  // probably unncecessary because Error causes an abort
}


/*

	int err = read_vector_from_request(r, double** vec, int* length )

    const char* c = get_value( r, "SIZE", 0 );
    size = (fortint)atoi(c);

    if( size < 1 )
    {
        marslog( LOG_WARN, "Zero sized vector, forcing size to 1" );
        size = 1;
    }

    values = new double[ size ];

    for( int i=0; i<size; ++i )
    {
        //-- 'msetv' returns one value even if size 0
        c = get_value( r, "VALUES", i );
        values[i] = atof(c);
    }
}
*/


/*******************************************************************************
 *
 * Function      : CVector::Write
 *
 * Description   : Writes the vector to the given file handle.
 *
 ******************************************************************************/

int CVector::Write(FILE* f)
{
    size_t written;
    char typeBuffer[20];

    // ensure the string has 10 chars
    sprintf(typeBuffer, "%-10s", CArray::stringFromValuesType(values->type()).c_str());

    fprintf(f, "METVIEW_VECTOR");
    fprintf(f, "%s", typeBuffer);

    int size = values->count();
    fwrite(&size, sizeof(int), 1, f);  // write the number of values

    if (values) {
        written = values->writeValuesToFile(f);

        if ((signed)written != values->count()) {
            Error("Tried to write %d elements - managed %d.", values->count(), written);
        }
    }
    fprintf(f, "METVIEW_VECTOR_END");

    return ferror(f);
}


/*******************************************************************************
 *
 * Function      : CVector::Read
 *
 * Description   : Writes the vector to the given file handle.
 *
 ******************************************************************************/
/*
int CVector::Read(FILE *f)
{
	size_t read;
	int size;
	char buf[20];

	fread (buf, sizeof(char), 14, f);
	buf[14] = '\0';

	if (strcmp(buf, "METVIEW_VECTOR"))
	{
		Error("Start of vector file should be METVIEW_VECTOR.");
	}


	read = fread (&size,   sizeof(int),    1,    f);  // write the number of values
	written = fwrite (values,  sizeof(double), size, f);  // write the values

	if (written != size)
	{
		Error("Tried to write %d elements - managed %d.", size, written);
	}

	return ferror(f);
}
*/

/*******************************************************************************
 *
 * Function      : CVector::ToRequest
 *
 * Description   : Creates a request from the vector, setting SIZE and VALUES.
 *
 ******************************************************************************/

void CVector::ToRequest(request*& x)
{
    static request* r = 0;

    if (r == 0)
        r = empty_request("VECTOR");

    char* path = marstmp();
    FILE* f    = fopen(path, "w");

    if (f) {
        Write(f);
        fclose(f);
        set_value(r, "TEMPORARY", "1");
        set_value(r, "PATH", "%s", path);
        x = r;
        if (mars.debug)
            print_all_requests(r);
        //Attach(); // Temp fix: if someone want the request
    }
    else {
        Error("Failed to open temporary file for writing: %s", path);
    }


    /*	static request *r = 0;
	if(r == 0)
	  r = empty_request("VECTOR");

	set_value( r, "SIZE", "%d", size );
	set_value( r, "VALUES", "%g", size > 0 ? values[0] : 0 );
	for( int i = 1; i < size; ++i )
	{
		add_value( r, "VALUES", "%g", values[i] );
	}

	if (mars.debug)
			print_all_requests(r);
	x = r;*/
}


/*******************************************************************************
 *
 * Function      : CVector::CopyValues
 *
 * Description   : Copies a number of values from one vector into another.
 *                 Indexes are zero-based.
 *
 ******************************************************************************/

void CVector::CopyValues(int targetIndex, const CVector& source, int sourceIndex, int numValues)
{
    // basic error checks

    if (targetIndex + numValues > Count())
        Error("Cannot copy %d values into vector at position %d", numValues, targetIndex);

    if (sourceIndex + numValues > source.Count())
        Error("Cannot copy %d values from vector at position %d", numValues, sourceIndex);


    // perform the copy
    if (values && numValues > 0)
        values->copyValues(targetIndex, source.values, sourceIndex, numValues);
}

void CVector::CopyValuesFromDoubleArray(int targetIndex, const double* source, int /*sourceIndex*/, int numValues)
{
    if (targetIndex + numValues > Count())
        Error("Cannot copy %d values into vector at position %d", numValues, targetIndex);


    // perform the copy
    if (values && numValues > 0)
        values->memorycopy(targetIndex, source, numValues);
}


/*******************************************************************************
 *
 * Function      : CVector::Copy
 *
 * Description   : Copies one vector into another
 *
 ******************************************************************************/

void CVector::Copy(const CVector& v)
{
    CleanUp();
    init(v.valuesType());

    if (v.Count() > 0) {
        values->allocateMemory(v.Count());
        CopyValues(0, v, 0, v.Count());
    }
}


/*******************************************************************************
 *
 * Function      : CVector::Resize
 *
 * Description   : Resizes a vector to a given number of elements. For
 *                 efficiency, the strategy is that if we are reducing the size
 *                 of the vector by a small amount, we don't re-allocate & copy
 *                 but instead just set the size variable to the new size.
 *                 Otherwise, if we can make a decent memory saving, we will
 *                 reallocate.
 *
 ******************************************************************************/

void CVector::Resize(int newSize)
{
    const int memorySavingThreshold = 1024 * 1024;                             // we will resize if it will save us this much memory (bytes)
    const int numElementsThreshold  = memorySavingThreshold / sizeof(double);  // expressed as number of elements
    int size                        = Count();

    if (newSize == size)  // shortcut for the trivial case
        return;

    // need more memory, or need significantly less memory?
    if ((newSize > size) || ((size - newSize) > numElementsThreshold)) {
        values->resize(newSize);
    }
    values->setSize(newSize);
}


/*******************************************************************************
 *
 * Function      : CVector::Dump/Print
 *
 * Description   : Output functions
 *
 ******************************************************************************/

void CVector::Dump(int)
{
    std::cout << "<vector(" << Count() << ')';
}

void CVector::Print(void)
{
    std::cout << '|';
    for (int i = 0; i < Count(); i++) {
        if (!isIndexedValueMissing(i))
            std::cout << getIndexedValue(i);
        else
            std::cout << "x";

        if (i != Count() - 1)
            std::cout << ',';
    }
    std::cout << '|';
}


/*******************************************************************************
 *
 * Function      : CVector::SetSubValue
 *
 * Description   : Sets the value of a given element
 *
 ******************************************************************************/

void CVector::SetSubValue(Value& val, int arity, Value* arg)
{
    // Note: Check() returns 1 if ok, 0 if fail
    //       first argument is 1 if we want the function to abort on failure

    // assigning a number?

    if (Check(0, val, arity, arg, tnumber, 1, tnumber)) {
        int n;
        arg[0].GetValue(n);

        double d;
        val.GetValue(d);

        if (n < 1 || n > Count()) {
            Error("Vector index [%d] is out of range (vector is %d long)",
                  n, Count());
            return;
        }
        setIndexedValue(n - 1, d);
    }

    // assigning a vector? e.g. v1[i] = v2 will copy all elements
    // of v2 into v1 starting at position i.

    else if (Check(1, val, arity, arg, tvector, 1, tnumber)) {
        int n;
        arg[0].GetValue(n);

        CVector* v2;
        val.GetValue(v2);

        if (n < 1 || n + v2->Count() - 1 > Count()) {
            Error("Vector indexes from %d to %d are out of range (vector is %d long)",
                  n, n + v2->Count() - 1, Count());
            return;
        }

        CopyValues(n - 1, *v2, 0, v2->Count());
    }

    else  // arguments did not match
        return;
}


/*******************************************************************************
 *
 * Function      : CVector::IndexOfFirstValidValue
 *
 * Description   : Returns the index of the first valid value. If there are no
 *                 valid values, then returns -1.
 *
 ******************************************************************************/

int CVector::IndexOfFirstValidValue()
{
    for (int i = 0; i < Count(); i++) {
        if (!isIndexedValueMissing(i))
            return i;
    }

    // did not find one

    return -1;
}


/*******************************************************************************
 *
 * Function      : SortFnAscending
 *
 * Description   : Sort function ('less than') used in CVector::Sort
 *
 ******************************************************************************/

int SortFnAscending(const void* v1, const void* v2)
{
    double difference = (*((double*)v1) - *((double*)v2));

    // can't return the result directly because the difference between
    // two doubles could be outwith the range of an int

    if (difference == 0.0)
        return 0;
    else if (difference < 0.0)
        return -1;
    else
        return 1;
}

int SortFnAscendingF32(const void* v1, const void* v2)
{
    float difference = (*((float*)v1) - *((float*)v2));

    // can't return the result directly because the difference between
    // two doubles could be outwith the range of an int

    if (difference == 0.0)
        return 0;
    else if (difference < 0.0)
        return -1;
    else
        return 1;
}

/*******************************************************************************
 *
 * Function      : SortFnDescending
 *
 * Description   : Sort function ('greater than') used in CVector::Sort
 *
 ******************************************************************************/

int SortFnDescending(const void* v1, const void* v2)
{
    double difference = (*((double*)v2) - *((double*)v1));


    // can't return the result directly because the difference between
    // two doubles could be outwith the range of an int

    if (difference == 0.0)
        return 0;
    else if (difference < 0.0)
        return -1;
    else
        return 1;
}

int SortFnDescendingF32(const void* v1, const void* v2)
{
    float difference = (*((float*)v2) - *((float*)v1));


    // can't return the result directly because the difference between
    // two doubles could be outwith the range of an int

    if (difference == 0.0)
        return 0;
    else if (difference < 0.0)
        return -1;
    else
        return 1;
}


/*******************************************************************************
 *
 * Function      : CVector::Sort
 *
 * Description   : Sort the given vector according to the operation '<' or'>'
 *
 ******************************************************************************/

void CVector::Sort(char op)
{
    switch (valuesType()) {
        case CArray::VALUES_F64: {
            if (op == '<')
                qsort(ValuesAsDoubles(0), Count(), sizeof(double), SortFnAscending);
            else
                qsort(ValuesAsDoubles(0), Count(), sizeof(double), SortFnDescending);
            break;
        }

        case CArray::VALUES_F32: {
            if (op == '<')
                qsort(ValuesAsFloat32s(0), Count(), sizeof(float), SortFnAscendingF32);
            else
                qsort(ValuesAsFloat32s(0), Count(), sizeof(float), SortFnDescendingF32);
            break;
        }

        default: {
            assert(false);
            break;
        }
    }
}


/*******************************************************************************
 *
 * Function      : CVector::SortIndices
 *
 * Description   : Sort the given vector according to the operation '<' or'>'.
 *                 The result is a sorted set of indices, not the actual values.
 *
 ******************************************************************************/

using ValueIndexPair = std::pair<double, int>;  // type to hold a value and its index

// comparison function to pass to the std::sort routine
bool ComparePairsDescending(ValueIndexPair p1, ValueIndexPair p2)
{
    return (p1 > p2);
}

void CVector::SortIndices(char op)
{
    if (Count() <= 0)
        return;

    // in order to sort the array of values and keep the sorted
    // indexes, we create a vector of <value, index> pairs. When sorted,
    // the .first will be used as the first comparison key, then .second.
    // In this way, as long as the value is the first item in the pair,
    // we will get the correct sorting.

    std::vector<ValueIndexPair> valuesAndIndices;  // will hold a list of these pairs

    valuesAndIndices.reserve(Count());  // initialise the vector of pairs
    for (int i = 0; i < Count(); i++)   // populate it
    {
        valuesAndIndices.push_back(std::make_pair(getIndexedValue(i), i));
    }

    // perform the actual sort
    if (op == '<')
        std::sort(valuesAndIndices.begin(), valuesAndIndices.end());  // ascending is default
    else
        std::sort(valuesAndIndices.begin(), valuesAndIndices.end(), ComparePairsDescending);


    // set the values in the CVector to be the sorted indexes
    int baseIndex = Context::BaseIndex();
    for (int i = 0; i < Count(); i++)
        setIndexedValue(i, valuesAndIndices[i].second + baseIndex);  // Macro wants indexes to start at 1
}

/*******************************************************************************
 *
 * Function      : CVector::Replace
 *
 * Description   : Replaces instances of the given old value with the new value.
 *
 ******************************************************************************/

void CVector::Replace(double dold, double dnew)
{
    if (!values)
        return;

    // we need a specical case if dealing with NaNs because the replace() function
    // will not find them
    int n = Count();
    if (is_number_nan_d(dold))
        std::replace_if(ValuesAsDoubles(0), ValuesAsDoubles(n), is_number_nan_d, dnew);
    else
        std::replace(ValuesAsDoubles(0), ValuesAsDoubles(n), dold, dnew);
}

void CVector::Replace(float fold, float fnew)
{
    if (!values)
        return;

    // we need a specical case if dealing with NaNs because the replace() function
    // will not find them
    int n = Count();
    if (is_number_nan_f(fold))
        std::replace_if(ValuesAsFloat32s(0), ValuesAsFloat32s(n), is_number_nan_f, fnew);
    else
        std::replace(ValuesAsFloat32s(0), ValuesAsFloat32s(n), fold, fnew);
}


/*******************************************************************************
 *
 * Function      : CVector::Percentile
 *
 * Description   : Computes the given set of percentiles on the current Vector
 *                 of data, given the specified interpolation method for when
 *                 the resulting index falls between two elements. The result
 *                 is a vector - one element for each percentile computed.
 *
 ******************************************************************************/

void CVector::Percentile(CVector& percs, PercentileInterpolationType /*interp*/, CVector& result)
{
    if (!values)
        return;

    Sort('<');  // sort into ascending order

    for (int pi = 0; pi < percs.Count(); pi++) {
        double fractionalIndex = (percs[pi] * (Count() + 1) * 0.01) - 1.0;  // 0.01 to convert from percent to fraction, -1 for C indexing
        if (fractionalIndex < 0.0)
            fractionalIndex = 0.0;
        else if (fractionalIndex > (Count() - 1))
            fractionalIndex = Count() - 1;
        int nearestIndex  = (int)(fractionalIndex + 0.5);  // nearest neighbour; may add more modes
        double foundValue = getIndexedValue(nearestIndex);
        result.setIndexedValue(pi, foundValue);
    }
}


//=============================================================================

/*
class VectorProductFunction : public Function {
public:
	VectorProductFunction(char *n) : Function(n,2,tvector,tvector) 
	{info = "Vector product";};
	virtual Value Execute(int arity,Value *arg);
};


Value VectorProductFunction::Execute(int ,Value *arg)
{
	CVector *a;
	CVector *b;
	CVector *c;

	arg[0].GetValue(a);
	arg[1].GetValue(b);

	if(a->Count() != b->Count())
		return Error("Cannot multiply vectors of different sizes: %d <> %d",
			a->Count(),b->Count());

	c = new CVector(a->Count());

	for(int i=0;i<a->Count();i++)
		;
		
	return Value(c);
}
*/

/*******************************************************************************
 *
 * Function      : Vector1Function
 *
 * Description   : Creates a new vector, where each element is a different
 *                 input argument. From macro: a = |1,2,3|
 *
 ******************************************************************************/

class Vector1Function : public Function
{
public:
    Vector1Function(const char* n) :
        Function(n) {}
    virtual Value Execute(int arity, Value* arg);
};


Value Vector1Function::Execute(int arity, Value* arg)
{
    auto* v = new CVector(arity);
    for (int i = 0; i < arity; i++) {
        double d;
        arg[i].GetValue(d);
        if (d != VECTOR_MISSING_VALUE)
            v->setIndexedValue(i, d);
        else
            v->setIndexedValueToMissing(i);
    }
    return Value(v);
}


/*******************************************************************************
 *
 * Function      : Vector2Function
 *
 * Description   : Creates a new vector of a given number of elements (all will
 *                 be initialised to 0.0). Macro: a = vector(5)
 *
 ******************************************************************************/

class Vector2Function : public Function
{
    bool fromList;

public:
    Vector2Function(const char* n) :
        Function(n, 1, tnumber)
    {
        info = "Builds a new vector either with 'n' zero elements or populated from a list";
    }
    virtual Value Execute(int arity, Value* arg);
    virtual int ValidArguments(int arity, Value* arg);
};

int Vector2Function::ValidArguments(int arity, Value* arg)
{
    if (arity != 1)
        return false;

    if (arg[0].GetType() == tlist)
        fromList = true;
    else if (arg[0].GetType() == tnumber)
        fromList = false;
    else
        return false;

    return true;
}


Value Vector2Function::Execute(int, Value* arg)
{
    if (fromList)  // input argument is a list - create a new vector with the same elements
    {
        CList* elements;
        arg[0].GetValue(elements);
        int numElements = elements->Count();

        auto* v = new CVector(numElements, false);  // no need to zero the elements

        for (int i = 0; i < numElements; i++)  // copy the values across
        {
            double d;
            vtype type = (*elements)[i].GetType();
            if (type == tnil)  // nil variables are convert to missing values
            {
                v->setIndexedValueToMissing(i);
            }
            else {
                (*elements)[i].GetValue(d);     // everything else - try to convert - GetValue will fail
                if (d != VECTOR_MISSING_VALUE)  // if the element cannot be converted to a double
                    v->setIndexedValue(i, d);
                else
                    v->setIndexedValueToMissing(i);
            }
        }
        return v;
    }

    else  // input argument is a number - just create a new vector with 'n' elements
    {
        double d;
        arg[0].GetValue(d);
        return Value(new CVector((int)d, true));
    }
}


/*******************************************************************************
 *
 * Function      : VectorRandomFunction
 *
 * Description   : Creates a new vector of a given number of elements (all will
 *                 be initialised to 0.0). Macro: a = vector(5)
 *
 ******************************************************************************/

class VectorRandomFunction : public Function
{
public:
    VectorRandomFunction(const char* n) :
        Function(n, 1, tnumber)
    {
        info = "Builds a new vector either with 'n' random value between 0 and 1";
    }
    virtual Value Execute(int arity, Value* arg);
};


Value VectorRandomFunction::Execute(int, Value* arg)
{
    double d;
    arg[0].GetValue(d);
    auto* v = new CVector((int)d, true);

    static int first = 1;

    if (first) {
        srand48(getpid() * time(0));
        first = 0;
    }

    for (int i = 0; i < d; i++) {
        v->setIndexedValue(i, drand48());
    }


    return Value(v);
}


/*******************************************************************************
 *
 * Function      : VectorSetTypeFunction
 *
 * Description   : Changes the default data type for all subsequent vectors
 *
 ******************************************************************************/

class VectorSetTypeFunction : public Function
{
public:
    VectorSetTypeFunction(const char* n) :
        Function(n, 1, tstring)
    {
        info = "Sets the default internal storage type for all subsequent vectors";
    }
    virtual Value Execute(int arity, Value* arg);
};

Value VectorSetTypeFunction::Execute(int /*arity*/, Value* arg)
{
    const char* typeString;
    arg[0].GetValue(typeString);
    std::string s(typeString);
    std::string oldString = CArray::stringFromValuesType(CVector::defaultValtype);

    CVector::defaultValtype = CArray::valuesTypeFromString(s);

    if (CVector::defaultValtype == CArray::VALUES_INVALID)
        return Error("vector_set_default_type accepts only 'float32' and 'float64', not %s", typeString);

    return Value(oldString.c_str());
}


/*******************************************************************************
 *
 * Function      : VectorDTypeFunction
 *
 * Description   : Returns a string describing the internal storage type
 *
 ******************************************************************************/

class VectorDTypeFunction : public Function
{
public:
    VectorDTypeFunction(const char* n) :
        Function(n, 1, tvector)
    {
        info = "Returns a string describing the vector's internal storage type";
    }
    virtual Value Execute(int arity, Value* arg);
};

Value VectorDTypeFunction::Execute(int /*arity*/, Value* arg)
{
    CVector* v;
    arg[0].GetValue(v);
    std::string typeString = CArray::stringFromValuesType(v->valuesType());
    return Value(typeString.c_str());
}


/*******************************************************************************
 *
 * Function      : VectorGetElemFunction
 *
 * Description   : Gets a specific indexed element (user supplies an index
 *                 starting from 1). A second argument is the index of the
 *                 last element to be extracted - if supplied, then a vector
 *                 is returned. A third argument is an optional step size.
 *                 A fourth argument tells us how many elements to extract
 *                 at each step. Alternatively, a vector can be supplied
 *                 instead of all these numbers - e.g. a vector index with
 *                 6 elements will return a resulting vector with 6 elements.
 *
 ******************************************************************************/

class VectorGetElemFunction : public Function
{
public:
    VectorGetElemFunction(const char* n) :
        Function(n, 2, tvector, tnumber) {}
    virtual Value Execute(int arity, Value* arg);
    virtual int ValidArguments(int arity, Value* arg);

private:
    eVectorIndexType indexType_;
};

int VectorGetElemFunction::ValidArguments(int arity, Value* arg)
{
    if (arity < 2 || arity > 5)
        return false;
    if (arg[0].GetType() != tvector)
        return false;


    // indexing can either be a set of numbers or a single vector
    if (arg[1].GetType() == tvector)  // vector
    {
        if (arity > 2)
            return false;
        else {
            indexType_ = VINDEX_VECTOR;
            return true;
        }
    }
    else  // number(s)
    {
        indexType_ = VINDEX_NUMBERS;

        for (int i = 1; i < arity; i++)  // all arguments after the first one must be numbers
            if (arg[i].GetType() != tnumber)
                return false;
    }

    return true;
}


Value VectorGetElemFunction::Execute(int arity, Value* arg)
{
    if (indexType_ == VINDEX_NUMBERS) {
        CVector* v;
        int first;
        int last   = 0;  // index of first element
        int step   = 0;  // step
        int length = 1;  // how many we extract at each step

        arg[0].GetValue(v);
        arg[1].GetValue(first);
        if (arity > 2)
            arg[2].GetValue(last);
        if (arity > 3)
            arg[3].GetValue(step);
        if (arity > 4)
            arg[4].GetValue(length);

        if (last < first)
            last = first;
        if (step <= 0)
            step = 1;

        if (last + length - step > v->Count())
            return Error("last element out of range. last is %d, vector size is %d",
                         last + length - step, v->Count());

        if (first < 1 || first > v->Count())
            return Error("first out of range. first is %d, vector size is %d",
                         first, v->Count());

        if ((last == first) && (length == 1) && arity == 2)  // single element requested, e.g. v[2]
        {
            if (v->isIndexedValueMissing(first - 1))
                return VECTOR_MISSING_VALUE;  // user only sees this constant for missing values
            else
                return (*v)[first - 1];
        }

        int c      = ((last - first) / step + 1) * length;
        auto* vnew = new CVector(c);
        int i, j;

        for (i = 0, j = first - 1; i < c; i += length, j += step)
            vnew->CopyValues(i, *v, j, length);

        return Value(vnew);
    }

    else  // vector indexing (i.e. the index is itself a vector of indexes)
    {
        CVector* v;
        CVector* vi;
        arg[0].GetValue(v);
        arg[1].GetValue(vi);

        // the result vector
        auto* vnew = new CVector(vi->Count());
        for (int i = 0; i < vi->Count(); i++) {
            int index = (*vi)[i];
            if (index < 1 || index > v->Count())
                return Error("index %d(%d) is out of range. Vector size is %d", i + 1, index, v->Count());

            vnew->setIndexedValue(i, (*v)[index - 1]);  // -1 to convert from 1-index (Macro) to 0-index (C)
        }

        return Value(vnew);
    }
}


/*******************************************************************************
 *
 * Function      : VectorCountFunction
 *
 * Description   : Returns the number of elements in the vector
 *
 ******************************************************************************/

class VectorCountFunction : public Function
{
public:
    VectorCountFunction(const char* n) :
        Function(n, 1, tvector) {}
    virtual Value Execute(int arity, Value* arg);
};

Value VectorCountFunction::Execute(int, Value* arg)
{
    CVector* m;
    arg[0].GetValue(m);

    return Value(m->Count());
}


/*******************************************************************************
 *
 * Function      : VectorVectorBinOp
 *
 * Description   : Generic function for performing an arithmetic operation
 *                 between two vectors
 *
 ******************************************************************************/

class VectorVectorBinOp : public Function
{
    binproc F_;

public:
    VectorVectorBinOp(const char* n, binproc f) :
        Function(n, 2, tvector, tvector) { F_ = f; }
    virtual Value Execute(int arity, Value* arg);
};

Value VectorVectorBinOp::Execute(int, Value* arg)
{
    CVector* v;
    CVector* w;

    arg[0].GetValue(v);
    arg[1].GetValue(w);

    if (v->Count() != w->Count())
        return Error("vectors have different numbers of points (%d and %d)", v->Count(), w->Count());

    auto* p = new CVector(v->Count());

    p->values->applyVectorBinProc(F_, v->values, w->values);

    return Value(p);
}


/*******************************************************************************
 *
 * Function      : NumVectorBinOp
 *
 * Description   : Generic function for performing an arithmetic operation
 *                 between a vector and a number
 *
 ******************************************************************************/

class NumVectorBinOp : public Function
{
    binproc F_;

public:
    NumVectorBinOp(const char* n, binproc f) :
        Function(n) { F_ = f; }
    virtual Value Execute(int arity, Value* arg);
    virtual int ValidArguments(int arity, Value* arg);
};

int NumVectorBinOp::ValidArguments(int arity, Value* arg)
{
    if (arity != 2)
        return false;
    if (arg[0].GetType() == tnumber && arg[1].GetType() == tvector)
        return true;
    if (arg[1].GetType() == tnumber && arg[0].GetType() == tvector)
        return true;
    return false;
}


Value NumVectorBinOp::Execute(int, Value* arg)
{
    double d;
    CVector* v;

    if (arg[0].GetType() == tnumber)  // number op vector
    {
        arg[0].GetValue(d);
        arg[1].GetValue(v);

        auto* p = new CVector(v->Count());
        p->CopyValues(0, *v, 0, v->Count());

        p->values->applyNumBinProc(F_, d, false);

        return Value(p);
    }
    else  // vector op number
    {
        arg[0].GetValue(v);
        arg[1].GetValue(d);

        auto* p = new CVector(v->Count());
        p->CopyValues(0, *v, 0, v->Count());

        p->values->applyNumBinProc(F_, d, true);

        return Value(p);
    }
}


/*******************************************************************************
 *
 * Function      : VectorUniOp
 *
 * Description   : Generic function for performing an arithmetic operation
 *                 between a vector and a number
 *
 ******************************************************************************/

class VectorUniOp : public Function
{
    uniproc F_;

public:
    VectorUniOp(const char* n, uniproc f) :
        Function(n, 1, tvector) { F_ = f; }
    virtual Value Execute(int arity, Value* arg);
};

Value VectorUniOp::Execute(int, Value* arg)
{
    CVector* v;  // original  vector
    CVector* p;  // resultant vector

    arg[0].GetValue(v);

    p = new CVector(v->Count());
    p->values->applyVectorUniProc(F_, v->values);

    return Value(p);
}


/*******************************************************************************
 *
 * Class        : VectorMergeFunction : Function
 *
 * Description  : Macro function that concatenates two vector variables. The second
 *                vector is simply appended to the first.
 *
 ******************************************************************************/


class VectorMergeFunction : public Function
{
public:
    VectorMergeFunction(const char* n) :
        Function(n, 2, tvector, tvector)
    {
        info = "Merges 2 vectors";
    }
    virtual Value Execute(int arity, Value* arg);
};

Value VectorMergeFunction::Execute(int, Value* arg)
{
    CVector* v1;
    CVector* v2;
    CVector* vmerged;

    // get our input vectors

    arg[0].GetValue(v1);
    arg[1].GetValue(v2);


    // create a new vector large enough to hold both sets

    vmerged = new CVector(v1->Count() + v2->Count());


    // copy the first set, followed by the second set

    vmerged->CopyValues(0, *v1, 0, v1->Count());
    vmerged->CopyValues(v1->Count(), *v2, 0, v2->Count());


    // return the result

    return Value(vmerged);
}


/*******************************************************************************
 *
 * Class        : VectorMergeNumberFunction : Function
 *
 * Description  : Macro function that appends a number to a vector variable.
 *
 ******************************************************************************/


class VectorMergeNumberFunction : public Function
{
public:
    VectorMergeNumberFunction(const char* n) :
        Function(n, 2, tvector, tnumber)
    {
        info = "Appends a number to a vector";
    }
    virtual Value Execute(int arity, Value* arg);
};

Value VectorMergeNumberFunction::Execute(int, Value* arg)
{
    CVector* v;
    double d;
    CVector* vmerged;

    // get our input vectors

    arg[0].GetValue(v);
    arg[1].GetValue(d);


    // create a new vector large enough to hold the original plus 1

    vmerged = new CVector(v->Count() + 1);


    // copy the first set, followed by the appended number

    vmerged->CopyValues(0, *v, 0, v->Count());
    vmerged->setIndexedValue(v->Count(), d);


    // return the result

    return Value(vmerged);
}

//-------------------------------------------------------------------


/*******************************************************************************
 *
 * Function      : VectorToListFunction
 *
 * Description   : Converts a vector to a list of numbers. Missing values are
 *                 converted to 'nil'. If the input is already a list, then
 *                 we simply return a copy of the list.
 *
 ******************************************************************************/

class VectorToListFunction : public Function
{
public:
    VectorToListFunction(const char* n) :
        Function(n)
    {
        info = "Converts a vector into a list of numbers";
    }

    virtual Value Execute(int arity, Value* arg);
    virtual int ValidArguments(int arity, Value* arg);

private:
    vtype inputType_;
};

int VectorToListFunction::ValidArguments(int arity, Value* arg)
{
    // we accept: a vector or a list

    if (arity == 1) {
        inputType_ = arg[0].GetType();
        if (inputType_ == tvector || inputType_ == tlist)
            return true;
    }

    return false;
}

Value VectorToListFunction::Execute(int, Value* arg)
{
    // special case - an input list means we simply return a copy of it
    if (inputType_ == tlist) {
        CList* l;
        arg[0].GetValue(l);
        auto* l2 = new CList(*l);
        return Value(l2);
    }


    CVector* v;  // input  vector

    arg[0].GetValue(v);


    int count = v->Count();

    auto* l = new CList(count);  // resultant list

    for (int i = 0; i < count; i++) {
        if (v->isIndexedValueMissing(i))  // if the input value is missing
            (*l)[i] = Value();            // set the output value to nil
        else                              // otherwise, insert the value
            (*l)[i] = (*v)[i];
    }

    return Value(l);
}


/*******************************************************************************
 *
 * Function      : VectorBitmap
 *
 * Description   : Returns a copy of a vector with all elements equal to a
 *                 certain value converted to missing values
 *
 ******************************************************************************/

class VectorBitmap : public Function
{
public:
    VectorBitmap(const char* n) :
        Function(n, 2, tvector, tnumber)
    {
        info = "Converts numbers to missing values in a vector";
    }

    virtual Value Execute(int arity, Value* arg);
};


Value VectorBitmap::Execute(int, Value* arg)
{
    double d;
    CVector* v;

    arg[0].GetValue(v);
    arg[1].GetValue(d);

    int count = v->Count();

    auto* p = new CVector(count);

    for (int i = 0; i < count; i++)  // for each element
    {
        if ((*v)[i] == d)                    // if the value is the given number
            p->setIndexedValueToMissing(i);  // then set the output to missing
        else
            p->setIndexedValue(i, (*v)[i]);  // otherwise use the original value
    }

    return Value(p);
}

/*******************************************************************************
 *
 * Function      : VectorNoBitmap
 *
 * Description   : Returns a copy of a vector with all missing values converted
 *                 to a given value
 *
 ******************************************************************************/

class VectorNoBitmap : public Function
{
public:
    VectorNoBitmap(const char* n) :
        Function(n, 2, tvector, tnumber)
    {
        info = "Converts missing values to numbers in a vector";
    }

    virtual Value Execute(int arity, Value* arg);
};


Value VectorNoBitmap::Execute(int, Value* arg)
{
    double d;
    CVector* v;

    arg[0].GetValue(v);
    arg[1].GetValue(d);

    int count = v->Count();

    auto* p = new CVector(count);

    for (int i = 0; i < count; i++)  // for each element
    {
        if (v->isIndexedValueMissing(i))  // if the value is the missing value
            p->setIndexedValue(i, d);     // then set the output to given number
        else
            p->setIndexedValue(i, (*v)[i]);  // otherwise use the original value
    }

    return Value(p);
}


/*******************************************************************************
 *
 * Function      : VectorFilter
 *
 * Description   : Takes two vectors, and returns a new vector containing only
 *                 the values of the first vector where the second vector's
 *                 values are non-zero.
 *
 ******************************************************************************/

class VectorFilter : public Function
{
public:
    VectorFilter(const char* n) :
        Function(n, 2, tvector, tvector)
    {
        info = "Filters a vector according to the values of a second vector";
    }

    virtual Value Execute(int arity, Value* arg);
};


Value VectorFilter::Execute(int, Value* arg)
{
    CVector *v1, *v2;

    arg[0].GetValue(v1);  // v1 is the vector whose subset we want to retain
    arg[1].GetValue(v2);  // v2 is the vector whose values we will test

    int count = v1->Count();

    if (count != v2->Count())
        return Error("vectors have different numbers of points (%d and %d)", count, v2->Count());

    auto* r = new CVector(count);  // set the result to the biggest it will have to be

    int n = 0;
    for (int i = 0; i < count; i++)  // for each element
    {
        if ((*v2)[i] && !v2->isIndexedValueMissing(i))
            r->setIndexedValue(n++, (*v1)[i]);
    }

    if (n == 0)  // result has no elements? return nil
        return Value();

    r->Resize(n);

    return Value(r);
}


/*******************************************************************************
 *
 * Function      : VectorSort
 *
 * Description   : Takes a vector and an optional sorting operator and returns
 *                 a sorted version of the input vector. Missing values are
 *                 not taken specifically into account - in what sense can we
 *                 sort missing values? The result is that missing values are
 *                 simply treated according to their numerical value, which at
 *                 the time of writing is very large.
 *
 ******************************************************************************/

class VectorSort : public Function
{
    eVectorSortType type;

public:
    VectorSort(const char* n, eVectorSortType t) :
        Function(n, 2, tvector, tstring),
        type(t)
    {
        info = "Sorts a vector according to an operator '<' (default) or '>'";
    }

    virtual Value Execute(int arity, Value* arg);
    virtual int ValidArguments(int arity, Value* arg);
    int SortFnLt(const void* v1, const void* v2);
};


int VectorSort::ValidArguments(int arity, Value* arg)
{
    if (arity > 0) {
        if (arg[0].GetType() == tvector) {
            if (arity == 1)
                return true;

            if (arity == 2 && arg[1].GetType() == tstring) {
                return true;
            }
        }
    }

    // ... otherwise not valid
    return false;
}


Value VectorSort::Execute(int arity, Value* arg)
{
    CVector* v1;
    char op = '<';  // default sort operator
    const char* user_op;

    arg[0].GetValue(v1);  // v1 is the input (unsorted) vector

    if (arity == 2) {
        arg[1].GetValue(user_op);  // user_op is the user-supplied operator

        if (!strcmp(user_op, "<") || !strcmp(user_op, ">"))
            op = user_op[0];
        else
            return Error("vector sort function should be either '<' or '>', not '%s'", user_op);
    }


    // create a copy of the input vector and sort it

    auto* v2 = new CVector(*v1);

    if (type == VSORT_VALUES)
        v2->Sort(op);
    else
        v2->SortIndices(op);

    return Value(v2);
}

//=============================================================================

class VectorFindFunction : public Function
{
public:
    VectorFindFunction(const char* n) :
        Function(n, 2, tany, tvector)
    {
        info = "Find where a number occurs in a vector";
    }
    virtual Value Execute(int arity, Value* arg);
    virtual int ValidArguments(int arity, Value* arg);
};


int VectorFindFunction::ValidArguments(int arity, Value* arg)
{
    if (arity != 2 && arity != 3)
        return false;
    if (arg[0].GetType() != tvector)
        return false;  // first arg must be a vector
    if (arg[1].GetType() != tnumber)
        return false;  // second arg must be a number
    if ((arity == 3) && arg[2].GetType() != tstring)
        return false;  // third (optional) arg must be a string
    return true;
}


Value VectorFindFunction::Execute(int arity, Value* arg)
{
    CVector* v;
    double d;
    bool findAll = false;

    if (arity == 3) {
        const char* s;
        arg[2].GetValue(s);
        if (s && !strcmp(s, "all"))
            findAll = true;
        else
            marslog(LOG_WARN, "find(vector, number, string) only accepts 'all' as its last argument. Supplied: '%s'", s);
    }

    arg[0].GetValue(v);
    arg[1].GetValue(d);

    // quick conversion - if the vector is f32 and the user wants to find a missing
    // value, then we need to change the 'find' number to the correct missing val type
    if (d == VECTOR_MISSING_VALUE)
        d = v->MissingValueIndicator();


    std::vector<int> results;  // only need a vector if we're looking for more than 1 result

    int baseIndex = Context::BaseIndex();
    for (int i = 0; i < v->Count(); i++) {
        if ((*v)[i] == d)  // does it match?
        {
            if (findAll) {
                results.push_back(i + baseIndex);  // we want 'all', so add to our list
            }
            else {
                return Value(i + baseIndex);  // we want just the first, to return it
            }
        }
    }


    if (findAll) {
        if (results.size() > 0) {
            auto* outvec = new CVector(results.size());

            for (unsigned int i = 0; i < results.size(); i++) {
                outvec->setIndexedValue(i, results[i]);
            }
            return Value(outvec);
        }
        else {
            return Value();  // looked for all the values, but did not find any - return nil
        }
    }
    else  // looked for the first value, but did not find it - return nil
    {
        return Value();
    }
}


//=============================================================================

class VectorUniqueFunction : public Function
{
public:
    VectorUniqueFunction(const char* n) :
        Function(n, 1, tvector)
    {
        info = "Returns a vector with all the unique items in the input vector";
    }
    virtual Value Execute(int arity, Value* arg);
};


Value VectorUniqueFunction::Execute(int /*arity*/, Value* arg)
{
    CVector* v;

    arg[0].GetValue(v);

    std::vector<double> u;  // this will hold the result

    for (int i = 0; i < v->Count(); i++) {
        double val = (*v)[i];
        if (std::find(u.begin(), u.end(), val) == u.end())
            u.push_back(val);
    }

    if (u.size() > 0) {
        auto* outvec = new CVector(u);
        return Value(outvec);
    }
    else {
        return Value();  // no elements found at all!
    }
}


//===========================================================================
class MinMaxValueVectorFunction : public Function
{
    int usemax;

public:
    MinMaxValueVectorFunction(const char* n, int m) :
        Function(n, 1, tvector),
        usemax(m) {}
    virtual Value Execute(int arity, Value* arg);
};

Value MinMaxValueVectorFunction::Execute(int, Value* arg)
{
    long i;
    long nFirstValid;
    double x;
    CVector* v;
    arg[0].GetValue(v);

    if (v->Count() == 0)
        return Value();


    // Get the index of the first valid point. If -1 is returned,
    // then there are no valid points.

    nFirstValid = v->IndexOfFirstValidValue();

    if (nFirstValid == -1) {
        return Value();
    }


    int count = v->Count();


    // store the first valid value, then replace it with each successive
    // greater or smaller value

    x = (*v)[nFirstValid];

    if (usemax) {
        for (i = nFirstValid + 1; i < count; i++) {
            if (!v->isIndexedValueMissing(i))  // only consider non-missing values
                if ((*v)[i] > x)
                    x = (*v)[i];
        }
    }
    else {
        for (i = nFirstValid + 1; i < count; i++) {
            if (!v->isIndexedValueMissing((i)))  // only consider non-missing values
                if ((*v)[i] < x)
                    x = (*v)[i];
        }
    }

    return Value(x);
}


//=============================================================================

class MeanVectorFunction : public Function
{
    int compute_mean;

public:
    MeanVectorFunction(const char* n, int m) :
        Function(n, 1, tvector),
        compute_mean(m)
    {
        info = "Returns the sum or mean of the values in a vector variable";
    }
    virtual Value Execute(int arity, Value* arg);
};

Value MeanVectorFunction::Execute(int, Value* arg)
{
    CVector* g;
    double dSum   = 0.0;
    int nNumValid = 0;

    arg[0].GetValue(g);

    int count = g->Count();
    for (int i = 0; i < count; i++) {
        if ((!g->isIndexedValueMissing(i))) {
            dSum += (*g)[i];
            nNumValid++;
        }
    }


    if (nNumValid > 0) {
        if (compute_mean)
            return Value(dSum / nNumValid);
        else
            return Value(dSum);
    }
    else
        return Value();
}

//=============================================================================

class VectorPercentileFunction : public Function
{
public:
    VectorPercentileFunction(const char* n) :
        Function(n, 1, tvector)
    {
        info = "Returns a set of percentiles of a vector";
    }
    virtual int ValidArguments(int arity, Value* arg);
    virtual Value Execute(int arity, Value* arg);
};

int VectorPercentileFunction::ValidArguments(int arity, Value* arg)
{
    if (arity != 2)
        return false;

    if (arg[0].GetType() != tvector)
        return false;

    if (arg[1].GetType() != tlist && arg[1].GetType() != tvector && arg[1].GetType() != tnumber)
        return false;

    return true;
}

/*
 * Input: a single vector, and a percentile or list/vector of percentiles
 * Output: a vector of 'p' numbers
 */
Value VectorPercentileFunction::Execute(int, Value* arg)
{
    CVector* inputVector;
    CVector* percents;
    bool multiple_percs = true;  // result will be a single number or a vector

    arg[0].GetValue(inputVector);


    // get the list of percentiles and put them into a vector if not already one
    argtype t = numberListOrVectorArgAsVector(Owner(), arg[1], &percents);
    if (t == tnumber)
        multiple_percs = false;
    if (t == terror)
        return Error("Percentile: bad data type for second argument, %d", t);


    int numPercentiles = percents->Count();

    auto* resultPercentiles = new CVector(numPercentiles);


    inputVector->Percentile((*percents), CVector::PERCENTILE_NEAREST, (*resultPercentiles));

    if (multiple_percs)
        return Value(resultPercentiles);
    else
        return Value((*resultPercentiles)[0]);
}


//=============================================================================

class ListOfVectorPercentileFunction : public Function
{
public:
    ListOfVectorPercentileFunction(const char* n) :
        Function(n, 1, tvector)
    {
        info = "Returns a set of percentiles of a list of vector variables";
    }
    virtual int ValidArguments(int arity, Value* arg);
    virtual Value Execute(int arity, Value* arg);
};

int ListOfVectorPercentileFunction::ValidArguments(int arity, Value* arg)
{
    //	if (arity != 2 && arity != 3) return false;  // if we do multiple interpolation types
    if (arity != 2)
        return false;

    if (arg[0].GetType() != tlist)
        return false;

    if (arg[1].GetType() != tlist && arg[1].GetType() != tvector && arg[1].GetType() != tnumber)
        return false;

    //if ((arity == 3) && arg[2].GetType() != tstring)  // if we do multiple interpolation types
    //	return false;

    return true;
}

/*
 * Input: a list of 'v' vectors, each containing 'n' numbers, plus a list of 'p' percentiles
 * Output: a list of 'p' vectors, each containing 'n' numbers, where the ith element of the qth vector
 *         is the qth percentile of all the ith elements of each of the input vectors.
 * Terminology: each inout vector is a 1-dimensional 'column', so we compute percentiles from
 *              the data values along each 'row'
 */
Value ListOfVectorPercentileFunction::Execute(int, Value* arg)
{
    CList* listOfInputVectors;
    CVector* percents;
    bool resultIsList = true;  // result will be a single vector or a list of vectors

    arg[0].GetValue(listOfInputVectors);


    // get the list of percentiles and put them into a vector if not already one
    argtype t = numberListOrVectorArgAsVector(Owner(), arg[1], &percents);
    if (t == tnumber)
        resultIsList = false;
    if (t == terror)
        return Error("Percentile: bad data type for second argument, %d", t);


    int numPercentiles = percents->Count();
    int numVectors     = listOfInputVectors->Count();
    CVector* c;
    (*listOfInputVectors)[0].GetValue(c);
    int numValsInVector = c->Count();


    // check that all the vectors have the same number of elements
    for (int i = 0; i < numVectors; i++) {
        CVector* v;
        (*listOfInputVectors)[i].GetValue(v);
        int thisCount = v->Count();
        if (thisCount != numValsInVector)
            return Error("percentile: all input vectors must have the same number of elements. First has %d, number %d has %d.",
                         numValsInVector, i + 1, thisCount);
    }

    CList* listOfResults;
    if (resultIsList)
        listOfResults = new CList(numPercentiles);

    auto* row                  = new CVector(numVectors);
    auto* pthPercentile        = new CVector(numValsInVector);
    auto* temporaryPercentiles = new CVector(numPercentiles);

    // loop over each 'row' of the input vectors and construct a new vector from it - we will compute the p'ctiles from this
    for (int r = 0; r < numValsInVector; r++) {
        // collect the values for this row from each column
        for (int i = 0; i < numVectors; i++) {
            CVector* v;
            (*listOfInputVectors)[i].GetValue(v);
            row->setIndexedValue(i, (*v)[r]);
        }

        // 'row' is now a vector containing all the values for this row - compute p-ctiles from it
        row->Percentile((*percents), CVector::PERCENTILE_NEAREST, (*temporaryPercentiles));

        // now we need to splice this result into the form we want to return it in - one vector
        // per percentile, containing the number of values of each of the input vectors
        for (int p = 0; p < numPercentiles; p++) {
            if (r == 0) {
                pthPercentile = new CVector(numValsInVector);
                if (resultIsList)
                    (*listOfResults)[p] = pthPercentile;
            }
            else {
                if (resultIsList)
                    (*listOfResults)[p].GetValue(pthPercentile);
            }
            pthPercentile->setIndexedValue(r, (*temporaryPercentiles)[p]);
        }
    }

    delete row;
    delete temporaryPercentiles;

    if (resultIsList)
        return Value(listOfResults);
    else
        return Value(pthPercentile);
}


//=============================================================================

static void install(Context* c)
{
    int i;

    // register the associations between value types and names
    CArray::valtypes["float64"] = CArray::VALUES_F64;
    CArray::valtypes["float32"] = CArray::VALUES_F32;


    c->AddGlobal(new Variable("vector_missing_value", Value(VECTOR_MISSING_VALUE)));


    c->AddFunction(new Vector1Function("_vector"));
    c->AddFunction(new Vector2Function("vector"));
    c->AddFunction(new VectorCountFunction("count"));
    c->AddFunction(new VectorGetElemFunction("[]"));
    c->AddFunction(new VectorSetTypeFunction("vector_set_default_type"));
    c->AddFunction(new VectorDTypeFunction("dtype"));
    //c->AddFunction(new VectorProductFunction("^"));

    c->AddFunction(new VectorMergeFunction("&"));
    c->AddFunction(new VectorMergeFunction("merge"));

    c->AddFunction(new VectorMergeNumberFunction("&"));
    c->AddFunction(new VectorMergeNumberFunction("merge"));

    c->AddFunction(new VectorToListFunction("tolist"));
    c->AddFunction(new VectorBitmap("bitmap"));
    c->AddFunction(new VectorNoBitmap("nobitmap"));
    c->AddFunction(new VectorFilter("filter"));
    c->AddFunction(new VectorFindFunction("find"));
    c->AddFunction(new VectorUniqueFunction("unique"));
    c->AddFunction(new VectorSort("sort", VSORT_VALUES));
    c->AddFunction(new VectorSort("sort_indices", VSORT_INDICES));
    c->AddFunction(new VectorRandomFunction("random_vector"));

    c->AddFunction(new MinMaxValueVectorFunction("maxvalue", 1));
    c->AddFunction(new MinMaxValueVectorFunction("minvalue", 0));
    c->AddFunction(new MeanVectorFunction("mean", 1));
    c->AddFunction(new MeanVectorFunction("sum", 0));
    c->AddFunction(new ListOfVectorPercentileFunction("percentile"));
    c->AddFunction(new VectorPercentileFunction("percentile"));

    // binary operations

    for (i = 0; BinOps[i].symb; i++)
        c->AddFunction(new VectorVectorBinOp(BinOps[i].symb, BinOps[i].proc));

    for (i = 0; BinOps[i].symb; i++)
        c->AddFunction(new NumVectorBinOp(BinOps[i].symb, BinOps[i].proc));


    // Mult op as Binary op
    // (the above comment is from bufr.cc, but it's not exactly clear how these
    // operations differ from the BinOps ones)

    for (i = 0; MulOps[i].symb; i++)
        c->AddFunction(new VectorVectorBinOp(MulOps[i].symb, MulOps[i].proc));

    for (i = 0; MulOps[i].symb; i++)
        c->AddFunction(new NumVectorBinOp(MulOps[i].symb, MulOps[i].proc));


    // Unary operations

    for (i = 0; UniOps[i].symb; i++)
        c->AddFunction(new VectorUniOp(UniOps[i].symb, UniOps[i].proc));
}

static Linkage linkage(install);
