/***************************** LICENSE START ***********************************

 Copyright 2016 ECMWF and INPE. This software is distributed under the terms
 of the Apache License version 2.0. In applying this license, ECMWF does not
 waive the privileges and immunities granted to it by virtue of its status as
 an Intergovernmental Organization or submit itself to any jurisdiction.

 ***************************** LICENSE END *************************************/

#include "BufrFilterEngine.h"

#include "MvException.h"
#include "MvMiscellaneous.h"
#include "Tokenizer.h"

#include "BufrFilterDef.h"
#include "MvKeyProfile.h"
#include "BufrMetaData.h"
#include "MvEccBufr.h"

#include <assert.h>

#include <exception>
#include <fstream>
#include <iostream>

#include "fstream_mars_fix.h"

long MvBufrValueGroup::longMissingValue_     = kBufrMissingIntValue;
double MvBufrValueGroup::doubleMissingValue_ = kBufrMissingValue;

MvBufrValueItem::MvBufrValueItem(const MvKeyValue& value, bool valueTypeKnown, const std::vector<MvKeyConditionDefinition>& coordCondDefs) :
    value_(value),
    valueTypeKnown_(valueTypeKnown),
    coordCondDefs_(coordCondDefs)
{
    init();
}

MvBufrValueItem::MvBufrValueItem(const MvKeyValue& value, const MvKeyConditionDefinition& condDef, bool valueTypeKnown, const std::vector<MvKeyConditionDefinition>& coordCondDefs) :
    value_(value),
    condDef_(condDef),
    valueTypeKnown_(valueTypeKnown),
    coordCondDefs_(coordCondDefs)
{
    init();
}

MvBufrValueItem::~MvBufrValueItem() = default;

//MvBufrValue::MvBufrValue(const MvKeyValue& value,MvKeyCondition* cond, bool valueTypeKnown) :
//    value_(value), cond_(cond), rankCond_(0), valueTypeKnown_(valueTypeKnown), hasRank_(false) {init();}

void MvBufrValueItem::init()
{
    keyNameWithoutRank_ = value_.key();
    if (value_.key().find("#") == 0) {
        hasRank_            = true;
        keyNameWithoutRank_ = MvObs::keyWithoutOccurrenceTag(keyNameWithoutRank_);
    }
    for (auto d : coordCondDefs_) {
        coordConds_.push_back(nullptr);
    }

    if (coordConds_.empty()) {
        conditionsAdjusted_ = true;
    }

    label_ = value_.key();
}

void MvBufrValueItem::setRankCondition(MvKeyCondition* rankCond)
{
    rankCond_ = rankCond;

    assert(rankCond_ && rankCond_->data() == "rank");
    assert(rankCond_ && hasRank_ == false);

    if (rankCond_) {
        if (rankCond_->data() != "rank")
            throw MvException("Invalid rank condition is defined for parameter " + keyNameWithoutRank_);

        if (hasRank_)
            throw MvException("Key name cannot contain rank! Keyname=" + value_.key());
    }
}

bool MvBufrValueItem::isSameKey(const std::string& name) const
{
    return (hasRank_) ? (value_.key() == name) : (keyNameWithoutRank_ == MvObs::keyWithoutOccurrenceTag(name));
}

bool MvBufrValueItem::adjustConditions(MvObs* obs)
{
    if (conditionsAdjusted_)
        return true;

    bool st = true;
    for (size_t i = 0; i < coordConds_.size(); i++) {
        if (coordConds_[i] == nullptr) {
            int eccType          = obs->elementValueType(coordCondDefs_[i].key());
            MvVariant::Type type = MvVariant::LongType;
            bool validType       = false;
            if (eccType == CODES_TYPE_LONG) {
                type      = MvVariant::LongType;
                validType = true;
            }
            else if (eccType == CODES_TYPE_DOUBLE) {
                type      = MvVariant::DoubleType;
                validType = true;
            }
            else if (eccType == CODES_TYPE_STRING) {
                type      = MvVariant::StringType;
                validType = true;
            }

            if (validType) {
                coordConds_[i] = MvKeyCondition::make(coordCondDefs_[i], type);
            }
            else {
                st = false;
            }
        }
    }

    if (st) {
        conditionsAdjusted_ = true;
    }

    return conditionsAdjusted_;
}

bool MvBufrValueItem::allCondsCreated() const
{
    for (auto item : coordConds_) {
        if (item == nullptr) {
            return false;
        }
    }
    return true;
}

bool MvBufrValueItem::allCoordCondsMatch() const
{
    if (coordConds_.empty())
        return true;

    for (auto item : coordConds_) {
        if (item == nullptr || !item->match()) {
            return false;
        }
    }
    return true;
}

void MvBufrValueItem::setEnabledCoordConds(bool st)
{
    for (auto item : coordConds_) {
        if (item)
            item->setEnabled(st);
    }
}

//======================================================================
//
// MvBufrValueGroup
//
//======================================================================

MvBufrValueGroup::~MvBufrValueGroup()
{
    for (std::size_t i = 0; i < items_.size(); i++)
        if (items_[i].cond_)
            delete items_[i].cond_;
}

void MvBufrValueGroup::add(const MvBufrValueItem& val)
{
    items_.push_back(val);
}

void MvBufrValueGroup::reset()
{
    for (auto& item : items_) {
        item.value_.resetValue();
        if (item.cond_)
            item.cond_->reset();
        item.setEnabledCoordConds(true);
    }
}

bool MvBufrValueGroup::isAllValueSet() const
{
    for (const auto& item : items_) {
        if (item.collectable() && !item.value_.isSet())
            return false;
    }
    return true;
}

bool MvBufrValueGroup::isNoValueSet() const
{
    for (const auto& item : items_) {
        if (item.collectable() && item.value_.isSet())
            return false;
    }
    return true;
}

bool MvBufrValueGroup::isMissingValue(double val)
{
    return fabs(val - doubleMissingValue_) < 10.0E-4;
}

bool MvBufrValueGroup::isMissingValue(long val)
{
    return val == longMissingValue_;
}

const MvBufrValueItem& MvBufrValueGroup::item(const std::string& keyName) const
{
    for (const auto& item : items_) {
        if (item.value_.key() == keyName)
            return item;
    }
    static MvBufrValueItem emptyVal;
    return emptyVal;
}

//const MvKeyValue& MvBufrValueGroup::value(const std::string& keyName) const
//{
//    for (std::size_t i = 0; i < items_.size(); i++) {
//        if (items_[i].value_.key() == keyName)
//            return items_[i].value_;
//    }
//    static MvKeyValue emptyVal;
//    return emptyVal;
//}

const MvBufrValueItem& MvBufrValueGroup::itemByResultIndex(int resultIndex) const
{
    for (const auto& item : items_) {
        if (item.resultIndex_ == resultIndex)
            return item;
    }
    static MvBufrValueItem emptyVal;
    return emptyVal;
}

//const MvKeyValue& MvBufrValueGroup::valueByResultIndex(int resultIndex) const
//{
//    for (std::size_t i = 0; i < items_.size(); i++) {
//        if (items_[i].resultIndex_ == resultIndex)
//            return items_[i].value_;
//    }
//    static MvKeyValue emptyVal;
//    return emptyVal;
//}

void MvBufrValueGroup::setResultIndex(int idx, int resultIdx)
{
    items_[idx].resultIndex_ = resultIdx;
}

void MvBufrValueGroup::adjustType(int idx, int eccType)
{
    assert(idx >= 0 && idx < static_cast<int>(items_.size()));

    if (items_[idx].valueTypeKnown_ == false) {
        std::string key      = items_[idx].value_.key();
        MvVariant::Type type = items_[idx].value_.type();
        bool validType       = false;

        if (eccType == CODES_TYPE_LONG) {
            validType = true;
            if (type != MvVariant::LongType) {
                items_[idx].value_ = MvKeyValue(key, MvVariant::LongType);
            }
        }
        else if (eccType == CODES_TYPE_DOUBLE) {
            validType = true;
            if (type != MvVariant::DoubleType) {
                items_[idx].value_ = MvKeyValue(key, MvVariant::DoubleType);
            }
        }
        else if (eccType == CODES_TYPE_STRING) {
            validType = true;
            if (type != MvVariant::StringType) {
                items_[idx].value_ = MvKeyValue(key, MvVariant::StringType);
            }
        }

        if (validType) {
            //Build the condition if there is a definition for it
            if (items_[idx].cond_ == 0 && !items_[idx].condDef_.isEmpty()) {
                items_[idx].cond_ = MvKeyCondition::make(items_[idx].condDef_,
                                                         items_[idx].value_.type());
            }

            items_[idx].valueTypeKnown_ = true;

            //CODES_TYPE_LONG, CODES_TYPE_DOUBLE, CODES_TYPE_STRING,
            //                  CODES_TYPE_MISSING, CODES_TYPE_UNDEFINED
        }
    }
}

void MvBufrValueGroup::adjustConditions(MvObs* obs)
{
    if (!conditionsAdjusted_) {
        bool st = true;
        for (auto& item : items_) {
            st = st && item.adjustConditions(obs);
        }
        if (st)
            conditionsAdjusted_ = true;
    }
}

bool MvBufrValueGroup::evalCondition(MvKeyCondition* cond, MvObs* obs, long& lVal, double& dVal, std::string& sVal)
{
    bool hasValue = false;
    if (cond) {
        switch (cond->type()) {
            case MvVariant::DoubleType:
                dVal = obs->currentValue();
                if (!isMissingValue(dVal)) {
                    cond->eval(dVal);
                    cond->setCurrentKeyValue(MvVariant(dVal));
                }
                else
                    cond->reset();
                hasValue = true;
                break;
            case MvVariant::LongType:
                lVal = obs->currentIntValue();
                if (!isMissingValue(lVal)) {
                    cond->eval(lVal);
                    cond->setCurrentKeyValue(MvVariant(lVal));
                }
                else
                    cond->reset();
                hasValue = true;
                break;
            case MvVariant::StringType:
                sVal = obs->stringValue();
                cond->eval(sVal);
                cond->setCurrentKeyValue(MvVariant(sVal));
                hasValue = true;
                break;
            default:
                break;
        }
    }

    return hasValue;
}


bool MvBufrValueGroup::checkCurrentKey(MvObs* obs, int idx)
{
    assert(obs);

    if (directMode_) {
        return checkCurrentKeyDirect(obs, idx);
    }

    //obs->currentKey() always contains the rank. If the rank is set on the item its
    //keyname contains the rank. The condition itself can never contain a rank for an item
    //in a valuegroup!

    //the main condition
    if (items_[idx].isSameKey(obs->currentKey())) {
        if (items_[idx].valueTypeKnown_ == false)
            adjustType(idx, obs->elementValueType());

        if (!items_[idx].collectable())
            return false;

        std::string sVal;
        long lVal     = 0;
        double dVal   = 0;
        bool hasValue = false;

        //Check rank condition
        if (items_[idx].rankCond_) {
            MvKeyCondition* rankCond = items_[idx].rankCond_;
            assert(rankCond->data() == "rank");
            int rank = MvObs::occurenceFromKey(obs->currentKey());
            rankCond->eval(rank);

            if (!rankCond->match()) {
                items_[idx].value_.resetValue();
                if (MvKeyCondition* cond = items_[idx].cond_) {
                    cond->reset();
                }
                return true;
            }
        }

        //Check the condition
        if (MvKeyCondition* cond = items_[idx].cond_) {
            evalCondition(cond, obs, lVal, dVal, sVal);

            //if the condition does not match we reset the collected value
            if (!cond->match()) {
                items_[idx].value_.resetValue();
                return true;
            }
        }

        //Collect the item value
        switch (items_[idx].value_.type()) {
            case MvVariant::DoubleType: {
                if (!hasValue)
                    dVal = obs->currentValue();

                if (includeMissingValue_ || !isMissingValue(dVal))
                    items_[idx].value_.setValue(dVal);
                else
                    items_[idx].value_.resetValue();

                break;
            }
            case MvVariant::LongType: {
                if (!hasValue)
                    lVal = obs->currentIntValue();

                if (includeMissingValue_ || !isMissingValue(lVal))
                    items_[idx].value_.setValue(lVal);
                else
                    items_[idx].value_.resetValue();

                break;
            }
            case MvVariant::StringType:
                if (!hasValue)
                    sVal = obs->stringValue();

                items_[idx].value_.setValue(sVal);
                break;
            default:
                break;
        }

        return true;
    }

    return false;
}

bool MvBufrValueGroup::checkCurrentKeyDirect(MvObs* obs, int idx)
{
    assert(obs);

    //obs->currentKey() always contains the rank. If the rank is set on the item its
    //keyname contains the rank. The condition itself can never contain a rank for an item
    //in a valuegroup!!

    //evaluate special coord=value/element coordinate conditions!
    for (auto cond : items_[idx].coordConds_) {
        if (cond && cond->isEnabled()) {
            if (cond->key() == MvObs::keyWithoutOccurrenceTag(obs->currentKey())) {
                std::string sVal;
                long lVal   = 0;
                double dVal = 0;
                evalCondition(cond, obs, lVal, dVal, sVal);
                //std::cout << idx << " " <<cond->key() << " " << lVal << " " << dVal << std::endl;
            }
        }
    }

    //the main condition
    if (items_[idx].isSameKey(obs->currentKey())) {
        if (items_[idx].valueTypeKnown_ == false)
            adjustType(idx, obs->elementValueType());

        if (!items_[idx].collectable())
            return false;

        std::string sVal;
        long lVal     = 0;
        double dVal   = 0;
        bool hasValue = false;

        //Check rank condition
        if (items_[idx].rankCond_) {
            MvKeyCondition* rankCond = items_[idx].rankCond_;
            assert(rankCond->data() == "rank");
            int rank = MvObs::occurenceFromKey(obs->currentKey());
            rankCond->eval(rank);

            if (!rankCond->match()) {
                items_[idx].value_.resetValue();
                if (MvKeyCondition* cond = items_[idx].cond_) {
                    cond->reset();
                }
                return true;
            }
        }

        //Check the condition
        if (MvKeyCondition* cond = items_[idx].cond_) {
            evalCondition(cond, obs, lVal, dVal, sVal);

            //if the condition does not match we reset the collected value
            if (!cond->match()) {
                items_[idx].value_.resetValue();
                return true;
            }
        }

        if (items_[idx].value_.isSet())
            return true;
        if (!items_[idx].allCoordCondsMatch())
            return true;

        //Collect the item value
        switch (items_[idx].value_.type()) {
            case MvVariant::DoubleType: {
                if (!hasValue)
                    dVal = obs->currentValue();

                if (includeMissingValue_ || !isMissingValue(dVal))
                    items_[idx].value_.setValue(dVal);
                else
                    items_[idx].value_.resetValue();

                break;
            }
            case MvVariant::LongType: {
                if (!hasValue)
                    lVal = obs->currentIntValue();

                if (includeMissingValue_ || !isMissingValue(lVal))
                    items_[idx].value_.setValue(lVal);
                else
                    items_[idx].value_.resetValue();

                break;
            }
            case MvVariant::StringType:
                if (!hasValue)
                    sVal = obs->stringValue();

                items_[idx].value_.setValue(sVal);
                break;
            default:
                break;
        }

        if (items_[idx].value_.isSet())
            items_[idx].setEnabledCoordConds(false);

        return true;
    }

    return false;
}


void MvBufrValueGroup::checkCurrentKey(MvObs* obs)
{
    assert(obs);

    for (std::size_t i = 0; i < items_.size(); i++) {
        if (checkCurrentKey(obs, i)) {
            if (!directMode_) {
                //if a given item is not collected (set) we return. A given
                //item can only be collected if all the previous ones were already collected (set)
                if (!items_[i].value_.isSet())
                    return;
            }
        }
    }
}

void MvBufrValueGroup::updateNonCollectables()
{
    for (std::size_t i = 0; i < items_.size(); i++) {
        if (!items_[i].collectable()) {
            bool found = false;
            for (size_t j = i + 1; j < items_.size() && !found; j++) {
                if (items_[j].collectable()) {
                    for (auto cond : items_[j].coordConds_) {
                        if (cond && cond->key() == items_[i].value_.key()) {
                            if (items_[j].value_.isSet() && cond->match()) {
                                items_[i].value_.setValue(cond->currentKeyValue());
                            }
                            found = true;
                            break;
                        }
                    }
                }
            }
        }
    }
}

//======================================================================
//
// MvBufrConditionGroup
//
//======================================================================

void MvBufrConditionGroup::add(const MvBufrValueItem& val)
{
    MvBufrValueGroup::add(val);
    updateMatchStatus();
}

void MvBufrConditionGroup::reset()
{
    MvBufrValueGroup::reset();
    updateMatchStatus();
}

void MvBufrConditionGroup::checkCurrentKey(MvObs* obs)
{
    assert(obs);

    for (std::size_t i = 0; i < items_.size(); i++) {
        if (MvBufrValueGroup::checkCurrentKey(obs, i)) {
            updateMatchStatus();
            return;
        }
    }
}

void MvBufrConditionGroup::checkConditions(MvObs* obs)
{
    assert(obs);

    for (std::size_t i = 0; i < items_.size(); i++) {
        std::string keyName = items_[i].value_.key();

        if (items_[i].valueTypeKnown_ == false)
            adjustType(i, obs->elementValueType(keyName));

        MvKeyCondition* cond = items_[i].cond_;
        if (cond) {
            switch (cond->type()) {
                case MvVariant::DoubleType: {
                    std::vector<double> dVal;
                    obs->allValues(cond->key(), dVal);
                    for (double iv : dVal) {
                        if (!isMissingValue(iv)) {
                            cond->eval(iv);
                            if (cond->match())
                                break;
                        }
                    }
                } break;
                case MvVariant::LongType: {
                    std::vector<long> lVal;
                    obs->allIntValues(cond->key(), lVal);
                    for (long iv : lVal) {
                        if (!isMissingValue(iv)) {
                            cond->eval(iv);
                            if (cond->match())
                                break;
                        }
                    }
                } break;
                case MvVariant::StringType: {
                    std::vector<std::string> sVal;
                    obs->allStringValues(cond->key(), sVal);
                    for (auto& iv : sVal) {
                        cond->eval(metview::simplified(iv));
                        if (cond->match())
                            break;
                    }
                } break;
                default:
                    break;
            }

            //all conditions must match!!!
            if (!cond->match())
                return;
        }

        updateMatchStatus();
    }
}

void MvBufrConditionGroup::updateMatchStatus()
{
    bool all = true;
    for (auto& item : items_) {
        if (!item.cond_) {
            if (!item.condDef_.isEmpty() || !item.value_.isSet()) {
                all = false;
                break;
            }
        }
        else if (!item.cond_->match()) {
            all = false;
            break;
        }
    }
    allMatch_ = all;
}


//======================================================================
//
// MvBufrStandardGroup
//
//======================================================================

void MvBufrStandardGroup::checkCurrentKey(MvObs* obs)
{
    assert(obs);

    for (std::size_t i = 0; i < items_.size(); i++) {
        if (MvBufrValueGroup::checkCurrentKey(obs, i)) {
            return;
        }
    }
}

//======================================================================
//
// MvBufrPreFilter
//
//======================================================================

void MvBufrPreFilter::setMessageNumber(int num)
{
    messageNumber_.push_back(num);
    enabled_ = true;
}

void MvBufrPreFilter::setEditionNumber(int edition)
{
    editionNumber_.push_back(edition);
    enabled_ = true;
}

void MvBufrPreFilter::setOriginatingCentre(int centre)
{
    originatingCentre_.push_back(centre);
    enabled_ = true;
}

void MvBufrPreFilter::setOriginatingCentreAsStr(const std::string& centre)
{
    originatingCentreStr_.push_back(centre);
    enabled_ = true;
}

void MvBufrPreFilter::setOriginatingSubCentre(int subCentre)
{
    originatingSubCentre_.push_back(subCentre);
    enabled_ = true;
}

void MvBufrPreFilter::setMasterTableVersion(int masterTable)
{
    masterTableVersion_.push_back(masterTable);
    enabled_ = true;
}

void MvBufrPreFilter::setLocalTableVersion(int localTable)
{
    localTableVersion_.push_back(localTable);
    enabled_ = true;
}

void MvBufrPreFilter::setMessageType(int type)
{
    messageType_.push_back(type);
    enabled_ = true;
}

void MvBufrPreFilter::setMessageSubType(int subtype)
{
    messageSubType_.push_back(subtype);
    enabled_ = true;
}

void MvBufrPreFilter::setMessageRdbType(int rdbtype)
{
    messageRdbType_.push_back(rdbtype);
    enabled_ = true;
}

bool MvBufrPreFilter::evalMessageNumber(int msgCnt) const
{
    if (!messageNumber_.empty()) {
        for (int i : messageNumber_)
            if (msgCnt == i)
                return true;

        return false;
    }
    return true;
}

bool MvBufrPreFilter::evalEditionNumber(MvEccBufrMessage* msg) const
{
    if (!editionNumber_.empty()) {
        for (int i : editionNumber_)
            if (msg->bufrEditionNumber() == i)
                return true;
        return false;
    }
    return true;
}

bool MvBufrPreFilter::evalOriginatingCentre(MvEccBufrMessage* msg) const
{
    if (!originatingCentre_.empty()) {
        for (int i : originatingCentre_)
            if (msg->centre() == i)
                return true;
        return false;
    }
    return true;
}

bool MvBufrPreFilter::evalOriginatingCentreAsStr(MvEccBufrMessage* msg) const
{
    if (!originatingCentreStr_.empty()) {
        for (const auto& i : originatingCentreStr_)
            if (msg->centreAsStr() == i)
                return true;
        return false;
    }
    return true;
}

bool MvBufrPreFilter::evalOriginatingSubCentre(MvEccBufrMessage* msg) const
{
    if (!originatingSubCentre_.empty()) {
        for (int i : originatingSubCentre_)
            if (msg->subCentre() == i)
                return true;

        return false;
    }
    return true;
}

bool MvBufrPreFilter::evalMasterTableVersion(MvEccBufrMessage* msg) const
{
    if (!masterTableVersion_.empty()) {
        for (int i : masterTableVersion_)
            if (msg->masterTablesVersionNumber() == i)
                return true;

        return false;
    }
    return true;
}

bool MvBufrPreFilter::evalLocalTableVersion(MvEccBufrMessage* msg) const
{
    if (!localTableVersion_.empty()) {
        for (int i : localTableVersion_)
            if (msg->localTablesVersionNumber() == i)
                return true;

        return false;
    }
    return true;
}

bool MvBufrPreFilter::evalMsgType(MvEccBufrMessage* msg) const
{
    if (!messageType_.empty()) {
        for (int i : messageType_)
            if (msg->dataCategory() == i)
                return true;

        return false;
    }
    return true;
}

bool MvBufrPreFilter::evalMsgSubType(MvEccBufrMessage* msg) const
{
    if (!messageSubType_.empty()) {
        for (int i : messageSubType_)
            if (msg->dataSubCategory() == i)
                return true;

        return false;
    }
    return true;
}

bool MvBufrPreFilter::evalMsgRdbType(MvEccBufrMessage* msg) const
{
    if (!messageRdbType_.empty()) {
        for (int i : messageRdbType_)
            if (msg->rdbType() == i)
                return true;

        return false;
    }
    return true;
}

bool MvBufrPreFilter::evalFilter(MvEccBufrMessage* msg, int msgCnt) const
{
    if (!msg || !msg->isHeaderValid())
        return false;

    if (msgCnt >= 0 && !evalMessageNumber(msgCnt))
        return false;

    if (!evalEditionNumber(msg))
        return false;

    if (!evalOriginatingCentre(msg))
        return false;

    if (!evalOriginatingCentreAsStr(msg))
        return false;

    if (!evalOriginatingSubCentre(msg))
        return false;

    if (!evalMasterTableVersion(msg))
        return false;

    if (!evalLocalTableVersion(msg))
        return false;

    if (!evalMsgType(msg))
        return false;

    if (!evalMsgSubType(msg))
        return false;

    if (!evalMsgRdbType(msg))
        return false;

    return true;
}


void MvBufrPreFilter::evalFilter(const std::vector<MvEccBufrMessage*>& msgData,
                                 std::vector<size_t>& matchVec, int& lastCnt) const
{
    lastCnt = msgData.size() - 1;
    for (size_t i = 0; i < msgData.size(); i++) {
        if (MvEccBufrMessage* msg = msgData[i]) {
            //in the preFilter message indexing starts at 1 !!!
            if (evalFilter(msg, i + 1))
                matchVec.push_back(i);
        }
        else {
            lastCnt = i - 1;
            return;
        }
    }
}


//======================================================================
//
// BufrFilterEngine
//
//======================================================================

BufrFilterEngine::BufrFilterEngine(const std::string& inFileName, FilterMode filterMode, BufrFilterEngineObserver* observer) :
    filterMode_(filterMode),
    outBufr_(0),
    inFileName_(inFileName),
    msgCnt_(0),
    obsOrMsg_(NR_returnMsg),
    outType_(NoOutput),
    extractedHasRank_(false),
    result_(0),
    collectMode_(CollectFirst),
    includeMissingValue_(true),
    includeMissingElement_(false),
    observer_(0),
    hasAttributeCondition_(false)
{
    inObs_   = new MvObsSet(inFileName_.c_str());
    obsIter_ = new MvObsSetIterator(*inObs_);
    if (observer)
        setObserver(observer);
}

BufrFilterEngine::~BufrFilterEngine()
{
    delete obsIter_;
    delete inObs_;
    if (outBufr_)
        delete outBufr_;
}

bool BufrFilterEngine::isExtractedDefined() const
{
    return extracted_.size() > 0;
}

void BufrFilterEngine::close()
{
    //if( outType_ == eBUFR )
    if (outBufr_)
        outBufr_->close();
    //else
    //   _outFile.close();
}


void BufrFilterEngine::runWithBufrData(const BufrFilterDef& def, const std::string& resFileName, MvKeyProfile* resProf, int totalMsgNum, MvEccBufr* bufrData)
{
    if (bufrData)
        run(def, resFileName, resProf, totalMsgNum, bufrData->messages());
    else {
        std::vector<MvEccBufrMessage*> v;
        run(def, resFileName, resProf, totalMsgNum, v);
    }
}

void BufrFilterEngine::run(const BufrFilterDef& def, const std::string& resFileName, MvKeyProfile* resProf, int totalMsgNum, const std::vector<MvEccBufrMessage*>& msgVec)
{
    //msgData=0;

    if (observer_) {
        assert(totalMsgNum > 0);
        if (totalMsgNum == 0)
            return;
    }

    result_ = resProf;

    def_ = def;

    getOutputOptions();
    getIndexOptions();
    getEditionOptions();
    getTypeOptions();
    getIdentOptions();
    getTimeOptions();
    getAreaOptions();
    getCustomOptions();

    if ((filterMode_ == GuiMode && outType_ != NoOutput) || outType_ == BufrOutput) {
        outBufr_ = new MvObsSet(resFileName.c_str(), "w");
    }

    try {
        bool progStep = totalMsgNum / 200;
        if (progStep < 1)
            progStep = 1;

        std::vector<size_t> preFilterMatch;
        int lastCheckedCnt = 0;
        bool doPreFilter   = (!msgVec.empty() && preFilter_.isEnabled());

        //---------------------------------------------------------
        // See if we can use the scanned messages - aka preFilter!
        //---------------------------------------------------------

        if (doPreFilter) {
            //We interrupt filter eval at the first NULL message (the message list might not be
            //fully scanned since the scan is running in a separate thread!!!)
            preFilter_.evalFilter(msgVec, preFilterMatch, lastCheckedCnt);
        }

        //----------------------------------------------
        // Filter only by headers
        //----------------------------------------------
        if (obsOrMsg_ == NR_returnMsg) {
            //Using prefilter
            for (int cnt : preFilterMatch) {
                currentObs_ = obsIter_->gotoMessage(msgVec[cnt]->offset(), cnt + 1);
                if (obsIter_->AcceptedObs(currentObs_, true))
                    filterOne();

                int currentNum = cnt + 1;
                if (currentNum % progStep == 0)
                    progress(currentNum);
            }

            bool allChecked = doPreFilter && lastCheckedCnt == static_cast<int>(msgVec.size()) - 1;

            if (!allChecked) {
                //If he had prefilter get the next message after the last checked position
                if (doPreFilter) {
                    //If he had prefilter get the next message after the last checked position
                    int cnt     = lastCheckedCnt;
                    currentObs_ = obsIter_->gotoMessage(msgVec[cnt]->offset(), cnt + 1);
                }

                //Otherwise we start the scan from the first message
                else {
                    currentObs_ = obsIter_->nextMessage();
                }

                //Check the rest of the messages
                while (currentObs_) {
                    if (obsIter_->AcceptedObs(currentObs_, false))
                        filterOne();

                    int currentNum = obsIter_->currentMessageNumber();
                    if (currentNum % progStep == 0)
                        progress(currentNum);

                    //get next message
                    currentObs_ = obsIter_->nextMessage();
                }
            }
        }

        //----------------------------------------------
        // Filter by using the data section as well
        //----------------------------------------------
        else {
            for (int cnt : preFilterMatch) {
                currentObs_ = obsIter_->gotoMessage(msgVec[cnt]->offset(), cnt + 1);

                //iterate through the susbsets
                while (currentObs_) {
                    bool match = false;
                    if ((match = obsIter_->AcceptedObs(currentObs_, true)))
                        filterOne();

                    if (!match || !currentObs_.Advance()) {
                        int currentNum = cnt + 1;
                        if (currentNum % progStep == 0)
                            progress(currentNum);

                        writeCompressed(&currentObs_);
                        break;
                    }
                }
            }

            bool allChecked = doPreFilter && lastCheckedCnt == static_cast<int>(msgVec.size()) - 1;

            if (!allChecked) {
                //If he had prefilter get the next message after the last checked position
                if (doPreFilter) {
                    int cnt     = lastCheckedCnt;
                    currentObs_ = obsIter_->gotoMessage(msgVec[cnt]->offset(), cnt + 1);
                }
                //Otherwise we start the scan from the first message
                else {
                    currentObs_ = obsIter_->nextMessage();
                }

                bool headerDidNotMatch = false;

                //Check the rest of the messages
                while (currentObs_) {
                    bool match = false;
                    if ((match = obsIter_->AcceptedObs(currentObs_, false, headerDidNotMatch))) {
                        filterOne();
                    }

                    if ((!match && headerDidNotMatch) || !currentObs_.Advance()) {
                        int currentNum = obsIter_->currentMessageNumber();
                        if (currentNum % progStep == 0)
                            progress(currentNum);

                        writeCompressed(&currentObs_);

                        //get the next message
                        currentObs_ = obsIter_->nextMessage();
                    }
                }
            }
        }

        close();
    }

    catch (std::exception& e) {
        //marslog( LOG_WARN, "ObsFilter: %s", e.what() );
        //if( failOnError_ )
        //throw MvException( "ObsFilter failed" );
    }
}

void BufrFilterEngine::filterOne()
{
    if (outType_ == NoOutput)
        return;

    bool didExtract = false;
    int collectCnt  = 0;
    extracted_.reset();
    location_.reset();
    coordCond_.reset();
    customCond_.reset();

    MvObs* obs = &currentObs_;
    MvObs subsetObs;

    //Subsets
    if (currentObs_.msgSubsetCount() > 1) {
        //Uncompressed data
        if (!currentObs_.compressData()) {
            int currentSubset = currentObs_.subsetNumber();
            subsetObs         = currentObs_.cloneSubset(currentSubset);
            obs               = &subsetObs;
        }
    }

    //Check custom condition - it is only a message/subset filter condition,
    //but it is not yet implemented in the bufr itartor so we need to
    //do it here
    if (!customCond_.isEmpty()) {
        //We have to make sure the message is upacked
        obs->setFirstDescriptor();
        customCond_.checkConditions(obs);
        if (!customCond_.match()) {
            currentObs_.clearIterator();
            return;
        }
    }

    //we need to extract values
    if (outType_ != BufrOutput) {
        assert(!extracted_.isEmpty());

        if (!directMode_) {
            extractValues(obs, didExtract, collectCnt);
        }
        else {
            extractValuesDirect(obs, didExtract, collectCnt);
        }
    }

    //If we do not do it we will have a crash
    currentObs_.clearIterator();

    //In GUI mode we always create a new bufr file containing only the messages/subsets matching the filter
    if ((filterMode_ == GuiMode && (outType_ == BufrOutput || didExtract)) ||
        outType_ == BufrOutput) {
        assert(outBufr_);
        //first time
        if (msgCnt_ == 0) {
            //-- changed 2011-12-30/vk:
            //--  o target BUFR file is set to contain single subsets, but this is in effect
            //--     only when filtering using data values from BUFR Section 4
            //--  o this setting has no effect when filtering only with metadata values in
            //--     BUFR Section 1; in such cases multisubset messages are copied as they
            //--     are (metadata is common to all subsets in a multisubset msg)
            outBufr_->setSubsetMax(1);  //-- obs_.msgSubsetCount() );
        }

        //If filtering requires values from data section then multisubset
        //messages have to be filtered on observation (subset) level!

        //filtering with header only i.e. write the whole message!
        if (obsOrMsg_ == NR_returnMsg) {
            outBufr_->write(currentObs_);
            //msgCnt_ += currentObs_.msgSubsetCount();
            msgCnt_++;
        }
        //filtering also with data values, write current subset only!
        else {
            if (obs->msgSubsetCount() > 1 && obs->compressData()) {
                //outBufr_->writeCompressed(obs);
                filteredSubSets_.push_back(currentObs_.subsetNumber());
            }
            else {
                outBufr_->add(*obs);
            }
            msgCnt_++;
        }
    }
}

void BufrFilterEngine::initCompressedCache(MvObs* obs)
{
    if (allNeededDataKeys_.empty()) {
        for (size_t i = 0; i < extracted_.size(); i++) {
            allNeededDataKeys_.insert(extracted_.item(i).keyNameWithoutRank());
        }
        for (size_t i = 0; i < location_.size(); i++) {
            allNeededDataKeys_.insert(location_.item(i).keyNameWithoutRank());
        }
        for (size_t i = 0; i < coordCond_.size(); i++) {
            allNeededDataKeys_.insert(coordCond_.item(i).keyNameWithoutRank());
        }
    }
    obs->initCompressedCache(allNeededDataKeys_);
}

void BufrFilterEngine::extractValues(MvObs* obs, bool& didExtract, int& collectCnt)
{
    assert(directMode_ == false);

    // initialise caching for compressed data
    initCompressedCache(obs);

    //this force data to be expanded
    bool cont = obs->setFirstDescriptor();

    //We iterate through the keys
    while (cont) {
        //------------------------------------------------------------------------------
        //We only filter by extracted conditions. No coordinate conditions are defined.
        //------------------------------------------------------------------------------
        if (coordCond_.isEmpty()) {
            extracted_.checkCurrentKey(obs);
            location_.checkCurrentKey(obs);

            //if all the values are collected and all the conditions match
            if (extracted_.isAllValueSet()) {
                addToResult(obs, coordCond_);
                extracted_.reset();
                didExtract = true;
                collectCnt++;
            }
        }

        //------------------------------------------------------------------------------
        //Has coordinate conditions
        //------------------------------------------------------------------------------
        else if (!coordCond_.isEmpty()) {
            if (coordCond_.match()) {
#if 0
                    std::cout << "coord match" << std::endl;
                    std::cout << "    current=" << obs->currentKey() << std::endl;
#endif
                extracted_.checkCurrentKey(obs);
                if (extracted_.isAllValueSet()) {
                    addToResult(obs, coordCond_);
                    extracted_.reset();
                    didExtract = true;
                    collectCnt++;
                }
            }

            coordCond_.checkCurrentKey(obs);
            location_.checkCurrentKey(obs);
        }

        if (collectMode_ == CollectFirst) {
            if (collectCnt == 1)
                break;
        }

        cont = obs->setNextDescriptor();
    }
}

void BufrFilterEngine::extractValuesDirect(MvObs* obs, bool& didExtract, int& collectCnt)
{
    assert(directMode_ == true);

    //this force data to be expanded
    bool cont = obs->setFirstDescriptor();

    //adjust(=create) all the conditions directly set on keys
    extracted_.adjustConditions(obs);

    //We iterate through the keys
    while (cont) {
        if (coordCond_.isEmpty()) {
            extracted_.checkCurrentKey(obs);
            location_.checkCurrentKey(obs);

            //if all the values are collected and all the conditions match
            if (extracted_.isAllValueSet()) {
                break;
            }
        }

        cont = obs->setNextDescriptor();
    }

    if (extracted_.isAllValueSet() ||
        (includeMissingElement_ && collectCnt == 0 && !extracted_.isNoValueSet())) {
        extracted_.updateNonCollectables();
        addToResult(obs, coordCond_);
        extracted_.reset();
        didExtract = true;
        collectCnt++;
    }
}

void BufrFilterEngine::writeCompressed(MvObs* obs)
{
    if (obs && !filteredSubSets_.empty()) {
        outBufr_->writeCompressed(obs, filteredSubSets_);
    }
    filteredSubSets_.clear();
}

void BufrFilterEngine::progress(int n)
{
    if (observer_)
        observer_->notifyBufrFilterProgress(n);
}

void BufrFilterEngine::addToResult(MvObs* obs, const MvBufrConditionGroup& coordCond)
{
    assert(result_);
    //MvLocation locData=obs->location();
    TStaticTime timeData = obs->obsTime();

    for (std::size_t i = 0; i < result_->size(); i++) {
        MvKey* rk = result_->at(i);
        assert(rk);
        std::string name = rk->name();
        if (rk->role() == MvKey::IndexRole) {
            int idx = rk->valueNum();
            rk->addIntValue(idx + 1);
        }
        //The message index (starts from 1) in the filtered messages
        else if (rk->role() == MvKey::MessageIndexRole) {
            //rk->addIntValue(obsIter_->currentMessageNumber());
            rk->addIntValue(msgCnt_ + 1);
        }
        else if (rk->role() == MvKey::SubsetIndexRole) {
            rk->addIntValue(1);
        }
        else if (rk->role() == MvKey::DateRole) {
            rk->addIntValue(timeData.dateAsInt());
        }
        else if (rk->role() == MvKey::TimeRole) {
            rk->addStringValue(timeData.timeAsString());
        }
        else if (rk->role() == MvKey::LatitudeRole) {
            rk->addValue(location_.item("latitude").value().value());
        }
        else if (rk->role() == MvKey::LongitudeRole) {
            rk->addValue(location_.item("longitude").value().value());
        }
        else if (rk->role() == MvKey::StationIdRole) {
            std::string ident = obs->headerIdent();
            if (ident.empty())
                ident = "?";
            rk->addValue(ident);
        }
        else {
            MvVariant::Type type = MvVariant::NoType;

            //Coordinates
            if (rk->role() == MvKey::LevelRole)
                type = coordCond.item(name).value().type();
            //extracted values
            else
                type = extracted_.itemByResultIndex(i).value().type();

            if (type == MvVariant::LongType) {
                if (rk->valueType() != MvKey::LongType) {
                    rk->setLongMissingValue(MvBufrValueGroup::longMissingValue_);
                    rk->setValueType(MvKey::LongType, true);
                }
            }
            else if (type == MvVariant::DoubleType) {
                if (rk->valueType() != MvKey::DoubleType) {
                    rk->setDoubleMissingValue(MvBufrValueGroup::doubleMissingValue_);
                    rk->setValueType(MvKey::DoubleType, true);
                }
            }
            else if (type == MvVariant::StringType) {
                if (rk->valueType() != MvKey::StringType) {
                    rk->setValueType(MvKey::StringType);
                }
            }

            if (rk->role() == MvKey::LevelRole)
                rk->addValue(coordCond.item(name).value().value());
            else {
                if (extracted_.itemByResultIndex(i).value().isSet()) {
                    rk->addValue(extracted_.itemByResultIndex(i).value().value());
                }
                else {
                    if (type == MvVariant::LongType)
                        rk->addValue(MvBufrValueGroup::longMissingValue_);
                    else if (type == MvVariant::DoubleType)
                        rk->addValue(MvBufrValueGroup::doubleMissingValue_);
                    else
                        rk->addValue("");
                }
            }
        }
    }

    extracted_.reset();
}

void BufrFilterEngine::makeColumnNamesUnique()
{
    for (size_t i = 0; i < result_->size(); i++) {
        MvKey* rkA = result_->at(i);
        int cnt    = 1;
        for (size_t j = i + 1; j < result_->size(); j++) {
            MvKey* rkB = result_->at(j);
            if (rkA->name() == rkB->name()) {
                rkB->setName(rkA->name() + "_" + std::to_string(cnt));
                cnt++;
            }
        }
    }
}

void BufrFilterEngine::toGeopoints(const std::string& fName, const std::string& format)
{
    makeColumnNamesUnique();

    std::ofstream out;
    out.open(fName.c_str());
    if (!out.is_open()) {
        throw MvException("Cannot open output file: " + fName);
    }

    assert(result_);
    std::vector<MvKey*> keys;

    const char mySeparator = '\t';
    out << "#GEO" << std::endl;
    if (format != "NCOLS") {
        int dataColNum = 0;
        if (format == "GEOPOINTS") {
            out << "#";
            dataColNum = 1;
        } else if (format == "POLAR_VECTOR" || format == "XY_VECTOR") {
            out << "#FORMAT " << format << std::endl
                << "#";
            dataColNum = 2;
        } else {
            throw MvException("Failed to generate geopoints! Invalid format=" + format + "!");
        }

        assert(dataColNum > 0);

        out << "latitude" << mySeparator << "longitude" << mySeparator
            << "level" << mySeparator << "date" << mySeparator
            << "time" << mySeparator;
        out << std::endl
            << "#DATA" << std::endl;

        keys.push_back(result_->key(MvKey::LatitudeRole));
        keys.push_back(result_->key(MvKey::LongitudeRole));
        keys.push_back(result_->key(MvKey::LevelRole));
        keys.push_back(result_->key(MvKey::DateRole));
        keys.push_back(result_->key(MvKey::TimeRole));

        if (dataColNum == 1)
            keys.push_back(result_->key(MvKey::NoRole, 0));
        else {
            keys.push_back(result_->key(MvKey::NoRole, 0));
            keys.push_back(result_->key(MvKey::NoRole, 1));
        }

        for (size_t i = 0; i < keys.size(); i++) {
            if (i != 2 && i != 3 && i != 4 && keys[i] == nullptr)
                throw MvException("Failed to generate geopoints output for format=" + format +
                                  "! keys[" + std::to_string(i) + "] == nullptr");
        }

        int nkeys = keys.size();

        int cnt = result_->valueNum();
        for (int i = 0; i < cnt; i++) {
            //Using this style caused a crash:
            //out << ((keys[2])?keys[2]->valueAsString(i):"0")

            out << outputToString(keys[0], i) << mySeparator << outputToString(keys[1], i) << mySeparator;

            //level
            if (keys[2])
                out << outputToString(keys[2], i) << mySeparator;
            else
                out << "0" << mySeparator;

            //date
            if (keys[3])
                out << outputToString(keys[3], i) << mySeparator;
            else
                out << "0" << mySeparator;

            //time
            if (keys[4])
                out << toGeopointsTime(keys[4]->valueAsString(i)) << mySeparator;
            else
                out << "0" << mySeparator;

            for (int j = 5; j < nkeys; j++) {
                out << outputToString(keys[j], i);
                if (j < nkeys - 1) {
                    out << mySeparator;
                }
            }

            out << std::endl;
        }
        // NCOLS
    }
    else {
        out << "#FORMAT NCOLS" << std::endl;
        out << "#COLUMNS" << std::endl;
        out << "stnid" << mySeparator << "latitude" << mySeparator << "longitude" << mySeparator
            << "level" << mySeparator << "date" << mySeparator
            << "time" << mySeparator;

        for (size_t j = 0; j < result_->size(); j++) {
            if (result_->at(j)->role() == MvKey::LevelRole ||
                result_->at(j)->role() == MvKey::NoRole) {
                out << result_->at(j)->name() << mySeparator;
            }
        }

        out << std::endl
            << "#DATA" << std::endl;

        keys.push_back(result_->key(MvKey::StationIdRole));
        keys.push_back(result_->key(MvKey::LatitudeRole));
        keys.push_back(result_->key(MvKey::LongitudeRole));
        keys.push_back(result_->key(MvKey::LevelRole));
        keys.push_back(result_->key(MvKey::DateRole));
        keys.push_back(result_->key(MvKey::TimeRole));

        for (size_t j = 0; j < result_->size(); j++) {
            if (result_->at(j)->role() == MvKey::LevelRole ||
                result_->at(j)->role() == MvKey::NoRole) {
                keys.push_back(result_->at(j));
            }
        }

        for (size_t i = 0; i < keys.size(); i++) {
            if (i != 3 && i != 4 && i != 5 && keys[i] == nullptr)
                throw MvException("Failed to generate geopoints output for format=" + format +
                                  "! keys[" + std::to_string(i) + "] == nullptr");
        }

        int nkeys = keys.size();

        int cnt = result_->valueNum();
        for (int i = 0; i < cnt; i++) {
            //Using this style caused a crash:
            //out << ((keys[2])?keys[2]->valueAsString(i):"0")

            //station id
            if (keys[0])
                out << metview::stationIdForWritingToFile(keys[0]->valueAsString(i)) << mySeparator;
            else
                out << "?" << mySeparator;

            //lat
            out << outputToString(keys[1], i) << mySeparator;

            //lon
            out << outputToString(keys[2], i) << mySeparator;

            //level
            if (keys[3])
                out << outputToString(keys[3], i) << mySeparator;
            else
                out << "0" << mySeparator;

            //date
            if (keys[4])
                out << outputToString(keys[4], i) << mySeparator;
            else
                out << "0" << mySeparator;

            //time
            if (keys[5])
                out << toGeopointsTime(keys[5]->valueAsString(i)) << mySeparator;
            else
                out << "0" << mySeparator;

            //the rest
            for (int j = 6; j < nkeys; j++) {
                out << outputToString(keys[j], i);
                if (j < nkeys - 1) {
                    out << mySeparator;
                }
            }

            out << std::endl;
        }
    }
}

std::string BufrFilterEngine::outputToString(MvKey* key, int i) const
{
    static std::string missingVal("3.0E+38");  //GEOPOINTS_MISSING_VALUE");
    if (key) {
        if (key->valueType() == MvKey::DoubleType) {
            if (key->isMissingValueDefined() && key->doubleValue()[i] == key->doubleMissingValue())
                return missingVal;
            else
                return key->valueAsString(i);
        }
        else if (key->valueType() == MvKey::LongType) {
            if (key->isMissingValueDefined() && key->longValue()[i] == key->longMissingValue())
                return missingVal;
            else
                return key->valueAsString(i);
        }
        else {
            return key->valueAsString(i);
        }
    }
    return missingVal;
}

void BufrFilterEngine::toCsv(const std::string& fName)
{
    makeColumnNamesUnique();

    std::ofstream out;
    out.open(fName.c_str());
    if (!out.is_open()) {
        throw MvException("Cannot open output file: " + fName);
    }

    assert(result_);

    const char mySeparator = ',';
    int rowNum             = result_->valueNum();
    size_t keyNum          = result_->size();

    //Header
    for (size_t key = 0; key < keyNum; key++) {
        out << result_->at(key)->name();
        if (key != keyNum - 1)
            out << mySeparator;
        else
            out << std::endl;
    }

    //Data rows
    for (int row = 0; row < rowNum; row++) {
        for (size_t key = 0; key < keyNum; key++) {
            out << result_->at(key)->valueAsString(row);
            if (key != keyNum - 1)
                out << mySeparator;
            else if (row != rowNum - 1)
                out << std::endl;
        }
    }
}

//--------------------------------------------
// Get filter options
//--------------------------------------------

void BufrFilterEngine::buildConditionDef(const std::string& key, const std::string& oper,
                                         const std::string& valueKey, MvKeyConditionDefinition& condDef)
{
    if (!oper.empty()) {
        //At this point we do not know the type of the parameter, so we
        //assume it is floating point value. We will determine the
        //real type from the data for the first time we have a
        //value to read and readjust the type of the param and that of the
        //conditions.
        std::vector<std::string> vals;
        getStringValues(valueKey, valueKey, vals);
        if (!vals.empty())
            condDef = MvKeyConditionDefinition(key, oper, vals);
    }
}

void BufrFilterEngine::buildKeyCoordConditionDefs(const std::string& nameIn, std::string& keyName, std::vector<MvKeyConditionDefinition>& conds, bool& needConditionInOutput)
{
    //We do not know the type of the parameter at this point, we set it to string
    std::vector<std::string> tmpVec;
    std::size_t pos;
    std::string name = nameIn;

    pos = nameIn.find("[s]");
    if (pos != std::string::npos) {
        if (pos == name.length() - 3) {
            needConditionInOutput = true;
            name                  = nameIn.substr(0, pos);
        }
        else {
            throw MvException("Invalid column name=" + nameIn);
        }
    }

    Tokenizer parseV("/");
    parseV(name, tmpVec);
    if (tmpVec.size() < 2) {
        keyName = name;
        return;
    }

    std::map<std::string, std::string> opers = {
        {"=", "VALUE"},
        {"!=", "NOT_VALUE"},
        {"<=", "LESS_EQUAL_THAN"},
        {"<", "LESS_THAN"},
        {">", "GREATER_THAN"},
        {">=", "GREATER_EQUAL_THAN"},
        {"~", "RANGE"},
        {"!~", "NOT_RANGE"}};

    if (tmpVec.size() >= 1)
        keyName = tmpVec.back();

    for (size_t i = 0; i < tmpVec.size() - 1; i++) {
        std::string cond = tmpVec[i];
        std::vector<std::string> present;
        for (auto it : opers) {
            if (cond.find(it.first) != std::string::npos) {
                present.push_back(it.first);
            }
        }

        std::string oper;
        if (present.size() == 1)
            oper = present[0];
        else if (present.size() == 2) {
            if (present[0].size() < present[1].size())
                oper = present[1];
            else if (present[0].size() > present[1].size())
                oper = present[0];
            else
                throw MvException("Invalid parameter=" + name + ". No valid operator found in condition=" + cond);
        }

        assert(!oper.empty());

        MvKeyConditionDefinition condDef;
        std::vector<std::string> sideVec;
        Tokenizer parseC(oper);
        parseC(cond, sideVec);
        if (sideVec.size() == 2) {
            std::vector<std::string> vals;
            parseArray(sideVec[1], vals);
            if (!vals.empty()) {
                condDef = MvKeyConditionDefinition(sideVec[0], opers[oper], vals);
            }
            conds.push_back(condDef);
        }
        else {
            throw MvException("Invalid parameter=" + name + ". Condition=" + cond + " cannot be parsed");
        }
    }
}

void BufrFilterEngine::getOutputOptions()
{
    //Get the output type
    std::string val = value("EXTRACT");
    if (val.empty()) {
        throw MvException("No output type is specified");
    }

    std::string outputFormat = value("OUTPUT", false);

    //No extraction
    if (val == "OFF") {
        outType_ = BufrOutput;
        return;
    }
    else {
        outType_ = CsvOutput;
    }

    //Missing data handling
    std::string missing = value("MISSING_DATA");
    if (missing == "IGNORE")
        includeMissingValue_ = false;
    else
        includeMissingValue_ = true;

    extracted_.setIncludeMissingValue(includeMissingValue_);

    //Missing element handling
    std::string missingElem = value("MISSING_ELEMENT", false);
    if (missingElem == "IGNORE")
        includeMissingElement_ = false;
    else
        includeMissingElement_ = true;

    assert(coordCond_.isEmpty());
    assert(extracted_.isEmpty());

    std::vector<std::string> columnVec;
    values("COLUMNS", columnVec, ":");
    if (!columnVec.empty() && columnVec[0] != "ANY") {
        directMode_ = true;
        for (auto colName : columnVec) {
            std::string keyName;
            std::vector<MvKeyConditionDefinition> condDefs;
            bool needConditionInOutput = false;
            buildKeyCoordConditionDefs(colName, keyName, condDefs,
                                       needConditionInOutput);
            //A condition is defined but we do not know its type (we only have
            //the definition)
            //We do not know the type of the parameter at this point, we set it to long
            for (auto cdef : condDefs) {
                MvKeyValue cdKeyVal(cdef.key(), MvVariant::LongType);
                MvBufrValueItem bval(cdKeyVal, false);
                bval.setCollectable(false);
                if (needConditionInOutput) {
                    extracted_.add(bval);
                }
            }

            MvBufrValueItem bval(MvKeyValue(keyName, MvVariant::LongType),
                                 false, condDefs);
            if (!needConditionInOutput) {
                std::string label = colName;
                metview::removeSpaces(label);
                bval.setLabel(label);
            }
            extracted_.add(bval);
        }
    }
    extracted_.setDirectMode(directMode_);

    if (!directMode_) {
        //Get the mumber of parameters
        int parCnt = 0;
        intValue("PARAMETER_COUNT", "parameterCount", 1, parCnt);

        //The paremeters to extract (with their conditions)
        for (int i = 0; i < parCnt; i++) {
            MvKeyConditionDefinition condDef;

            std::string parKey      = value("PARAMETER_" + std::to_string(i + 1));
            std::string parOper     = value("PARAMETER_OPERATOR_" + std::to_string(i + 1));
            std::string parValueKey = "PARAMETER_VALUE_" + std::to_string(i + 1);

            if (parKey.find("->") != std::string::npos)
                hasAttributeCondition_ = true;

            std::string parRank;
            if (getRank("PARAMETER_RANK_" + std::to_string(i + 1), parRank)) {
                extractedHasRank_ = true;
                parKey            = "#" + parRank + "#" + parKey;
            }

            buildConditionDef(parKey, parOper, parValueKey, condDef);

            //We do not know the type of the parameter at this point, we set it to string
            MvKeyValue kVal(parKey, MvVariant::StringType);

            //A condition is defined but we do not know its type (we only have
            //the definition)
            if (!condDef.isEmpty()) {
                extracted_.add(MvBufrValueItem(kVal, condDef, false));
            }
            //There is no condition
            else {
                extracted_.add(MvBufrValueItem(kVal, false));
            }
        }

        //The coordinate conditions
        int coordCnt = 0;
        intValue("COORDINATE_COUNT", "coordKeyCount", 0, coordCnt);
        for (int i = 0; i < coordCnt; i++) {
            std::string coordKey      = value("COORDINATE_" + std::to_string(i + 1));
            std::string coordOper     = value("COORDINATE_OPERATOR_" + std::to_string(i + 1));
            std::string coordValueKey = "COORDINATE_VALUE_" + std::to_string(i + 1);

            if (coordKey.find("->") != std::string::npos)
                hasAttributeCondition_ = true;

            //The rank for a coordinate can be a list!!!
            std::vector<int> coordRanks;
            MvKeyCondition* rankCond = 0;
            if (getRanks("COORDINATE_RANK_" + std::to_string(i + 1), coordRanks)) {
                if (coordRanks.size() == 1) {
                    coordKey = "#" + std::to_string(coordRanks[0]) + "#" + coordKey;
                }
                else {
                    assert(coordRanks.size() > 1);
                    std::vector<MvVariant> vv = MvVariant::makeVector(coordRanks);
                    rankCond                  = MvKeyCondition::make("RANK", coordKey, vv);
                    if (rankCond)
                        rankCond->setData("rank");
                }
            }

            //We do not know the type of the parameter at this point, we set it to string
            MvKeyValue kVal(coordKey, MvVariant::StringType);

            //The condition
            //MvKeyCondition *cond=0;
            MvKeyConditionDefinition condDef;
            buildConditionDef(coordKey, coordOper, coordValueKey, condDef);

            //A condition is defined but we do not know its type (we only have
            //the definition)
            if (!condDef.isEmpty()) {
                MvBufrValueItem bval(kVal, condDef, false);
                if (rankCond)
                    bval.setRankCondition(rankCond);
                coordCond_.add(bval);
            }
            //There is no condition. We just need the existence of the coordinate.
            else {
                MvBufrValueItem bval(kVal, false);
                if (rankCond)
                    bval.setRankCondition(rankCond);
                coordCond_.add(bval);
            }
        }
    }

    if (!extracted_.isEmpty() || !coordCond_.isEmpty()) {
        std::string collect = value("EXTRACT_MODE");
        if (collect == "FIRST")
            collectMode_ = CollectFirst;
        else if (collect == "ALL")
            collectMode_ = CollectAll;
        else
            throw MvException("Invalid value specified for parameter EXTRACT_MODE: " + collect);

        assert(result_);

        MvKeyValue latVal("latitude", MvVariant::DoubleType);
        MvKeyValue lonVal("longitude", MvVariant::DoubleType);

        location_.add(MvBufrValueItem(latVal, false));
        location_.add(MvBufrValueItem(lonVal, false));

        MvKey* mvk = 0;

#if 0
        mvk=new MvKey("index","#");
        mvk->setRole(MvKey::IndexRole);
        mvk->setValueType(MvKey::IntType);
        result_->addKey(mvk);
#endif

#if 0
        for(std::size_t i=0; i < standard_.size(); i++)
        {
            mvk=new MvKey(standard_.value(i).key(),standard_.value(i).key());
            mvk->setValueType(MvKey::DoubleType);
            result_->addKey(mvk);
        }
#endif

        mvk = new MvKey("message", "message");
        mvk->setRole(MvKey::MessageIndexRole);
        mvk->setValueType(MvKey::IntType);
        result_->addKey(mvk);

        if (outputFormat == "NCOLS") {
            mvk = new MvKey("stnid", "stationid");
            mvk->setRole(MvKey::StationIdRole);
            mvk->setValueType(MvKey::StringType);
            result_->addKey(mvk);
        }

        mvk = new MvKey("latitude", "latitude");
        mvk->setRole(MvKey::LatitudeRole);
        mvk->setValueType(MvKey::DoubleType);
        mvk->setDoubleMissingValue(MvBufrValueGroup::doubleMissingValue_);
        mvk->setValuePrecision(2);
        result_->addKey(mvk);

        mvk = new MvKey("longitude", "longitude");
        mvk->setRole(MvKey::LongitudeRole);
        mvk->setValueType(MvKey::DoubleType);
        mvk->setDoubleMissingValue(MvBufrValueGroup::doubleMissingValue_);
        mvk->setValuePrecision(2);
        result_->addKey(mvk);

        //Add coordinates to result
        if (value("EXTRACT_COORDINATE") == "ON") {
            //At this point we do not know the the type of these params. We set them to
            //double and will adjust them on first read
            for (std::size_t i = 0; i < coordCond_.size(); i++) {
                mvk = new MvKey(coordCond_.item(i).value().key(), coordCond_.item(i).value().key());
                mvk->setRole(MvKey::LevelRole);
                mvk->setValueType(MvKey::DoubleType);
                mvk->setDoubleMissingValue(MvBufrValueGroup::doubleMissingValue_);
                mvk->setKeyType(MvKey::EccodesKey);
                result_->addKey(mvk);
            }
        }

        //Add date/time to result
        if (value("EXTRACT_DATE") == "ON") {
            mvk = new MvKey("date", "date");
            mvk->setRole(MvKey::DateRole);
            mvk->setValueType(MvKey::IntType);
            result_->addKey(mvk);

            mvk = new MvKey("time", "time");
            mvk->setRole(MvKey::TimeRole);
            mvk->setValueType(MvKey::StringType);
            result_->addKey(mvk);
        }

        //At this point we do not know the the type of these params. We set them to
        //double and will adjust them on first read
        for (std::size_t i = 0; i < extracted_.size(); i++) {
            mvk = new MvKey(extracted_.item(i).label(), extracted_.item(i).label());
            mvk->setValueType(MvKey::DoubleType);
            mvk->setDoubleMissingValue(MvBufrValueGroup::doubleMissingValue_);
            mvk->setKeyType(MvKey::EccodesKey);
            extracted_.setResultIndex(i, static_cast<int>(result_->size()));
            result_->addKey(mvk);
        }

        //We  need data from the data section
        obsOrMsg_ = NR_returnObs;
    }
}


void BufrFilterEngine::getIndexOptions()
{
    std::vector<int> vals;
    getIntValues("MESSAGE_INDEX", "Message index", 1, vals);
    for (int val : vals) {
        obsIter_->setMessageNumber(val);
        preFilter_.setMessageNumber(val);
    }
}

void BufrFilterEngine::getEditionOptions()
{
    std::vector<int> vals;

    //Edition
    getIntValues("EDITION", "edition", 1, vals);
    for (int val : vals) {
        obsIter_->setEditionNumber(val);
        preFilter_.setEditionNumber(val);
    }

    //Centre - can be int or string
    vals.clear();
    if (isKeyValueNumber("CENTRE")) {
        getIntValues("CENTRE", "Centre", 0, vals);
        for (int val : vals) {
            obsIter_->setOriginatingCentre(val);
            preFilter_.setOriginatingCentre(val);
        }
    }
    else {
        std::vector<std::string> strVals;
        getStringValues("CENTRE", "Centre", strVals);
        for (auto& strVal : strVals) {
            obsIter_->setOriginatingCentreAsStr(strVal);
            preFilter_.setOriginatingCentreAsStr(strVal);
        }
    }

    //SubCentre
    vals.clear();
    getIntValues("SUBCENTRE", "subCentre", 0, vals);
    for (int val : vals) {
        obsIter_->setOriginatingSubCentre(val);
        preFilter_.setOriginatingSubCentre(val);
    }

    //Master table version
    vals.clear();
    getIntValues("MASTERTABLE", "masterTable", 0, vals);
    for (int val : vals) {
        obsIter_->setMasterTableVersion(val);
        preFilter_.setMasterTableVersion(val);
    }

    //Local table version
    vals.clear();
    getIntValues("LOCALTABLE", "localTable", 0, vals);
    for (int val : vals) {
        obsIter_->setLocalTableVersion(val);
        preFilter_.setLocalTableVersion(val);
    }
}

void BufrFilterEngine::getTypeOptions()
{
    std::vector<int> vals;

    //Types
    getIntValues("DATA_TYPE", "dataCategory", 1, vals);
    for (int val : vals) {
        obsIter_->setMessageType(val);
        preFilter_.setMessageType(val);
    }

    //SubTypes
    vals.clear();
    getIntValues("DATA_SUBTYPE", "dataSubCategory", 1, vals);
    for (int val : vals) {
        obsIter_->setMessageSubtype(val);
        preFilter_.setMessageSubType(val);
    }

    //RdbTypes
    vals.clear();
    getIntValues("RDB_TYPE", "rdbType", 1, vals);
    for (int val : vals) {
        obsIter_->setMessageRdbtype(val);
        preFilter_.setMessageRdbType(val);
    }
}

void BufrFilterEngine::getIdentOptions()
{
    std::string keyName = value("IDENT_KEY");

    std::vector<std::string> vals;
    values("IDENT_VALUE", vals);

    if (vals.size() == 1 &&
        (vals[0].empty() || vals[0] == "ANY"))
        return;


    if (keyName == "ident") {
        for (auto& val : vals) {
            obsIter_->setHeaderIdent(val);
        }
    }
    else if (keyName == "wmo_station_id") {
        for (auto& val : vals) {
            //1000*blockNumber+stationNumber;
            obsIter_->setWmoStation(atoi(val.c_str()));
        }

        //It is taken from the data section
        obsOrMsg_ = NR_returnObs;
    }
    else {
        obsIter_->setIdentKey(keyName);
        for (auto& val : vals) {
            obsIter_->setIdentValue(val);
        }

        //It is taken from the data section
        obsOrMsg_ = NR_returnObs;
    }
}

void BufrFilterEngine::getTimeOptions()
{
    std::string mode = value("DATEMODE");
    std::string err;

    //Time window
    if (mode == "WINDOW") {
        int year = 0, month = 0, day = 0, hour = 0, minute = 0, second = 0;
        std::string dateVal = value("DATE");
        if (dateVal == "ANY")
            dateVal.clear();
        if (parseDate(dateVal, year, month, day, err) == false) {
            throw MvException("Paramater <b>Date</b>:<br>" + err);
        }

        std::string timeVal = value("TIME");
        if (timeVal == "ANY")
            timeVal.clear();
        if (parseTime(timeVal, hour, minute, second, err) == false) {
            throw MvException("Paramater <b>Time</b>:<br>" + err);
        }

        std::string winVal = value("WINDOW_IN_MINUTES");
        int winSec         = -1;
        if (parseTimeWindow(winVal, winSec, err) == false) {
            throw MvException("Paramater <b>Window</b>:<br>" + err);
        }

        bool hasDate = (!dateVal.empty());
        bool hasTime = (!timeVal.empty());

        //Has date and time
        if (hasDate && hasTime) {
            TDynamicTime dt(year, month, day, hour, minute, second);

            //TODO: make it work for seconds!!! Currently only takes minutes
            obsIter_->setTimeRange(dt, winSec / 60);
            obsOrMsg_ = NR_returnObs;
        }

        //Only has time
        else if (!hasDate && hasTime) {
            int t = hour * 3600 + minute * 60 + second;
            obsIter_->setTimeRangeInSecWithoutDate(t - winSec, t + winSec);
            obsOrMsg_ = NR_returnObs;
        }
        else if (hasDate && !hasTime) {
            throw MvException("Parameter <b>Time</b> is not speficied properly.");

#if 0
            //This should also be supported!!!!!
            if(winSec == 0)
            {
                if(hasDate && !hasTime)
                {
                }
            }
#endif
        }
    }  //end window mode

    //Period mode
    else {
        int year1 = 0, month1 = 0, day1 = 0, hour1 = 0, minute1 = 0, second1 = 0;
        int year2 = 0, month2 = 0, day2 = 0, hour2 = 0, minute2 = 0, second2 = 0;

        std::string dateVal1 = value("DATE_1");
        if (dateVal1 == "ANY")
            dateVal1.clear();
        if (parseDate(dateVal1, year1, month1, day1, err) == false) {
            throw MvException("Paramater <b>Date1</b>:<br>" + err);
        }

        std::string timeVal1 = value("TIME_1");
        if (timeVal1 == "ANY")
            timeVal1.clear();
        if (parseTime(timeVal1, hour1, minute1, second1, err) == false) {
            throw MvException("Paramater <b>Time1</b>:<br>" + err);
        }

        std::string dateVal2 = value("DATE_2");
        if (dateVal2 == "ANY")
            dateVal2.clear();
        if (parseDate(dateVal2, year2, month2, day2, err) == false) {
            throw MvException("Paramater <b>Date1</b>:<br>" + err);
        }

        std::string timeVal2 = value("TIME_2");
        if (timeVal2 == "ANY")
            timeVal2.clear();
        if (parseTime(timeVal2, hour2, minute2, second2, err) == false) {
            throw MvException("Paramater <b>Time2</b>:<br>" + err);
        }

        bool hasDate1 = (!dateVal1.empty());
        bool hasTime1 = (!timeVal1.empty());

        bool hasDate2 = (!dateVal2.empty());
        bool hasTime2 = (!timeVal2.empty());

        //Has date and time
        if (hasDate1 && hasDate2 && hasTime1 && hasTime2) {
            TDynamicTime dt1(year1, month1, day1, hour1, minute1, second1);
            TDynamicTime dt2(year2, month2, day2, hour2, minute2, second2);

            obsIter_->setTimeRange(dt1, dt2);
            obsOrMsg_ = NR_returnObs;
        }
        //only dates, no times
        else if (hasDate1 && hasDate2 && !hasTime1 && !hasTime2) {
            TDynamicTime dt1(year1, month1, day1);
            TDynamicTime dt2(year2, month2, day2, 23, 59, 59);

            obsIter_->setTimeRange(dt1, dt2);
            obsOrMsg_ = NR_returnObs;
        }
        //only times, no dates
        else if (!hasDate1 && !hasDate2 && hasTime1 && hasTime2) {
            //TODO should be also supported!!!
            throw MvException("Period is not speficied properly.");
        }
        else if (!hasDate1 && !hasDate2 && !hasTime1 && !hasTime2) {
            return;
        }
        else {
            throw MvException("Period is not speficied properly.");
        }
    }

    obsIter_->useObsTime(true);  //always use data
}

void BufrFilterEngine::getAreaOptions()
{
    std::vector<std::string> vals;
    values("AREA", vals);

    if (vals.empty())
        return;

    if (vals.size() == 4) {
        if (vals[0] == "-180" && vals[1] == "90" &&
            vals[2] == "180" && vals[3] == "-90")
            return;

        auto w = metview::fromString<float>(vals[0]);
        auto n = metview::fromString<float>(vals[1]);
        auto e = metview::fromString<float>(vals[2]);
        auto s = metview::fromString<float>(vals[3]);

        std::string err;
        if (checkLon(w, e, err) == false) {
            throw MvException("Parameter <b>Area</b>:<br>" + err);
        }

        if (checkLat(n, s, err) == false) {
            throw MvException("Parameter <b>Area</b>:<br>" + err);
        }

        MvLocation c1(n, w);
        MvLocation c2(s, e);
        obsIter_->setArea(c1, c2);

        //It is taken from the data section
        obsOrMsg_ = NR_returnObs;
    }
}

void BufrFilterEngine::getCustomOptions()
{
    assert(customCond_.isEmpty());

    int customCnt = 0;
    intValue("CUSTOM_COUNT", "customKeyCount", 0, customCnt);

    for (int i = 0; i < customCnt; i++) {
        std::string customKey      = value("CUSTOM_KEY_" + std::to_string(i + 1));
        std::string customOper     = value("CUSTOM_OPERATOR_" + std::to_string(i + 1));
        std::string customValueKey = "CUSTOM_VALUE_" + std::to_string(i + 1);
        std::string customValue    = value(customValueKey);

        if (customOper.empty())
            throw MvException("Invalid operator is specified for custom condition " + customKey);

        if (customValue.empty() || customValue == "ANY")
            throw MvException("Invalid value is specified for custom condition " + customKey);

        if (customKey.find("->") != std::string::npos)
            hasAttributeCondition_ = true;

        std::string customRank;
        if (getRank("CUSTOM_RANK_" + std::to_string(i + 1), customRank)) {
            customKey = "#" + customRank + "#" + customKey;
        }

        //We do not know the type of the parameter at this point, we set it to string
        MvKeyValue kVal(customKey, MvVariant::StringType);

        MvKeyConditionDefinition condDef;
        buildConditionDef(customKey, customOper, customValueKey, condDef);

        //A condition is defined but we do not know its type (we only have
        //the definition)
        if (!condDef.isEmpty()) {
            customCond_.add(MvBufrValueItem(kVal, condDef, false));
        }
        //There is no condition
        else {
            throw MvException("Invalid custom condition for customKey_" +
                              std::to_string(i + 1));
        }
    }

    //We need data from the data section
    if (customCnt > 0)
        obsOrMsg_ = NR_returnObs;
}

bool BufrFilterEngine::getRank(const std::string& rankKey, std::string& rankValue) const
{
    rankValue = value(rankKey);
    if (rankValue.empty() || rankValue == "ANY")
        return false;

    if (!isKeyValueNumber(rankKey)) {
        throw MvException("Invalid value <b>" + rankValue +
                          "</b> specified for parameter <b>" + rankKey + "</b>");

        return false;
    }

    int rank = -1;
    intValue(rankKey, rankKey, 1, rank);
    return rank >= 1;
}

bool BufrFilterEngine::getRanks(const std::string& rankKey, std::vector<int>& rankValue) const
{
    rankValue.clear();

    std::vector<std::string> rankStrVals;
    values(rankKey, rankStrVals);
    if (rankStrVals.empty() || rankStrVals[0] == "ANY")
        return false;

    for (auto& rankStrVal : rankStrVals) {
        if (!isNumber(rankStrVal)) {
            throw MvException("Invalid value <b>" + rankStrVal +
                              "</b> specified for parameter <b>" + rankKey + "</b>");

            return false;
        }
        if (metview::fromString<int>(rankStrVal) < 1) {
            throw MvException("Invalid value <b>" + rankStrVal +
                              "</b> specified for parameter <b>" + rankKey + "</b>");

            return false;
        }
    }

    for (auto& rankStrVal : rankStrVals)
        rankValue.push_back(metview::fromString<int>(rankStrVal));

    return true;
}

void BufrFilterEngine::add(const std::string& key, const std::string& value)
{
    def_.vals_[key] = value;
}

//========================================================================
// Getting values
//========================================================================

bool BufrFilterEngine::isNumber(const std::string& val) const
{
    if (val.empty() || val == "ANY")
        return false;

    return metview::isNumber(val);
}

bool BufrFilterEngine::isKeyValueNumber(const std::string& key) const
{
    std::vector<std::string> v;
    values(key, v);

    if (v.size() > 0) {
        return isNumber(v[0]);
    }

    return false;
}

void BufrFilterEngine::parseArray(const std::string& val, std::vector<std::string>& vals) const
{
    vals.clear();
    std::string v = metview::simplified(val);
    if (!v.empty()) {
        if (v[0] == '[' && v[val.length() - 1] == ']') {
            v = v.substr(1, v.length() - 2);
            Tokenizer parseV(",");
            parseV(v, vals);
        }
        else {
            vals.push_back(v);
        }
    }
    else {
        vals.push_back(v);
    }
}

const std::string& BufrFilterEngine::value(const std::string& key, bool mustExist) const
{
    auto it = def_.vals_.find(key);
    if (it != def_.vals_.end())
        return it->second;

    if (mustExist) {
        throw MvException("No parameter <b>" + key + "</b> specified!");
    }

    static std::string emptyStr;
    return emptyStr;
}

void BufrFilterEngine::intValue(const std::string& key,
                                const std::string& param,
                                int minVal, int& outVal) const
{
    std::string val = value(key);

    if (val.empty() || val == "ANY")
        return;

    outVal = atoi(val.c_str());
    if (outVal < minVal) {
        throw MvException("Invalid value <b>" + val + "</b> specified for parameter <b>" + param + "</b>");
    }
}

void BufrFilterEngine::values(const std::string& key, std::vector<std::string>& valueVec, const std::string& separator) const
{
    valueVec.clear();

    auto it = def_.vals_.find(key);
    if (it != def_.vals_.end()) {
        std::string v = it->second;
        if (v.find(separator) == std::string::npos) {
            valueVec.push_back(v);
            return;
        }
        else {
            std::vector<std::string> tmpVec;
            Tokenizer parseV(separator);
            parseV(v, tmpVec);
            for (std::size_t i = 0; i < tmpVec.size(); i++) {
                valueVec.push_back(metview::simplified(tmpVec[i]));
            }
        }
    }
}

void BufrFilterEngine::getIntValues(const std::string& key,
                                    const std::string& param,
                                    int minVal, std::vector<int>& outVec) const
{
    std::vector<std::string> vals;
    values(key, vals);

    if (vals.size() == 1 &&
        (vals[0].empty() || vals[0] == "ANY"))
        return;

    for (auto& val : vals) {
        if (val.empty()) {
            throw MvException("No value specified for parameter <b>" + param + "</b>");
        }

        int v = atoi(val.c_str());
        if (v < minVal) {
            throw MvException("Invalid value <b>" + val + "</b> specified for parameter <b>" + param + "</b>");
        }

        outVec.push_back(v);
    }
}

void BufrFilterEngine::getDoubleValues(const std::string& key,
                                       const std::string& param,
                                       double minVal, std::vector<double>& outVec) const
{
    std::vector<std::string> vals;
    values(key, vals);

    if (vals.size() == 1 &&
        (vals[0].empty() || vals[0] == "ANY"))
        return;

    for (auto& val : vals) {
        if (val.empty()) {
            throw MvException("No value specified for parameter <b>" + param + "</b>");
        }

        double v = atof(val.c_str());
        if (v < minVal) {
            throw MvException("Invalid value <b>" + val + "</b> specified for parameter <b>" + param + "</b>");
        }

        outVec.push_back(v);
    }
}

void BufrFilterEngine::getStringValues(const std::string& key,
                                       const std::string& /*param*/,
                                       std::vector<std::string>& outVec) const
{
    std::vector<std::string> vals;
    values(key, vals);

    if (vals.size() == 1 &&
        (vals[0].empty() || vals[0] == "ANY"))
        return;

    outVec = vals;
}

//===================================================
// Check parameters
//===================================================

bool BufrFilterEngine::parseDate(const std::string& val, int& year, int& month, int& day, std::string& err) const
{
    if (val.empty())
        return true;

    if (val.size() == 8) {
        year  = metview::fromString<int>(val.substr(0, 4));
        month = metview::fromString<int>(val.substr(4, 2));
        day   = metview::fromString<int>(val.substr(6, 2));

        if (year < 1000 || year > 9999) {
            err = "Invalid year specified: <b>" + std::to_string(year) + "</b>";
            return false;
        }
        if (month < 1 || month > 12) {
            err = "Invalid month specified: <b>" + std::to_string(month) + "</b>";
            ;
            return false;
        }
        if (day < 1 || day > 31) {
            err = "Invalid day specified: <b>" + std::to_string(day) + "</b>";
            return false;
        }
        return true;
    }
    else {
        err = "Invalid date: " + metview::toBold(val);
        return false;
    }

    return false;
}

bool BufrFilterEngine::parseTime(const std::string& inVal, int& hour, int& minute, int& second, std::string& err) const
{
    if (inVal.empty())
        return true;

    hour   = -1;
    minute = -1;
    second = -1;

    std::string val = inVal;

    //hmmss
    if (val.size() == 5)
        val = "0" + val;
    //hhmm
    else if (val.size() == 4)
        val = val + "00";
    //hmm
    else if (val.size() == 3)
        val = "0" + val + "00";
    //hh
    else if (val.size() == 2)
        val = val + "0000";
    //h
    else if (val.size() == 1)
        val = "0" + val + "0000";
    //invalid len
    else if (val.size() != 6) {
        err = "Invalid time: " + metview::toBold(val) + " Time can only contain up to 6 digits!";
        return false;
    }

    hour   = metview::fromString<int>(val.substr(0, 2));
    minute = metview::fromString<int>(val.substr(2, 2));
    second = metview::fromString<int>(val.substr(4, 2));
    if (!checkHour(hour, err))
        return false;
    if (!checkMinute(minute, err))
        return false;
    if (!checkSecond(second, err))
        return false;

    return true;
}

bool BufrFilterEngine::checkHour(int h, std::string& err) const
{
    if (h < 0 || h > 24) {
        err = "Invalid hour: " + metview::toBold(h);
        return false;
    }
    return true;
}

bool BufrFilterEngine::checkMinute(int m, std::string& err) const
{
    if (m < 0 || m > 59) {
        err = "Invalid minute: " + metview::toBold(m);
        return false;
    }
    return true;
}

bool BufrFilterEngine::checkSecond(int s, std::string& err) const
{
    if (s < 0 || s > 59) {
        err = "Invalid second: " + metview::toBold(s);
        return false;
    }
    return true;
}

bool BufrFilterEngine::parseTimeWindow(std::string& winVal, int& winSec, std::string& err) const
{
    winSec = 0;
    if (winVal.empty())
        return true;

    winSec = metview::fromString<int>(winVal);
    if (winSec < 0) {
        err = "Invalid time window: " + metview::toBold(winVal);
        return false;
    }
    winSec *= 60;
    return true;
}

bool BufrFilterEngine::checkLon(float lon1, float lon2, std::string& err) const
{
    if (lon1 < -180) {
        err = "Invalid longitude: " + metview::toBold(lon1);
        return false;
    }
    if (lon2 > 180) {
        err = "Invalid longitude: " + metview::toBold(lon2);
        return false;
    }
    return true;
}

bool BufrFilterEngine::checkLat(float lat1, float lat2, std::string& err) const
{
    if (lat1 > 90) {
        err = "Invalid latitude: " + metview::toBold(lat1);
        return false;
    }
    if (lat2 < -90) {
        err = "Invalid latitude: " + metview::toBold(lat2);
        return false;
    }
    return true;
}

std::string BufrFilterEngine::toGeopointsTime(const std::string& tval) const
{
    //time is in HH:MM:SS format originally
    if (tval.size() == 8)  // && tval[2] == ":" && tval[5] == ":")
    {
        return tval.substr(0, 2) + tval.substr(3, 2);
    }
    return std::string("0000");
}
