/***************************** LICENSE START ***********************************
 Copyright 2012 ECMWF and INPE. This software is distributed under the terms
 of the Apache License version 2.0. In applying this license, ECMWF does not
 waive the privileges and immunities granted to it by virtue of its status as
 an Intergovernmental Organization or submit itself to any jurisdiction.

 ***************************** LICENSE END *************************************/

// MvObsSet.cc,   vk July94
//            rev vk 980306
//            rev fi 20170801  replace BUFRDC by eccodes

//--------------------------------------------------------------------
//  MvObsSet hides the data structures and parameter driven subroutines
//  needed to access BUFR files by wrapping them to a nicer C++ cover.
//--------------------------------------------------------------------
//  Routines and required data structures are described in
//  ECMWF Meteorological Bulletin:
//
//  "Accessing GRIB and BUFR data."
//       by J.D.Chambers
//
//  May 1994 Original version.
//--------------------------------------------------------------------

#include "MvObsSet.h"
#include "MagException.h"
#include "MagicsGlobal.h"

#include <cerrno>
#include <cstring>
#include <iostream>

#ifdef METVIEW_PREPBUFR
#include "MvPrepBufrPrep.h"
#endif

using namespace magics;

int MAX_MESSAGE_LENGTH = 32000;  // e maybe delete in the future

const int cMSG_TYPE_BUFR_TABLES = 11;

static std::string WRITE("w");  // I/O mode

//____________________________________________________________________
//==================================================================== MvObsSet
//____________________________________________________________________

MvObsSet::MvObsSet(const char* aName) :
    _unpacked(false),
    _msgCount(0),
    _minTime(2247, 6, 20),
    _maxTime(1799, 12, 31),
    useSkipExtraAttributes_(true),
    cacheCompressedData_(true) {
    _IO_mode = "r";
    _init(aName);
}

//____________________________________________________________________
MvObsSet::MvObsSet(const char* aName, const char* aMode) :
    _unpacked(false),
    _msgCount(0),
    _minTime(2247, 6, 20),
    _maxTime(1799, 12, 31),
    useSkipExtraAttributes_(true),
    cacheCompressedData_(true) {
    _IO_mode = aMode;
    _init(aName);
}

//____________________________________________________________________
#ifdef METVIEW
MvObsSet::MvObsSet(MvRequest& aRequest, const char* aMode) :
    _unpacked(false), _msgCount(0), _minTime(2247, 6, 20), _maxTime(1799, 12, 31) {
    _IO_mode          = aMode;
    const char* aName = 0;
    aRequest.getValue(aName, "PATH");
    _init(aName);
}
#endif

//____________________________________________________________________
MvObsSet::~MvObsSet() {
    close();

#ifdef METVIEW_PREPBUFR
    if (_prepBufr) {
        delete _prepBufr;
        _prepBufr = 0;
    }
#endif
}

//____________________________________________________________________
void MvObsSet::_init(const char* aName) {
    _ecFile       = 0;
    _obsCount     = -1;
    _msgNumber    = -1;
    _minMaxDone   = false;
    _IO_buffer_OK = false;

#ifdef METVIEW_PREPBUFR
    _isPrepBufrFile = false;
    _prepBufr       = 0;
#endif

    // Open input bufr file
    Open(aName);

    // Initialise variables
    codes_count_in_file(NULL, _ecFile, &_msgCount);  // number of messages

    // Check if output file is BUFR and if PREPBUFR support is available
    if (_IO_mode == WRITE)  // should be two classes!!!!!!! (but this a working one...)
    {
        _bufrOut = new MvBufrOut(this);
    }
    else {
        _bufrOut = 0;
    }

    return;
}

//____________________________________________________________________ setSubsetMax
void MvObsSet::setSubsetMax(int subsetMax) {
    // I think ecCodes does not need this function (_maxNrSubsets).
    //               If this is the case delete it later.
}

//____________________________________________________________________ Open
bool MvObsSet::Open(const char* aFileName) {
    _msgCount  = 0;
    _msgNumber = 0;

    // Open bufr file
    _ecFile = fopen(aFileName, _IO_mode.c_str());
    if (!_ecFile) {
        if (MagicsGlobal::strict()) {
            throw CannotOpenFile(aFileName);
        }

        std::cerr << " >>> MvObsSet::Open - ERROR opening file \'" << aFileName << "\' - " << std::strerror(errno)
                  << std::endl;
        return false;
    }
    return true;
}

//____________________________________________________________________ close
bool MvObsSet::close() {
    long myReturnValue = -1;
    if (_ecFile) {
        myReturnValue = fclose(_ecFile);
        _ecFile       = 0;

        // Clean eccodes handler
        if (_ecH && _ecH->handle()) {
            codes_handle_delete(_ecH->handle());
            _ecH->clear();
            _ecH.reset();
        }
    }

    _IO_buffer_OK = false;
    return !myReturnValue;
}

//____________________________________________________________________ rewind
void MvObsSet::rewind() {
    _msgNumber = 0;
    if (_ecFile) {
        // Clean previous handler
        if (_ecH && _ecH->handle()) {
            codes_handle_delete(_ecH->handle());
            _ecH->clear();
            _ecH.reset();
        }
        ::rewind(_ecFile);  // execute C command 'rewind'!
    }

    return;
}

// msgCnt indexed from one!!!!
MvObs MvObsSet::gotoMessage(long offset, int msgCnt) {
    if (!_ecFile)
        return MvObs();  // nothing if file not ok

    if (_IO_mode == WRITE)
        return MvObs();  // no next when writing !

    // Clean previous handler
    if (_ecH && _ecH->handle()) {
        codes_handle_delete(_ecH->handle());
        _ecH->clear();
        _ecH.reset();
    }

    _msgNumber = msgCnt;

    fseek(_ecFile, offset, SEEK_SET);

    int err          = 0;
    codes_handle* ch = codes_handle_new_from_file(NULL, _ecFile, PRODUCT_BUFR, &err);
    _ecH             = std::make_shared<MvEccHandle>(ch);
    if (ch || err != CODES_SUCCESS) {
        if (!ch) {
            std::cout << "Failed reading next BUFR msg: unable to create handle for message = " << _msgNumber
                      << std::endl;
            _IO_buffer_OK = false;
            return MvObs();
        }

        // Expand all the descriptors i.e. unpack the data values
        _unpacked = false;  // it is a new message
        // if ( unpack )
        //  this->expand();

        _IO_buffer_OK = true;
    }

    // subset number = 1
    return MvObs(_ecH, 1, _unpacked, cacheCompressedData_);
}

//__________________________________________________________________ next
// Reads next BUFR message and returns an MvObs constructed with it.
// An MvObs without a message is returned at EOF.
//------------------------------------------------
MvObs MvObsSet::next(bool unpack) {
    if (!_ecFile)
        return MvObs();  // nothing if file not ok

    if (_IO_mode == WRITE)
        return MvObs();  // no next when writing !

    _msgNumber++;

    // Clean previous handler
    if (_ecH && _ecH->handle()) {
        codes_handle_delete(_ecH->handle());
        _ecH->clear();
        _ecH.reset();
    }

    // Get next message
    int err          = 0;
    codes_handle* ch = codes_handle_new_from_file(NULL, _ecFile, PRODUCT_BUFR, &err);
    _ecH             = std::make_shared<MvEccHandle>(ch);
    if (ch || err != CODES_SUCCESS) {
        if (!ch) {
            std::cout << "Failed reading next BUFR msg: unable to create handle for message = " << _msgNumber
                      << std::endl;
            _IO_buffer_OK = false;
            return MvObs();
        }

        // Expand all the descriptors i.e. unpack the data values
        _unpacked = false;  // it is a new message
        if (unpack)
            this->expand();

        _IO_buffer_OK = true;

        // subset number = 1
        return MvObs(_ecH, 1, _unpacked, cacheCompressedData_);
    }

    _IO_buffer_OK = false;
    return MvObs();
}


//____________________________________________________________________ add
void MvObsSet::add(MvObs& anObs) {
    if (_IO_mode != WRITE)
        return;  // no add when reading !

    _bufrOut->add(anObs);
    _msgCount++;
}

//____________________________________________________________________ write
bool MvObsSet::write(const void* aMsg, const size_t aMsgLen) {
    if (_IO_mode != WRITE)
        return false;

    size_t size = aMsgLen;
    if (fwrite(aMsg, 1, size, _ecFile) != aMsgLen) {
        std::cout << "ERROR -  MvObsSet::write(const void*,int) -> could not write a field" << std::endl;
        return false;
    }

    _msgNumber++;
    return true;
}

bool MvObsSet::write(MvObs& anObs) {
    // Clone the input handle
    codes_handle* clone_handle = codes_handle_clone(anObs.getHandle());
    if (clone_handle == NULL) {
        std::cout << "ERROR -  MvObsSet::write(MvObs&) -> could not clone field" << std::endl;
        return false;
    }

    // Get the coded message in a buffer
    const void* buffer = NULL;
    size_t size        = 0;
    if (codes_get_message(clone_handle, &buffer, &size)) {
        std::cout << "ERROR -  MvObsSet::write(MvObs&) -> could not create a buffer message" << std::endl;

        // Release the clone's handle
        codes_handle_delete(clone_handle);
        return false;
    }

    // Write the buffer to a file
    bool err = write(buffer, size);

    // Release the clone's handle
    codes_handle_delete(clone_handle);

    return err;
}

bool MvObsSet::writeCompressed(MvObs* obs) {
    ASSERT(obs);

    if (!obs->compressData())
        return false;

    // Clone the input handle
    codes_handle* cloneH = codes_handle_clone(obs->getHandle());
    if (cloneH == NULL) {
        std::cout << "ERROR -  MvObsSet::write(MvObs&) -> could not clone field" << std::endl;
        return false;
    }

    codes_set_long(cloneH, "skipExtraKeyAttributes", 1);
    codes_set_long(cloneH, "unpack", 1);
    codes_set_long(cloneH, "extractSubset", obs->subsetNumber());
    codes_set_long(cloneH, "doExtractSubsets", 1);

    // Get the coded message in a buffer
    const void* buffer = NULL;
    size_t size        = 0;
    if (codes_get_message(cloneH, &buffer, &size)) {
        std::cout << "ERROR -  MvObsSet::write(MvObs&) -> could not create a buffer message" << std::endl;

        // Release the clone's handle
        codes_handle_delete(cloneH);
        return false;
    }

    // Write the buffer to a file
    bool err = write(buffer, size);

    // Release the clone's handle
    codes_handle_delete(cloneH);

    return err;
}

bool MvObsSet::writeCompressed(MvObs* obs, const std::vector<int>& subsetVec) {
    ASSERT(obs);

    if (!obs->compressData())
        return false;

    if (subsetVec.empty())
        return false;

    // Clone the input handle
    codes_handle* cloneH = codes_handle_clone(obs->getHandle());
    if (cloneH == NULL) {
        std::cout << "ERROR -  MvObsSet::write(MvObs&) -> could not clone field" << std::endl;
        return false;
    }

    size_t arrSize  = subsetVec.size();
    long* subsetArr = new long[arrSize];
    for (size_t i = 0; i < subsetVec.size(); i++)
        subsetArr[i] = subsetVec[i];

    codes_set_long(cloneH, "skipExtraKeyAttributes", 1);
    codes_set_long(cloneH, "unpack", 1);
    codes_set_long_array(cloneH, "extractSubsetList", subsetArr, arrSize);
    codes_set_long(cloneH, "doExtractSubsets", 1);

    // Get the coded message in a buffer
    const void* buffer = NULL;
    size_t size        = 0;
    if (codes_get_message(cloneH, &buffer, &size)) {
        std::cout << "ERROR -  MvObsSet::write(MvObs&) -> could not create a buffer message" << std::endl;

        // Release the clone's handle
        codes_handle_delete(cloneH);
        delete[] subsetArr;
        return false;
    }

    // Write the buffer to a file
    bool err = write(buffer, size);

    // Release the clone's handle
    codes_handle_delete(cloneH);

    delete[] subsetArr;

    return err;
}


//____________________________________________________________________ messageCount
// Returns the number of BUFR messages found in the file.
// WARNING: a packed BUFR message may contain several observations
//          i.e. the nr of BUFR msgs found <= nr of observation msgs
//          i.e. messageCount() <= obsCount()
//------------------------------------------------------------------
int MvObsSet::messageCount() {
#if 0  // Fec
   // old code removed. Counts message using eccodes only.
   if( _msgCount < 1 )
   {
      if( _bufrFile )
      {
         long myOriginalFilePos = ftell( _bufrFile );
	 rewind();

         _msgCount = 0;
         while( next() )
	    _msgCount++;

printf("file bufrdc pos %ld\n",myOriginalFilePos);
	 fseek( _bufrFile, myOriginalFilePos, SEEK_SET );
       }
   }
//Fec   return _msgCount;
#endif

    return _msgCount;
}

//____________________________________________________________________ obsCount
int MvObsSet::obsCount() {
    // The number of total messages has been already computed
    if (_obsCount >= 1)
        return _obsCount;

    // Save current position of the file
    long myOriginalFilePos = ftell(_ecFile);
    rewind();

    // Compute total number of messages, including sub-messages
    long numberOfSubsets;
    int err           = 0;
    codes_handle* ecH = NULL;
    _obsCount         = 0;
    while ((ecH = codes_handle_new_from_file(NULL, _ecFile, PRODUCT_BUFR, &err)) != NULL || err != CODES_SUCCESS) {
        if (ecH == NULL) {
            std::cout << "Error: unable to create handle for message" << std::endl;
            _obsCount = 0;
            break;
        }

        // Find out the number of subsets and update counter
        codes_get_long(ecH, "numberOfSubsets", &numberOfSubsets);
        _obsCount += numberOfSubsets;

        // Delete handle
        codes_handle_delete(ecH);
    }

    // Restore original position of the file
    fseek(_ecFile, myOriginalFilePos, SEEK_SET);

    return _obsCount;
}

//_________________________________________________________ searchMinMaxTime
void MvObsSet::searchMinMaxTime() {
    std::cout << "MvObsSet::searchMinMaxTime() -> not implemented yet" << std::endl;
    exit(0);

    // It has been computed
    if (_minMaxDone)
        return;

    _minMaxDone = true;
}

//_________________________________________________________ expand
void MvObsSet::expand() {
    if (_unpacked)
        return;  // nothing to be done, it is already expanded

    if (!_ecH || !_ecH->handle())
        return;

    if (useSkipExtraAttributes_) {
        codes_set_long(_ecH->handle(), "skipExtraKeyAttributes", 1);
    }
    codes_set_long(_ecH->handle(), "unpack", 1);
    _unpacked = true;
}

#ifdef METVIEW_PREPBUFR
//____________________________________________________________________ prepBufrFile
bool MvObsSet::prepBufrFile() {
    if (_isPrepBufrFile) {
        cout << "in MvObsSet::prepBufrFile()" << endl;
        cout << "BUFR file contains BUFR tables, processing..." << endl;

        //-- Create new PrepBUFR table files etc...
        _prepBufr = new MvPrepBufrPrep(*this);
        bool ok   = _prepBufr->prepareAll();
        if (!ok) {
            cerr << "Errors: unable to process the PrepBUFR file" << endl;
            return false;
        }
    }
    return _isPrepBufrFile;
}
#endif

//____________________________________________________________________ minDate
#ifdef METVIEW
MvDate MvObsSet ::minDate() {
    searchMinMaxTime();
    double timfloat = 10000.0L * (double)_minTime.GetYear() + 100.0L * (double)_minTime.GetMonth() +
                      (double)_minTime.GetDay() + (double)_minTime.GetHour() / 24.0L +
                      ((double)_minTime.GetMin() + 0.5L) / 60.0L / 24.0L;  // rounded
    return MvDate(timfloat);
}
//____________________________________________________________________ maxDate
MvDate MvObsSet ::maxDate() {
    searchMinMaxTime();
    double timfloat = 10000.0L * (double)_maxTime.GetYear() + 100.0L * (double)_maxTime.GetMonth() +
                      (double)_maxTime.GetDay() + (double)_maxTime.GetHour() / 24.0L +
                      ((double)_maxTime.GetMin() + 0.5L) / 60.0L / 24.0L;  // rounded
    return MvDate(timfloat);
}
#endif

//___________________________________________________________________
//=================================================================== MvObsSetIterator
//___________________________________________________________________

MvObsSetIterator::MvObsSetIterator(MvObsSet& s) :
    _NoFiltersSet(true),
    useObsTime_(false),
    _SelectValueCount(0),
    _MsgTypeCount(0),
    _MsgSubtypeCount(0),
    _TimeFilterState(kTFS_notSet),
    _SelectState(SF_notSet),
    ObsSet(&s),
    observer_(0),
    filterProgressStep_(20) {}

//___________________________________________________________________
MvObsSetIterator::~MvObsSetIterator() {}

bool MvObsSetIterator::checkOptionSize(std::size_t num, const std::string& functionName) {
    if (num >= MAX_FILTER_LIST_ARRAY_SIZE) {
        std::cerr << ">>> MvObsIterator::" + functionName + " array overflow!!!" << std::endl;
        return false;
    }
    return true;
}

int MvObsSetIterator::currentMessageNumber() const {
    return (ObsSet ? (static_cast<int>(ObsSet->_msgNumber)) : -1);
}

MvObs MvObsSetIterator::operator()(ENextReturn returnType) {
    if (!current || (returnType == NR_returnMsg) || !current.Advance()) {
        if (observer_) {
            observer_->notifyObsIteratorProgress(currentMessageNumber());
        }
        next();
    }

    while (current && !AcceptedObs(current)) {
        if (!current || (returnType == NR_returnMsg) || !current.Advance()) {
            // Notify the observer about the progress
            if (observer_) {
                observer_->notifyObsIteratorProgress(currentMessageNumber());
            }
            next();
        }
    }

    return current;
}

void MvObsSetIterator::setFilterProgressStep(int filterProgressStep) {
    filterProgressStep_ = filterProgressStep;
    ASSERT(filterProgressStep_ >= 0);
    if (filterProgressStep_ == 0)
        filterProgressStep_ = 1;
}


MvObs MvObsSetIterator::gotoMessage(long offset, int msgCnt) {
    return ObsSet->gotoMessage(offset, msgCnt);
}

MvObs MvObsSetIterator::nextMessage() {
    return ObsSet->next(false);
}

void MvObsSetIterator::next() {
    current = ObsSet->next(false);

#ifdef METVIEW
//   static int change_me_with_debugger = 0;
// e  if( mars.debug && ( ( ObsSet->messageNumber() == 1 ) || change_me_with_debugger ) )
// e    current.printAllValues();
#endif
}

void MvObsSetIterator::setTime(const TDynamicTime& anObsTime) {
    setTimeRange(anObsTime, anObsTime);
}

void MvObsSetIterator::setTimeRange(const TDynamicTime& anObsTime, short deltaInMinutes) {
    fBeginTime = fEndTime = anObsTime;
    fBeginTime.ChangeByMinutes(-deltaInMinutes);
    fEndTime.ChangeByMinutes(deltaInMinutes);
    _TimeFilterState = kTFS_bothSet;
    _NoFiltersSet    = false;
}

void MvObsSetIterator::setTimeRange(const TDynamicTime& aBeginTime, const TDynamicTime& anEndTime) {
    fBeginTime       = aBeginTime;
    fEndTime         = anEndTime;
    _TimeFilterState = kTFS_bothSet;
    _NoFiltersSet    = false;
}

//_____________________________________________________________ setTimeRangeWithoutDate
// add 941229/vk
//
// parameters: in format HHMM, i.e. 1200 == 12.00, 15 == 0.15, i.e. 100*hour+min!!!
//             values are normalized into range [0000..2400)
//_________________________________________
void MvObsSetIterator::setTimeRangeWithoutDate(int aBegin, int anEnd) {
    TDynamicTime aTime;  // date part is not used, so let's use today..
    int myBegin = aBegin;
    int myEnd   = anEnd;

    while (myBegin < 0)
        myBegin += 2400;
    aTime.SetTime(myBegin / 100, myBegin % 100, 0);
    fBeginTime = aTime;

    while (myEnd >= 2400)
        myEnd -= 2400;
    aTime.SetTime(myEnd / 100, myEnd % 100, 0);
    fEndTime = aTime;

    _TimeFilterState = kTFS_clockSet;
    _NoFiltersSet    = false;
}

void MvObsSetIterator::setTimeRangeInSecWithoutDate(int aBegin, int anEnd) {
    TDynamicTime aTime;  // date part is not used, so let's use today..
    int myBegin = aBegin;
    int myEnd   = anEnd;

    while (myBegin < 0)
        myBegin += 86400;
    int h = myBegin / 3600;
    int m = (myBegin - h * 3600) / 60;
    int s = myBegin - h * 3600 - m * 60;
    aTime.SetTime(h, m, s);
    fBeginTime = aTime;

    while (myEnd >= 86400)
        myEnd -= 86400;
    h = myEnd / 3600;
    m = (myEnd - h * 3600) / 60;
    s = myEnd - h * 3600 - m * 60;
    aTime.SetTime(h, m, s);
    fEndTime = aTime;

    _TimeFilterState = kTFS_clockSet;
    _NoFiltersSet    = false;
}


//____________________________________________________________________
void MvObsSetIterator::setWmoBlock(int blockNumber) {
    if (!checkOptionSize(wmoBlock_.size(), "setWmoBlock"))
        return;

    wmoBlock_.push_back(blockNumber);
    _NoFiltersSet = false;
}

void MvObsSetIterator::setWmoStation(long wmoStation) {
    if (!checkOptionSize(wmoStation_.size(), "setWmoStation"))
        return;

    wmoStation_.push_back(wmoStation);
    _NoFiltersSet = false;
}

void MvObsSetIterator::select(const std::string& aDescriptor, double aValue) {
    if (_SelectValueCount >= MAX_FILTER_LIST_ARRAY_SIZE) {
        std::cerr << ">>> MvObsIterator::select: array overflow!!!" << std::endl;
        return;
    }

    if (_SelectValueCount > 0 && aDescriptor != _SelectDescriptor) {
        std::cerr << ">>> MvObsIterator::select: changing the descriptor while building the list!!!" << std::endl;
    }

    _SelectDescriptor                 = aDescriptor;
    _SelectValue[_SelectValueCount++] = aValue;
    _SelectState                      = SF_listSet;
    _NoFiltersSet                     = false;
}

void MvObsSetIterator::selectRange(const std::string& aDescriptor, double firstValue, double secondValue) {
    _SelectDescriptor = aDescriptor;
    _SelectValue[0]   = firstValue < secondValue ? firstValue : secondValue;
    _SelectValue[1]   = secondValue > firstValue ? secondValue : firstValue;
    _SelectValueCount = 2;
    _SelectState      = SF_rangeSet;
    _NoFiltersSet     = false;
}

void MvObsSetIterator::excludeRange(const std::string& aDescriptor, double firstValue, double secondValue) {
    _SelectDescriptor = aDescriptor;
    _SelectValue[0]   = firstValue < secondValue ? firstValue : secondValue;
    _SelectValue[1]   = secondValue > firstValue ? secondValue : firstValue;
    _SelectValueCount = 2;
    _SelectState      = SF_excludeRangeSet;
    _NoFiltersSet     = false;
}

void MvObsSetIterator::setXSectionLine(const MvLocation& aStartPoint, const MvLocation& anEndPoint,
                                       float aDeltaInMeters) {
    fXSectionLine.setLine(aStartPoint, anEndPoint);
    fXSectionLine.setMaxDelta(aDeltaInMeters);
    _NoFiltersSet = false;
}

void MvObsSetIterator::setArea(const MvLocation& aCorner1, const MvLocation& aCorner2) {
    fArea.set(aCorner1, aCorner2);
    _NoFiltersSet = false;
}

void MvObsSetIterator::setMessageType(int aMsgType) {
    if (_MsgTypeCount >= MAX_FILTER_LIST_ARRAY_SIZE) {
        std::cerr << ">>> MvObsIterator::setMessageType: array overflow!!!" << std::endl;
        return;
    }
    _MsgType[_MsgTypeCount++] = aMsgType;
    _NoFiltersSet             = false;
}

void MvObsSetIterator::setMessageSubtype(int aMsgSubtype) {
    if (_MsgSubtypeCount >= MAX_FILTER_LIST_ARRAY_SIZE) {
        std::cerr << ">>> MvObsIterator::setMessageSubtype: array overflow!!!" << std::endl;
        return;
    }
    _MsgSubtype[_MsgSubtypeCount++] = aMsgSubtype;
    _NoFiltersSet                   = false;
}

void MvObsSetIterator::setMessageNumber(int num) {
    if (!checkOptionSize(messageNumber_.size(), "setMessageNumber"))
        return;

    messageNumber_.push_back(num);
    _NoFiltersSet = false;
}

void MvObsSetIterator::setEditionNumber(int edition) {
    if (!checkOptionSize(editionNumber_.size(), "setEditionNumber"))
        return;

    editionNumber_.push_back(edition);
    _NoFiltersSet = false;
}

void MvObsSetIterator::setOriginatingCentre(int centre) {
    if (!checkOptionSize(originatingCentre_.size(), "setOriginatingCentre"))
        return;

    originatingCentre_.push_back(centre);
    _NoFiltersSet = false;
}

void MvObsSetIterator::setOriginatingCentreAsStr(const std::string& centre) {
    if (!checkOptionSize(originatingCentreStr_.size(), "setOriginatingCentreAsStr"))
        return;

    originatingCentreStr_.push_back(centre);
    _NoFiltersSet = false;
}

void MvObsSetIterator::setOriginatingSubCentre(int subCentre) {
    if (!checkOptionSize(originatingSubCentre_.size(), "setOriginatingSubCentre"))
        return;

    originatingSubCentre_.push_back(subCentre);
    _NoFiltersSet = false;
}

void MvObsSetIterator::setMasterTableVersion(int masterTable) {
    if (!checkOptionSize(masterTableVersion_.size(), "setMasterTableVersion"))
        return;

    masterTableVersion_.push_back(masterTable);
    _NoFiltersSet = false;
}

void MvObsSetIterator::setLocalTableVersion(int localTable) {
    if (!checkOptionSize(localTableVersion_.size(), "setLocalTableVersion"))
        return;

    localTableVersion_.push_back(localTable);
    _NoFiltersSet = false;
}

void MvObsSetIterator::setHeaderIdent(const std::string& headerIdent) {
    if (!checkOptionSize(headerIdent_.size(), "setHeaderIdent"))
        return;

    headerIdent_.push_back(simplified(headerIdent));
    _NoFiltersSet = false;
}

void MvObsSetIterator::setIdentKey(const std::string& identKey) {
    identKey_     = simplified(identKey);
    _NoFiltersSet = false;
}

void MvObsSetIterator::setIdentValue(const std::string& identVal) {
    if (!checkOptionSize(identValue_.size(), "setIdentValue"))
        return;

    identValue_.push_back(identVal);
    _NoFiltersSet = false;
}

void MvObsSetIterator::setMessageRdbtype(int rdbType) {
    if (!checkOptionSize(rdbType_.size(), "setMessageRdbtype"))
        return;

    rdbType_.push_back(rdbType);
    _NoFiltersSet = false;
}

//____________________________________________________________________
float MvObsSetIterator::distanceFromXSectionLine(const MvLocation& aPoint) {
    if (fXSectionLine.startPoint().latitude() == MISSING_LOC_VALUE)
        return MISSING_LOC_VALUE;

    return fXSectionLine.deltaInMeters(aPoint);
}

//____________________________________________________________________
bool MvObsSetIterator::TimeOk(MvObs* anObs) const {
    switch (_TimeFilterState) {
        case kTFS_notSet:
            break;

        case kTFS_clockSet: {
            long obsTime;
            if (useObsTime_) {
                // We need time from the  data section so we need to expand the message
                anObs->expand();
                obsTime = anObs->obsTime().ClockInSeconds();  //-- need to decode => slow
            }
            else {
                obsTime = anObs->msgTime().ClockInSeconds();  //-- use metadata => fast
            }
            long time1 = fBeginTime.ClockInSeconds();
            long time2 = fEndTime.ClockInSeconds();

            if (time1 <= time2)  // e.g.  12-18
            {
                if (obsTime < time1 || obsTime > time2)
                    return false;
            }
            else  // time1 > time2       e.g.  21-03
            {
                if (obsTime < time1 && obsTime > time2)
                    return false;
            }
        } break;

        case kTFS_bothSet: {
            TDynamicTime myTime;
            if (useObsTime_) {
                // We need time from the  data section so we need to expand the message
                anObs->expand();
                myTime = anObs->obsTime();  //-- time from obs => decode first => slow
            }
            else {
                myTime = anObs->msgTime();  //-- time from sec1 => no decode => fast
            }

            if (myTime < fBeginTime || myTime > fEndTime)
                return false;
        } break;
    }

    return true;
}

//____________________________________________________________________
bool MvObsSetIterator::WmoBlockOk(MvObs* anObs) const {
    if (!wmoBlock_.empty()) {
        // we need to expand the message
        anObs->expand();
        for (std::size_t i = 0; i < wmoBlock_.size(); i++) {
            if (anObs->WmoBlockNumber() == wmoBlock_[i])
                return true;
        }
        return false;
    }
    return true;
}

bool MvObsSetIterator::WmoStationOk(MvObs* anObs) const {
    if (!wmoStation_.empty()) {
        // we need to expand the message
        anObs->expand();
        for (std::size_t i = 0; i < wmoStation_.size(); i++) {
            if (anObs->WmoIdentNumber() == wmoStation_[i])
                return true;
        }
        return false;
    }
    return true;
}

//____________________________________________________________________
bool MvObsSetIterator::WithinXSectionLine(MvObs* anObs) const {
    if (fXSectionLine.maxDelta() < 0)  // not set ?
        return true;

    // We need lat/lon from the  data section so we need to expand the message
    anObs->expand();

    if (fXSectionLine.withinDelta(anObs->location()))
        return true;
    else
        return false;
}

//____________________________________________________________________
bool MvObsSetIterator::InsideArea(MvObs* anObs) const {
    if (fArea.lowerLeft().latitude() == MISSING_LOC_VALUE)
        return true;
    else {
        // We need lat/lon from the  data section so we need to expand the message
        anObs->expand();
        return fArea.inside(anObs->location());
    }
}

//____________________________________________________________________
bool MvObsSetIterator::msgTypeOk(MvObs* anObs) const {
    if (_MsgTypeCount < 1)
        return true;

    for (int i = 0; i < _MsgTypeCount; i++)
        if (anObs->messageType() == _MsgType[i])
            return true;

    return false;
}

bool MvObsSetIterator::msgSubtypeOk(MvObs* anObs) const {
    if (_MsgSubtypeCount < 1)
        return true;

    for (int i = 0; i < _MsgSubtypeCount; i++)
        if (anObs->messageSubtype() == _MsgSubtype[i])
            return true;

    return false;
}

bool MvObsSetIterator::messageNumberOk(MvObs* anObs) const {
    if (!messageNumber_.empty()) {
        for (std::size_t i = 0; i < messageNumber_.size(); i++) {
            if (currentMessageNumber() == messageNumber_[i])
                return true;
        }
        return false;
    }
    return true;
}

bool MvObsSetIterator::editionNumberOk(MvObs* anObs) const {
    if (!editionNumber_.empty()) {
        for (std::size_t i = 0; i < editionNumber_.size(); i++) {
            if (anObs->editionNumber() == editionNumber_[i])
                return true;
        }
        return false;
    }
    return true;
}

bool MvObsSetIterator::originatingCentreOk(MvObs* anObs) const {
    if (!originatingCentre_.empty()) {
        for (std::size_t i = 0; i < originatingCentre_.size(); i++) {
            if (anObs->originatingCentre() == originatingCentre_[i])
                return true;
        }
        return false;
    }
    return true;
}

bool MvObsSetIterator::originatingCentreAsStrOk(MvObs* anObs) const {
    if (!originatingCentreStr_.empty()) {
        for (std::size_t i = 0; i < originatingCentreStr_.size(); i++) {
            if (anObs->originatingCentreAsStr() == originatingCentreStr_[i])
                return true;
        }
        return false;
    }
    return true;
}

bool MvObsSetIterator::originatingSubCentreOk(MvObs* anObs) const {
    if (!originatingSubCentre_.empty()) {
        for (std::size_t i = 0; i < originatingSubCentre_.size(); i++) {
            if (anObs->originatingSubCentre() == originatingSubCentre_[i])
                return true;
        }
        return false;
    }
    return true;
}

bool MvObsSetIterator::masterTableVersionOk(MvObs* anObs) const {
    if (!masterTableVersion_.empty()) {
        for (std::size_t i = 0; i < masterTableVersion_.size(); i++) {
            if (anObs->masterTableVersion() == masterTableVersion_[i])
                return true;
        }
        return false;
    }
    return true;
}

bool MvObsSetIterator::localTableVersionOk(MvObs* anObs) const {
    if (!localTableVersion_.empty()) {
        for (std::size_t i = 0; i < localTableVersion_.size(); i++) {
            if (anObs->localTableVersion() == localTableVersion_[i])
                return true;
        }
        return false;
    }
    return true;
}

bool MvObsSetIterator::headerIdentOk(MvObs* anObs) const {
    if (!headerIdent_.empty()) {
        for (std::size_t i = 0; i < headerIdent_.size(); i++) {
            if (simplified(anObs->headerIdent()) == headerIdent_[i])
                return true;
        }
        return false;
    }
    return true;
}

bool MvObsSetIterator::identValueOk(MvObs* anObs) const {
    if (!identValue_.empty()) {
        // We need this key from the data section so we need to expand the message
        anObs->expand();
        for (std::size_t i = 0; i < identValue_.size(); i++) {
            if (anObs->stringValue(identKey_, 1) == identValue_[i])
                return true;
        }
        return false;
    }
    return true;
}

bool MvObsSetIterator::msgRdbtypeOk(MvObs* anObs) const {
    if (!rdbType_.empty()) {
        for (std::size_t i = 0; i < rdbType_.size(); i++) {
            if (anObs->messageRdbtype() == rdbType_[i])
                return true;
        }
        return false;
    }
    return true;
}

//____________________________________________________________________
bool MvObsSetIterator::selectOk(MvObs* anObs) const {
    if (_SelectState == SF_notSet)
        return true;

    // We can assume that this key is from the  data section so we need to expand the message
    anObs->expand();

    double myValue = anObs->valueC(_SelectDescriptor);
    if (myValue == kBufrMissingValue)
        return false;

    switch (_SelectState) {
        case SF_notSet:
            break;

        case SF_listSet: {
            for (int i = 0; i < _SelectValueCount; i++)
                if (myValue == _SelectValue[i])
                    return true;
        }
            return false;

        case SF_rangeSet:
            if (myValue < _SelectValue[0] || myValue > _SelectValue[1])
                return false;
            break;

        case SF_excludeRangeSet:
            if (myValue >= _SelectValue[0] && myValue <= _SelectValue[1])
                return false;
            break;
    }

    return true;
}

//____________________________________________________________________
bool MvObsSetIterator::AcceptedObs(MvObs& anObs, bool skipPreFilterCond) const {
    if (!anObs)
        return false;

    if (_NoFiltersSet)
        return true;

    // Prefilter conditions are fully based on the BUFR header
    if (!skipPreFilterCond) {
        // Index of the message within the bufr file
        if (!messageNumberOk(&anObs))
            return false;

        // BUFR edition
        if (!editionNumberOk(&anObs))
            return false;

        // Edition parameters
        if (!originatingCentreOk(&anObs))
            return false;

        if (!originatingCentreAsStrOk(&anObs))
            return false;

        if (!originatingSubCentreOk(&anObs))
            return false;

        if (!masterTableVersionOk(&anObs))
            return false;

        if (!localTableVersionOk(&anObs))
            return false;

        // Type parameters - fully based on the BUFR header
        if (!msgTypeOk(&anObs))
            return false;

        if (!msgSubtypeOk(&anObs))
            return false;

        if (!msgRdbtypeOk(&anObs))
            return false;
    }

    //"ident" key defined in ecmwf (centre=98) header
    if (!headerIdentOk(&anObs))
        return false;

    // User defined identifier from the data section
    if (!identValueOk(&anObs))
        return false;

    if (!TimeOk(&anObs))  // from Section 1: no decoding required
        return false;

    if (!WmoBlockOk(&anObs))  // others require decoding of the msg
        return false;
    if (!WmoStationOk(&anObs))
        return false;

    if (!selectOk(&anObs))
        return false;

    if (!WithinXSectionLine(&anObs))
        return false;

    if (!InsideArea(&anObs))
        return false;

    return true;
}


bool MvObsSetIterator::AcceptedObs(MvObs& anObs, bool skipPreFilterCond, bool& headerDidNotMatch) const {
    headerDidNotMatch = false;

    if (!anObs)
        return false;

    if (_NoFiltersSet)
        return true;

    // Prefilter conditions are fully based on the BUFR header
    if (!skipPreFilterCond) {
        // Index of the message within the bufr file
        if (!messageNumberOk(&anObs)) {
            headerDidNotMatch = true;
            return false;
        }

        // BUFR edition
        if (!editionNumberOk(&anObs)) {
            headerDidNotMatch = true;
            return false;
        }

        // Edition parameters
        if (!originatingCentreOk(&anObs)) {
            headerDidNotMatch = true;
            return false;
        }

        if (!originatingCentreAsStrOk(&anObs)) {
            headerDidNotMatch = true;
            return false;
        }

        if (!originatingSubCentreOk(&anObs)) {
            headerDidNotMatch = true;
            return false;
        }

        if (!masterTableVersionOk(&anObs)) {
            headerDidNotMatch = true;
            return false;
        }

        if (!localTableVersionOk(&anObs)) {
            headerDidNotMatch = true;
            return false;
        }

        // Type parameters - fully based on the BUFR header
        if (!msgTypeOk(&anObs)) {
            headerDidNotMatch = true;
            return false;
        }

        if (!msgSubtypeOk(&anObs)) {
            headerDidNotMatch = true;
            return false;
        }

        if (!msgRdbtypeOk(&anObs)) {
            headerDidNotMatch = true;
            return false;
        }
    }

    //"ident" key defined in ecmwf (centre=98) header
    if (!headerIdentOk(&anObs)) {
        headerDidNotMatch = true;
        return false;
    }

    // User defined identifier from the data section
    if (!identValueOk(&anObs))
        return false;

    if (!TimeOk(&anObs)) {
        headerDidNotMatch = !useObsTime_;
        return false;
    }

    if (!WmoBlockOk(&anObs))
        return false;

    if (!WmoStationOk(&anObs))
        return false;

    if (!selectOk(&anObs))
        return false;

    if (!WithinXSectionLine(&anObs))
        return false;

    if (!InsideArea(&anObs))
        return false;

    return true;
}


//_____________________________________________________________ operator<<
std::ostream& operator<<(std::ostream& aStream, const MvObsSetIterator& aFilter) {
    int i;

    aStream << "Observation Filter values set:\n";
    if (aFilter._NoFiltersSet) {
        aStream << "   No filter values set!";
        aStream << std::endl;
    }
    else {
        if (aFilter._TimeFilterState != kTFS_notSet) {
            aStream << "   Timerange: ";
            switch (aFilter._TimeFilterState) {
                case kTFS_notSet:
                    aStream << "[not set!]";
                    break;

                case kTFS_clockSet:
                    aStream << aFilter.fBeginTime.GetHour() << ".";
                    aStream.width(2);
                    aStream.fill('0');
                    aStream << aFilter.fBeginTime.GetMin();
                    if (aFilter.fBeginTime != aFilter.fEndTime) {
                        aStream << " - " << aFilter.fEndTime.GetHour() << ".";
                        aStream.width(2);
                        aStream.fill('0');
                        aStream << aFilter.fEndTime.GetMin();
                    }
                    break;

                case kTFS_bothSet:
                    aStream << aFilter.fBeginTime;
                    if (aFilter.fBeginTime != aFilter.fEndTime)
                        aStream << " - " << aFilter.fEndTime;
                    break;
            }
            aStream << std::endl;
        }

        if (aFilter._MsgTypeCount > 0) {
            aStream << "   Message types: ";
            for (i = 0; i < aFilter._MsgTypeCount; i++)
                aStream << " " << aFilter._MsgType[i];
            aStream << std::endl;
        }

        if (aFilter._MsgSubtypeCount > 0) {
            aStream << "   Message subtypes: ";
            for (i = 0; i < aFilter._MsgSubtypeCount; i++)
                aStream << " " << aFilter._MsgSubtype[i];
            aStream << std::endl;
        }

        if (!aFilter.wmoBlock_.empty()) {
            aStream << "   WMO Blocks:";
            for (std::size_t i = 0; i < aFilter.wmoBlock_.size(); i++)
                aStream << " " << aFilter.wmoBlock_[i];
            aStream << std::endl;
        }

        if (!aFilter.wmoStation_.empty()) {
            aStream << "   WMO Stations:";
            for (std::size_t i = 0; i < aFilter.wmoStation_.size(); i++)
                aStream << " " << aFilter.wmoStation_[i];
            aStream << std::endl;
        }

        if (aFilter._SelectState != SF_notSet) {
            aStream << "   Select ";
            switch (aFilter._SelectState) {
                case SF_notSet:
                    aStream << "by values/range: [not set!]";
                    break;

                case SF_listSet:
                    aStream << "by values: ";
                    break;

                case SF_rangeSet:
                    aStream << "by range: ";
                    break;
                case SF_excludeRangeSet:
                    aStream << "by excluding range: ";
                    break;
            }

            for (i = 0; i < aFilter._SelectValueCount; i++)
                aStream << aFilter._SelectValue[i] << " ";
            aStream << "(descr. " << aFilter._SelectDescriptor.c_str() << ")";
            aStream << std::endl;
        }

        if (aFilter.fXSectionLine.startPoint().latitude() != MISSING_LOC_VALUE) {
            aStream << "   Cross Section Line: " << aFilter.fXSectionLine;
            aStream << std::endl;
        }

        if (aFilter.fArea.lowerLeft().latitude() != MISSING_LOC_VALUE) {
            aStream << "   Area: " << aFilter.fArea;
            aStream << std::endl;
        }
        // aStream << endl;  // calling method should add the final 'endl'!
    }

    return aStream;
}

//________________________________________________________ simplified
std::string simplified(const std::string& str) {
    std::size_t pos1 = str.find_first_not_of(" ");
    std::size_t pos2 = str.find_last_not_of(" ");

    if (pos1 != std::string::npos && pos2 != std::string::npos && pos2 >= pos1) {
        return str.substr(pos1, pos2 - pos1 + 1);
    }
    return std::string();
}
