/***************************** LICENSE START ***********************************

 Copyright 2012 ECMWF and INPE. This software is distributed under the terms
 of the Apache License version 2.0. In applying this license, ECMWF does not
 waive the privileges and immunities granted to it by virtue of its status as
 an Intergovernmental Organization or submit itself to any jurisdiction.

 ***************************** LICENSE END *************************************/

#include <stdio.h>

#include <cstdio>
#include <fstream>
#include <iostream>
#include <sstream>
#include <string>

#include "fstream_mars_fix.h"

#include "BufrFilterDef.h"
#include "BufrLocationCollector.h"
#include "BufrMetaData.h"
#include "LogHandler.h"
#include "MvKeyProfile.h"
#include "MvMiscellaneous.h"
#include "MvEccBufr.h"
#include "MvTmpFile.h"
#include "MvWindef.h"

#include "eccodes.h"

#define MV_CODES_CHECK(a, msg) MvEccBufr::codesCheck(#a, __FILE__, __LINE__, a, msg)

static std::string BUFR_DEFINITION_PATH_ENV;
static std::string BUFR_DUMP_EXE;
static std::string BUFR_FILTER_EXE;
static std::string BUFR_COPY_EXE;

#if 0
static bool shellCommand(const std::string &command, std::stringstream& out,std::stringstream& err,
                         std::stringstream& otherErr, const std::string &ftmp,int& exitCode)
{
    FILE *in;
    char cbuf[1024];

    exitCode=-1;

    std::string cmd;
    cmd+=command + " 2>" + ftmp;

    if (!(in = popen(cmd.c_str() ,"r")) )
    {
        otherErr << "Failed to execute command: " << command;
        return false;
    }

    while(fgets(cbuf, sizeof(cbuf), in) != nullptr)
    {
        out << cbuf;
    }

    int ret=pclose(in);
    exitCode=WEXITSTATUS(ret);

    if (!(in = fopen(ftmp.c_str() ,"r")) )
    {
        otherErr << "Failed to read file " << ftmp << " containing STDERR of command";
        return false;
    }

    while(fgets(cbuf, sizeof(cbuf), in) != nullptr)
    {
        err << cbuf;
    }

    fclose(in);

    return true;
}
#endif

static bool shellCommand(const std::string& command, std::stringstream& out, std::stringstream& err,
                         std::stringstream& otherErr, int& exitCode)
{
    FILE* in;
    char cbuf[1024];
    MvTmpFile tmpFile;

    exitCode = -1;

    std::string cmd;
    cmd += command + " 2>" + tmpFile.path();

    if (!(in = popen(cmd.c_str(), "r"))) {
        otherErr << "Failed to execute command: " << command;
        return false;
    }

    while (fgets(cbuf, sizeof(cbuf), in) != nullptr) {
        out << cbuf;
    }

    int ret  = pclose(in);
    exitCode = WEXITSTATUS(ret);

    if (!(in = fopen(tmpFile.path().c_str(), "r"))) {
        otherErr << "Failed to read file " << tmpFile.path() << " containing STDERR of command";
        return false;
    }

    while (fgets(cbuf, sizeof(cbuf), in) != nullptr) {
        err << cbuf;
    }

    fclose(in);

    return true;
}

//=====================================
//
// BufrDataDump
//
//=====================================

BufrDataDump::~BufrDataDump()
{
    clear();
}

void BufrDataDump::clear()
{
    text_.clear();
    debug_.clear();
}

bool BufrDataDump::filterMessage(const std::string& inFile, int msgCnt, long offset, const std::string& outFile, std::string& errOut)
{
    std::string cmd;

    if (offset > -1) {
        cmd = BUFR_COPY_EXE + " -w count=1 -X " + std::to_string(offset) + " " + inFile + " " + outFile;
    }
    else {
        cmd = BUFR_COPY_EXE + " -w count=" + std::to_string(msgCnt) + " " + inFile + " " + outFile;
    }

    GuiLog().task() << "Filtering out message: " << msgCnt << GuiLog::commandKey() << cmd;

    //Run the command
    std::stringstream in, err;
    bool hasError = false;
    int exitCode;
    std::stringstream shellErr;
    bool ret = shellCommand(cmd, in, err, shellErr, exitCode);

    if (exitCode > 0) {
        hasError = true;
        GuiLog().error() << "Command exited with code: " << exitCode;
        if (err.str().empty())
            errOut += "<b>Command</b>" + cmd + " exited with <b>code:</b> " + std::to_string(exitCode) + " ";
    }

    if (!ret) {
        hasError = true;
        GuiLog().error() << shellErr.str();
        errOut += shellErr.str();
    }

    //Get stderr of command
    if (!err.str().empty()) {
        hasError = true;
        GuiLog().error() << err.str();
        errOut = "<b>Command </b>" + cmd + " <b>failed.</b> <br>" + err.str();
    }

    return !hasError;
}

bool BufrDataDump::filterSubset(const std::string& inFile, int msgCnt, int subsetCnt, const std::string& outFile, std::string& errOut)
{
    MvTmpFile rulesFile;

    std::string cmd = BUFR_FILTER_EXE + " -o " + outFile + " " + rulesFile.path() + " " + inFile;

    GuiLog().task() << "Filtering out subset: " << subsetCnt << " from message: " << msgCnt << GuiLog::commandKey() << cmd;

    //Create the filter
    std::ofstream out;
    out.open(rulesFile.path().c_str());
    if (!out.good()) {
        GuiLog().error() << "Cannot create temporary rules file: " << rulesFile.path();
        errOut = "Cannot create temporary rules file: " + rulesFile.path();
        return false;
    }

    out << "set unpack=1;" << std::endl;
    out << "set extractSubset=" << subsetCnt << ";" << std::endl;
    out << "set doExtractSubsets=1;" << std::endl;
    out << "if(count == " << msgCnt << " ) { write; } " << std::endl;
    out.close();

    //Run the filter
    std::stringstream in, err;
    bool hasError = false;
    int exitCode;
    std::stringstream shellErr;
    bool ret = shellCommand(cmd, in, err, shellErr, exitCode);

    if (exitCode > 0) {
        hasError = true;
        GuiLog().error() << "Command exited with code: " << exitCode;
        if (err.str().empty())
            errOut += "<b>Command</b>" + cmd + " exited with <b>code:</b> " + std::to_string(exitCode) + " ";
    }

    if (!ret) {
        hasError = true;
        GuiLog().error() << shellErr.str();
        errOut += shellErr.str();
    }

    //Get stderr of command
    if (!err.str().empty()) {
        hasError = true;
        GuiLog().error() << err.str();
        errOut = "<b>Command </b>" + cmd + " <b>failed.</b> <br>" + err.str();
    }

    return !hasError;
}

//msgCnt starts from 1!!!
bool BufrDataDump::read(const std::string& sourceFile, int msgCnt, int subsetCnt, int subsetNum,
                        bool compressed, long offset, std::string& errOut)
{
    std::stringstream in, err;
    std::string inFile;
    MvTmpFile subsetFile;
    std::string cmd;

    //from eccodes 2.9.0 we can specify the message offset to bufr_dump
    bool useOffset = (offset > -1 && codes_get_api_version() >= 20900);

    if (useOffset) {
        //For non-compressed messages we need to filter out the required subset
        if (!compressed && subsetNum > 1) {
            MvTmpFile msgFile;

            if (!filterMessage(sourceFile, msgCnt, offset, msgFile.path(), errOut))
                return false;
            if (!filterSubset(msgFile.path(), 1, subsetCnt, subsetFile.path(), errOut))
                return false;

            inFile = subsetFile.path();  //safe to do
            cmd    = BUFR_DUMP_EXE + " -ja  -w count=1 \"" + inFile + "\"";
        }
        else {
            inFile = sourceFile;
            cmd    = BUFR_DUMP_EXE + " -ja -w count=1 -X " + std::to_string(offset) + " " + "\"" + inFile + "\"";
        }
    }
    else {
        //For non-compressed messages we need to filter out the required subset
        if (!compressed && subsetNum > 1) {
            MvTmpFile msgFile;

            if (!filterMessage(sourceFile, msgCnt, -1, msgFile.path(), errOut))
                return false;
            if (!filterSubset(msgFile.path(), 1, subsetCnt, subsetFile.path(), errOut))
                return false;

            inFile = subsetFile.path();  //safe to do
            cmd    = BUFR_DUMP_EXE + " -ja  -w count=1 \"" + inFile + "\"";
        }
        else {
            inFile = sourceFile;
            cmd    = BUFR_DUMP_EXE + " -ja  -w count=" + std::to_string(msgCnt) + " " + "\"" + inFile + "\"";
        }
    }

    std::stringstream msg;
    if (!compressed && subsetNum > 1) {
        msg << "Generating json dump for message: " << msgCnt << " subset: " << subsetCnt;
    }
    else {
        msg << "Generating json dump for message: " << msgCnt;
    }

    GuiLog().task() << msg.str() << GuiLog::commandKey() << cmd;


    bool hasError = false;
    int exitCode;
    std::stringstream shellErr;
    bool ret = shellCommand(cmd, in, err, shellErr, exitCode);

    //Get stdout of command
    text_ = in.str();

    //For uncompressed subsets we need to set the correct numberOfSubsets
    //in the json file. At the moment it contains 1, because to access the subset
    //first we filter it out as a separate message and then run the bufr dump on it to
    //generate json. Since we run the dump on a single subset the numberOfSubsets
    //in the resulting dump will always be 1.
    if (!compressed && subsetNum > 1) {
        size_t pos = text_.find("numberOfSubsets");
        if (pos != std::string::npos) {
            pos = text_.find("value\" : 1", pos);
            if (pos != std::string::npos) {
                std::stringstream nosTxt;
                nosTxt << "value\" : " << subsetNum;
                text_.replace(pos, 10, nosTxt.str());
            }
        }
    }

    if (exitCode > 0) {
        hasError = true;
        GuiLog().error() << "Command exited with code: " << exitCode;
        if (err.str().empty())
            errOut += "<b>Command</b>" + cmd + " exited with <b>code:</b> " + std::to_string(exitCode) + " ";
    }

    if (!ret) {
        hasError = true;
        GuiLog().error() << shellErr.str();
        errOut += shellErr.str();
    }

    //Get stderr of command
    if (!err.str().empty()) {
        hasError = true;
        GuiLog().error() << err.str();
        errOut = "<b>Command </b>" + cmd + " <b>failed.</b> <br>" + err.str();
    }

    return !hasError;
}

bool BufrDataDump::debug(const std::string& sourceFile, int msgCnt, int /*subsetCnt*/, int subsetCnt, bool compressed, long offset, std::string& errOut)
{
    std::stringstream in, err;
    std::string inFile;
    MvTmpFile msgFile;
    MvTmpFile subsetFile;
    std::string cmd;

    bool useOffset = (offset > -1 && codes_get_api_version() >= 20900);

    if (useOffset) {
        //For non-compressed messages we need to filter out the required susbset
        if (!compressed && subsetCnt > 1) {
            //Filter out the given message
            if (!filterMessage(sourceFile, msgCnt, offset, msgFile.path(), errOut))
                return false;

            if (!filterSubset(msgFile.path(), 1, subsetCnt, subsetFile.path(), errOut))
                return false;

            inFile = subsetFile.path();
            cmd    = "export ECCODES_DEBUG=1; " + BUFR_DUMP_EXE + " -ja  -w count=1 \"" + inFile + "\"";
        }
        else {
            inFile = sourceFile;
            cmd    = "export ECCODES_DEBUG=1; " + BUFR_DUMP_EXE + " -ja  -w count=1 -X " + std::to_string(offset) + " " + "\"" + inFile + "\"";
        }
    }
    else {
        //Filter out the given message
        if (!filterMessage(sourceFile, msgCnt, -1, msgFile.path(), errOut))
            return false;

        //For non-compressed messages we need to filter out the required susbset
        if (!compressed && subsetCnt > 1) {
            if (!filterSubset(msgFile.path(), 1, subsetCnt, subsetFile.path(), errOut))
                return false;

            inFile = subsetFile.path();
        }
        else {
            inFile = msgFile.path();
        }

        cmd = "export ECCODES_DEBUG=1; " + BUFR_DUMP_EXE + " -ja  -w count=1 \"" + inFile + "\"";
    }

    GuiLog().task() << "Generating debug information for message: " << msgCnt << GuiLog::commandKey() << cmd;

    //Run command
    int exitCode;
    std::stringstream shellErr;
    bool ret = shellCommand(cmd, in, err, shellErr, exitCode);

    //Debud is dumped to stderr
    debug_ = err.str();

    //Remove unwanted strings from debug text
    std::string rs   = "ECCODES DEBUG   :  ";
    size_t start_pos = 0;
    while ((start_pos = debug_.find(rs, start_pos)) != std::string::npos) {
        debug_.replace(start_pos, rs.length(), "");
    }

    bool hasError = false;
    if (exitCode > 0) {
        if (debug_.empty()) {
            GuiLog().error() << "Command exited with code: " << exitCode;
            errOut += "Command exited with code: " + std::to_string(exitCode) + " ";
            hasError = true;
        }
    }
    if (!ret) {
        GuiLog().error() << shellErr.str();
        errOut += shellErr.str();
        hasError = true;
    }

    return !hasError;
}

bool BufrDataDump::runMessageFilter(const std::string& filterStr, const std::string& inFile,
                                    const std::string& outFile)
{
    MvTmpFile rulesFile;

    std::string cmd = BUFR_FILTER_EXE + " -o " + outFile + " " + rulesFile.path() + " " + inFile;

    //GuiLog().task() << "Filtering out subset: " << subsetCnt << " from message: " << msgCnt <<
    //              GuiLog::commandKey() << cmd;

    //Create the filter
    std::string errOut;
    std::ofstream out;
    out.open(rulesFile.path().c_str());
    if (!out.good()) {
        GuiLog().error() << "Cannot create temporary rules file: " << rulesFile.path();
        errOut = "Cannot create temporary rules file: " + rulesFile.path();
        return false;
    }

    //out << "set unpack=1;" << std::endl;
    //out << "set extractSubset=" << subsetCnt << ";" << std::endl;
    //out << "set doExtractSubsets=1;" << std::endl;
    //out << "if(count == " << msgCnt << " ) { write; } " << std::endl;
    out << filterStr;
    out.close();

    //Run the filter
    std::stringstream in, err;
    bool hasError = false;
    int exitCode;
    std::stringstream shellErr;
    bool ret = shellCommand(cmd, in, err, shellErr, exitCode);

    if (exitCode > 0) {
        hasError = true;
        GuiLog().error() << "Command exited with code: " << exitCode;
        if (err.str().empty())
            errOut += "<b>Command</b>" + cmd + " exited with <b>code:</b> " + std::to_string(exitCode) + " ";
    }

    if (!ret) {
        hasError = true;
        GuiLog().error() << shellErr.str();
        errOut += shellErr.str();
    }

    //Get stderr of command
    if (!err.str().empty()) {
        hasError = true;
        GuiLog().error() << err.str();
        errOut = "<b>Command </b>" + cmd + " <b>failed.</b> <br>" + err.str();
    }

    return !hasError;
}

//=====================================
//
// BufrExpandDataDump
//
//=====================================

BufrExpandDataDump::~BufrExpandDataDump()
{
    clear();
}

void BufrExpandDataDump::clear()
{
    text_.clear();
}

//msgCnt starts from 1!!!
bool BufrExpandDataDump::read(const std::string& sourceFile, int msgCnt, long offset, std::string& errOut)
{
    std::stringstream in, err;
    std::string inFile;
    std::string cmd;

    if (codes_get_api_version() < 21300) {
        errOut = "Expanded descriptor dump is only available in ecCodes versions > 2.13.0";
        return false;
    }


    inFile = sourceFile;
    cmd    = BUFR_DUMP_EXE + " -d -w count=1 -X " + std::to_string(offset) + " " + "\"" + inFile + "\"";

    std::stringstream msg;
    msg << "Generating expanded descriptor dump for message: " << msgCnt;
    GuiLog().task() << msg.str() << GuiLog::commandKey() << cmd;

    if (offset < 0) {
        errOut = "Invalid offset=" + std::to_string(offset) + " defined!";
        GuiLog().error() << errOut;
        return false;
    }

    bool hasError = false;
    int exitCode;
    std::stringstream shellErr;
    bool ret = shellCommand(cmd, in, err, shellErr, exitCode);

    //Get stdout of command
    text_ = in.str();

    if (exitCode > 0) {
        hasError = true;
        GuiLog().error() << "Command exited with code: " << exitCode;
        if (err.str().empty())
            errOut += "<b>Command</b>" + cmd + " exited with <b>code:</b> " + std::to_string(exitCode) + " ";
    }

    if (!ret) {
        hasError = true;
        GuiLog().error() << shellErr.str();
        errOut += shellErr.str();
    }

    //Get stderr of command
    if (!err.str().empty()) {
        hasError = true;
        GuiLog().error() << err.str();
        errOut = "<b>Command </b>" + cmd + " <b>failed.</b> <br>" + err.str();
    }

    return !hasError;
}


//=====================================
//
// BufrMetaData
//
//=====================================

BufrMetaData::BufrMetaData() :
    MvMessageMetaData(BufrType)
{
#if 0
    std::string cmd;

    GRIB_DEFINITION_PATH_ENV="";
    const char* grb_def_path=getenv("GRIB_DEFINITION_PATH");
    if(grb_def_path)
    {
        GRIB_DEFINITION_PATH_ENV=string(grb_def_path);
    }
#endif

    //Define the commnad line tools
#ifdef ECCODES_UI
    char* bdexe = getenv("CODES_UI_BUFR_DUMP");
    char* bfexe = getenv("CODES_UI_BUFR_FILTER");
    char* bcexe = getenv("CODES_UI_BUFR_COPY");
#else
    char* bdexe = getenv("METVIEW_BUFR_DUMP");
    char* bfexe = getenv("METVIEW_BUFR_FILTER");
    char* bcexe = getenv("METVIEW_BUFR_COPY");
#endif

    if (bdexe) {
        BUFR_DUMP_EXE = bdexe;
    }
    else {
        BUFR_DUMP_EXE = "bufr_dump";
    }

    if (bfexe) {
        BUFR_FILTER_EXE = bfexe;
    }
    else {
        BUFR_FILTER_EXE = "bufr_filter";
    }

    if (bcexe) {
        BUFR_COPY_EXE = bcexe;
    }
    else {
        BUFR_COPY_EXE = "bufr_copy";
    }
}

BufrMetaData::BufrMetaData(const std::string& fileName, int totalMessageNum) :
    MvMessageMetaData(BufrType),
    totalSubsetNum_(0)
{
    fileName_        = fileName;
    totalMessageNum_ = totalMessageNum;
    messageNum_      = totalMessageNum_;
}

BufrMetaData::~BufrMetaData()
{
    clear();
}

void BufrMetaData::clearData()
{
    for (auto& message : messages_) {
        delete message;
    }
    messages_.clear();
    totalSubsetNum_ = 0;
    MvMessageMetaData::clearData();
}

void BufrMetaData::clear()
{
    for (auto& message : messages_) {
        delete message;
    }
    messages_.clear();
    firstTypicalDate_.clear();
    firstKeys_.clear();
    totalSubsetNum_ = 0;

    MvMessageMetaData::clear();
}

int BufrMetaData::computeTotalMessageNum()
{
    FILE* fp;
    fp = fopen(fileName_.c_str(), "rb");
    if (!fp) {
        //log->error("GribMetaData::readMessages() ---> Cannot open grib file: \n        " + fileName_ + "\n");
        return 0;
    }

    int res = 0;
    if (codes_count_in_file(0, fp, &res) != CODES_SUCCESS)
        res = 0;

    fclose(fp);
    return res;
}

void BufrMetaData::setTotalMessageNum(int num)
{
    totalMessageNum_ = num;
    messageNum_      = totalMessageNum_;
    allocateMessages();
}

size_t BufrMetaData::totalSubsetNum(bool forceCompute)
{
    if (totalSubsetNum_ == 0 || forceCompute) {
        totalSubsetNum_ = 0;
        for (auto msg : messages_) {
            if (msg && msg->subsetNum() > 0) {
                totalSubsetNum_ += static_cast<size_t>(msg->subsetNum());
            }
        }
    }
    return totalSubsetNum_;
}

void BufrMetaData::getKeyList(std::string section, MvKeyProfile* prof)
{
    for (auto& it : *allKey_) {
        if (it->metaData("section") == section) {
            prof->addKey(it->clone());
        }
    }
}

void BufrMetaData::setFileName(std::string fname)
{
    clear();

    fileName_ = fname;
    //totalMessageNum_=computeTotalMessageNum();
    //messageNum_=totalMessageNum_;
}

void BufrMetaData::setMessage(size_t chunkStart, const std::vector<MvEccBufrMessage*>& msgVec)
{
    assert(messages_.size() > chunkStart);
    assert(messages_.size() >= chunkStart + msgVec.size());
    for (size_t i = 0; i < msgVec.size(); i++) {
        size_t pos = i + chunkStart;
        assert(messages_[pos] == nullptr);
        messages_[pos] = msgVec[i];
        messages_[pos]->setFileInfo(this);
    }
}

void BufrMetaData::setMessage(size_t pos, MvEccBufrMessage* msg)
{
    assert(messages_.size() > pos);
    assert(messages_[pos] == nullptr);
    messages_[pos] = msg;
    messages_[pos]->setFileInfo(this);
}

MvEccBufrMessage* BufrMetaData::message(int index) const
{
    assert(index >= 0 && index < static_cast<int>(messages_.size()));
    return messages_.at(index);
}

void BufrMetaData::allocateMessages()
{
    assert(messages_.empty());
    if (totalMessageNum_ > 0)
        messages_ = std::vector<MvEccBufrMessage*>(totalMessageNum_, 0);
}

const std::set<std::string>& BufrMetaData::firstMessageKeys()
{
    if (!firstKeys_.empty())
        return firstKeys_;

    readMessageKeys(0, firstKeys_);
    return firstKeys_;
}

void BufrMetaData::readMessageKeys(int index, std::set<std::string>& keys)
{
    if (!(index >= 0 && index < static_cast<int>(messages_.size())))
        return;

    FILE* fp;
    codes_handle* ch = nullptr;
    int err          = 0;
    int bufr_count   = 0;

    std::string cbuff;

    GuiLog().task() << "Collecting bufr keys from message " << index << GuiLog::methodKey() << "ecCodes C interface";

    //open bufr file for reading
    fp = fopen(fileName_.c_str(), "rb");
    if (!fp) {
        GuiLog().error() << "BufrMetaData::readMessageKeys() ---> Cannot open bufr file: \n        " << fileName_;
        return;
    }

    if (fseek(fp, messages_[index]->offset(), 0) != 0) {
        fclose(fp);
        return;
    }

    //fseek(fp,messages_[index]->offset(),SEEK_SET);

    //Get messages form the file
    while ((ch = codes_handle_new_from_file(nullptr, fp, PRODUCT_BUFR, &err)) != nullptr || err != CODES_SUCCESS) {
        if (!ch) {
            GuiLog().error() << "BufrMetaData::readFirstMessageKeys() --->  Unable to create code handle for message count: " << bufr_count + 1;
            continue;
        }
        else {
            bufr_keys_iterator* kiter = nullptr;
            char* name                = 0;

            codes_set_long(ch, "skipExtraKeyAttributes", 1);
            codes_set_long(ch, "unpack", 1);

            kiter = codes_bufr_data_section_keys_iterator_new(ch);
            if (!kiter) {
                codes_handle_delete(ch);
                continue;
            }

            while (codes_bufr_keys_iterator_next(kiter)) {
                if ((name = codes_bufr_keys_iterator_get_name(kiter))) {
                    std::string nameStr(name);

                    if (!nameStr.empty() && nameStr.find("->") == std::string::npos) {
                        if (nameStr[0] == '#') {
                            std::size_t ipos = nameStr.find("#", 1);
                            if (ipos != std::string::npos)
                                nameStr = nameStr.substr(ipos + 1);
                        }

                        keys.insert(nameStr);
                    }
                }
            }

            codes_handle_delete(ch);
            break;
        }
    }

    fclose(fp);
}

void BufrMetaData::getKeyProfileForFirstMessage(MvKeyProfile* prof)
{
    prof->clearKeyData();

    FILE* fp;
    codes_handle* ch = nullptr;
    int err          = 0;

    //open bufr file for reading
    fp = fopen(fileName_.c_str(), "rb");
    if (!fp) {
        //GuiLog().error() << "BufrMetaData::readMessageKeys() ---> Cannot open bufr file: \n        " << fileName_;
        return;
    }

    //Get messages form the file
    while ((ch = codes_handle_new_from_file(nullptr, fp, PRODUCT_BUFR, &err)) != nullptr || err != CODES_SUCCESS) {
        if (!ch) {
            //GuiLog().error() << "BufrMetaData::readFirstMessageKeys() --->  Unable to create code handle for message count: " << bufr_count + 1;
            continue;
        }
        else {
            readMessage(prof, ch);
            codes_handle_delete(ch);
            fclose(fp);
            return;
        }
    }

    fclose(fp);
}

void BufrMetaData::readMessage(MvKeyProfile* prof, codes_handle* ch)
{
    if (!ch) {
        //readMessageFailed(prof, msgCnt, profIndex);
        return;
    }

    const int MAX_VAL_LEN = 1024;
    long longValue;
    char value[MAX_VAL_LEN];
    std::string svalue;
    size_t vlen = MAX_VAL_LEN;

    //long centre;
    // if (MV_CODES_CHECK(codes_get_long(ch, "bufrHeaderCentre", &centre), 0) == false) {
    //     //readMessageFailed(prof, msgCnt, profIndex);
    //     return;
    // }

    // if (MV_CODES_CHECK(codes_get_long(ch, "section1Flags", &longValue), 0) == false) {
    //     //readMessageFailed(prof, msgCnt, profIndex);
    //     return;
    // }


    //bool hasLocal2 = (centre == 98 && longValue != 0);

    for (unsigned int i = 0; i < prof->size(); i++) {
        size_t len;
        int keyType;
        const char* name    = prof->at(i)->name().c_str();
        std::string strName = prof->at(i)->name();
        bool foundValue     = false;

        //std::size_t pos;
        // if ((pos = strName.find(":98")) == strName.size() - 3) {
        //     if (hasLocal2) {
        //         strName = strName.substr(0, strName.size() - 3);
        //         name    = strName.c_str();
        //     }
        //     else {
        //         prof->at(i)->setStringValue(profIndex, "N/A");
        //         continue;
        //     }
        // }

        if (grib_get_native_type(ch, name, &keyType) == CODES_SUCCESS &&
            grib_get_size(ch, name, &len) == CODES_SUCCESS &&
            (keyType == CODES_TYPE_STRING || keyType == CODES_TYPE_LONG ||
             keyType == CODES_TYPE_DOUBLE)) {
            if (keyType == CODES_TYPE_STRING || len == 1) {
                if ((prof->at(i)->readIntAsString() == false) &&
                    keyType == CODES_TYPE_LONG) {
                    if (codes_get_long(ch, name, &longValue) == CODES_SUCCESS) {
                        std::stringstream s;
                        s << longValue;
                        prof->at(i)->addValue(s.str().c_str());
                        foundValue = true;
                    }
                }
                else {
                    vlen = MAX_VAL_LEN;
                    if (codes_get_string(ch, name, value, &vlen) == CODES_SUCCESS) {
                        if (prof->at(i)->name() == "section1Flags") {
                            prof->at(i)->addValue((strcmp(value, "0") == 0) ? "0" : "1");
                        }
                        else {
                            prof->at(i)->addValue(value);
                        }
                        foundValue = true;
                    }
                }
            }
            else {
                std::stringstream s;
                s << len;
                svalue = "Array (" + s.str() + ")";
                prof->at(i)->addValue(svalue);
                foundValue = true;
            }
        }

        if (!foundValue) {
            if (prof->at(i)->name() != "MV_Index")
                prof->at(i)->addValue("N/A");
        }
    }
}


void BufrMetaData::readCompressedData(MvKeyProfile* prof, int index, int subsetNum)
{
    FILE* fp;
    codes_handle* ch = nullptr;
    int err          = 0;
    int bufr_count   = 0;

    assert(prof);
    assert(prof->size() == 0);

    std::string cbuff;

    GuiLog().task() << "Generating bufr key list for all the messages" << GuiLog::methodKey() << "ecCodes C interface";

    //open bufr file for reading
    fp = fopen(fileName_.c_str(), "rb");
    if (!fp) {
        GuiLog().error() << "BufrMetaData::readMessages() ---> Cannot open bufr file: \n        " << fileName_;
        return;
    }

    //For filtered datasets
    if (filterEnabled_ == true) {
    }
    else {
        //The first key will be the subset count
        auto* subsetKey = new MvKey("subset", "subset");
        subsetKey->setValueType(MvKey::IntType);
        subsetKey->setConstant(true);  //!!!!
        prof->addKey(subsetKey);

        //Get messages form the file
        while ((ch = codes_handle_new_from_file(nullptr, fp, PRODUCT_BUFR, &err)) != nullptr || err != CODES_SUCCESS) {
            if (!ch) {
                GuiLog().error() << "BufrMetaData::readMessages() --->  Unable to create code handle for message count: " << bufr_count + 1;
                //continue;
            }

            if (bufr_count == index) {
                readCompressedData(prof, ch);
                int n = prof->valueNum();
                if (n > 1 && n == subsetNum)
                    subsetKey->setIntRange(1, subsetNum);
                else
                    prof->clear();

                if (ch)
                    codes_handle_delete(ch);
                break;
            }

            bufr_count++;
            if (ch)
                codes_handle_delete(ch);
        }
    }

    fclose(fp);
}


void BufrMetaData::readCompressedData(MvKeyProfile* prof, codes_handle* ch)
{
    //int codes_bufr_copy_data(grib_handle* hin, grib_handle* hout)
    //{
    bufr_keys_iterator* kiter = nullptr;
    char* name                = 0;
    int nkeys                 = 0;
    int keyType               = 0;
    long longValue            = 0;
    long* longArray           = 0;
    size_t longArrayLen       = 0;
    double doubleValue        = 0.;
    double* doubleArray       = 0;
    size_t doubleArrayLen     = 0;
    char** strArray           = 0;
    size_t strArrayLen        = 0;
    size_t len;
    size_t maxStrLen = 128;

    if (ch == nullptr) {
        return;
    }

    codes_set_long(ch, "skipExtraKeyAttributes", 1);
    codes_set_long(ch, "unpack", 1);

    kiter = codes_bufr_data_section_keys_iterator_new(ch);
    if (!kiter)
        return;

    while (codes_bufr_keys_iterator_next(kiter)) {
        if ((name = codes_bufr_keys_iterator_get_name(kiter))) {
            std::string nameStr(name);
            auto* mvk = new MvKey(nameStr, nameStr);
            prof->addKey(mvk);

            if (codes_get_native_type(ch, name, &keyType) == CODES_SUCCESS &&
                codes_get_size(ch, name, &len) == CODES_SUCCESS) {
                if (keyType == CODES_TYPE_STRING) {
                    mvk->setValueType(MvKey::StringType);
                    if (len == 1) {
                        char* buf = new char[maxStrLen];
                        mvk->setConstant(true);
                        codes_get_string(ch, name, buf, &maxStrLen);
                        auto sv = std::string(buf);
                        checkStringValue(sv);
                        mvk->addStringValue(sv);
                        delete[] buf;
                    }
                    else if (len > 1) {
                        mvk->setConstant(false);
                        if (len != strArrayLen) {
                            if (strArray) {
                                for (size_t i = 0; i < strArrayLen; ++i) {
                                    free(strArray[i]);
                                }
                                free(strArray);
                            }
                            strArrayLen = len;
                            strArray    = (char**)malloc(len * sizeof(char*));
                            for (size_t i = 0; i < len; ++i) {
                                strArray[i] = new char[maxStrLen];
                            }
                        }

                        //Get all values
                        size_t itotal = maxStrLen * len;
                        codes_get_string_array(ch, name, strArray, &itotal);

                        for (size_t i = 0; i < len; ++i) {
                            auto sv = std::string(strArray[i]);
                            checkStringValue(sv);
                            mvk->addStringValue(sv);
                        }
                    }
                }
                else if (keyType == CODES_TYPE_LONG) {
                    mvk->setValueType(MvKey::IntType);
                    mvk->setIntMissingValue(CODES_MISSING_LONG);
                    if (len == 1) {
                        mvk->setConstant(true);
                        if (codes_get_long(ch, name, &longValue) == CODES_SUCCESS) {
                            mvk->addIntValue(longValue);
                        }
                    }
                    else if (len > 1) {
                        mvk->setConstant(false);
                        if (len != longArrayLen) {
                            if (longArray) {
                                free(longArray);
                            }
                            longArrayLen = len;
                            longArray    = (long*)malloc(len * sizeof(long));
                        }
                        assert(len > 1);
                        assert(longArray);
                        size_t itotal = len;
                        CODES_CHECK(codes_get_long_array(ch, name, longArray, &itotal), 0);
                        for (size_t i = 0; i < len; i++) {
                            int iv = longArray[i];
                            mvk->addIntValue(iv);
                        }
                    }
                }
                else if (keyType == CODES_TYPE_DOUBLE) {
                    mvk->setValueType(MvKey::DoubleType);
                    mvk->setDoubleMissingValue(CODES_MISSING_DOUBLE);
                    if (len == 1) {
                        mvk->setConstant(true);
                        if (codes_get_double(ch, name, &doubleValue) == CODES_SUCCESS) {
                            mvk->addDoubleValue(doubleValue);
                        }
                    }
                    else if (len > 1) {
                        mvk->setConstant(false);
                        if (len != doubleArrayLen) {
                            if (doubleArray) {
                                free(doubleArray);
                            }
                            doubleArrayLen = len;
                            doubleArray    = (double*)malloc(len * sizeof(double));
                        }
                        assert(len > 1);
                        assert(doubleArray);
                        size_t itotal = len;
                        CODES_CHECK(codes_get_double_array(ch, name, doubleArray, &itotal), 0);
                        for (size_t i = 0; i < len; i++)
                            mvk->addDoubleValue(doubleArray[i]);
                    }
                }
            }

            /* if the copy fails we want to keep copying without any error messages.
    This is because the copy can be between structures that are not
    identical and we want to copy what can be copied and skip what
    cannot be copied because is not in the output handle
    */
            //err=codes_copy_key(hin, hout, name, 0);
            //if (err==0) nkeys++;
            //grib_context_free(ch->context,name);
            //free(name);
            //name=0;
        }
    }

    if (nkeys > 0) {
        /* Do the pack if something was copied */
        //err=grib_set_long(hout, "pack", 1);
    }

    codes_bufr_keys_iterator_delete(kiter);

    if (longArray)
        free(longArray);
    if (doubleArray)
        free(doubleArray);

    if (strArray) {
        for (size_t i = 0; i < strArrayLen; ++i)
            free(strArray[i]);
    }
}

std::string BufrMetaData::formatDate(std::string y, std::string m, std::string d)
{
    std::string res = y;
    res.append((m.size() == 1) ? "0" + m : m);
    res.append((d.size() == 1) ? "0" + d : d);

    if (res.find("N/A") != std::string::npos)
        res = "N/A";

    return res;
}

std::string BufrMetaData::formatTime(std::string h, std::string m, std::string s)
{
    std::string res = (h.size() == 1) ? "0" + h : h;
    res.append((m.size() == 1) ? "0" + m : m);
    res.append((s.size() == 1) ? "0" + s : s);

    if (res.find("N/A") != std::string::npos)
        res = "N/A";

    return res;
}

//Read all locations in the BUFR file
bool BufrMetaData::readLocations(BufrLocationCollector* collector, std::string& errOut)
{
    FILE* fp;
    codes_handle* ch = nullptr;
    int err          = 0;
    int bufr_count   = 1;

    //assert(prof);
    //assert(prof->size() == 0);

    std::string cbuff;

    GuiLog().task() << "Scanning for locations" << GuiLog::methodKey() << "ecCodes C interface";

    //open bufr file for reading
    fp = fopen(fileName_.c_str(), "rb");
    if (!fp) {
        GuiLog().error() << "BufrMetaData::readLocations() ---> Cannot open bufr file: \n        " << fileName_;
        errOut += "Cannot open bufr file: " + fileName_;
        return false;
    }

    //For filtered datasets
    if (filterEnabled_ == true) {
    }
    else {
        while ((ch = codes_handle_new_from_file(nullptr, fp, PRODUCT_BUFR, &err)) != nullptr || err != CODES_SUCCESS) {
            if (!ch) {
                GuiLog().error() << "BufrMetaData::readLocations() --->  Unable to create code handle for message count: " << bufr_count + 1;
                //continue;
            }

            readMessageLocations(ch, bufr_count, collector);

            broadcast(&MvMessageMetaDataObserver::locationScanStepChanged, bufr_count);

            bufr_count++;
            if (ch)
                codes_handle_delete(ch);
        }
    }

    fclose(fp);

    return true;
}

//Reads the locations from a given message amd collect them into an MvKeyProfile
void BufrMetaData::readMessageLocations(codes_handle* ch, int msgCnt, BufrLocationCollector* collector)
{
    if (!ch) {
        //readMessageFailed(prof,msgCnt);
        return;
    }

    long longValue;

    //Header keys
    long centre = 0;
    if (MV_CODES_CHECK(codes_get_long(ch, "bufrHeaderCentre", &centre), 0) == false) {
        //readMessageFailed(prof,msgCnt);
        return;
    }

    if (MV_CODES_CHECK(codes_get_long(ch, "section1Flags", &longValue), 0) == false) {
        //readMessageFailed(prof,msgCnt);
        return;
    }

    long subsetNum = 0;
    if (MV_CODES_CHECK(codes_get_long(ch, "numberOfSubsets", &subsetNum), 0) == false) {
        //readMessageFailed(prof,msgCnt);
        return;
    }

    long compressed = 0;
    if (MV_CODES_CHECK(codes_get_long(ch, "compressedData", &compressed), 0) == false) {
        //readMessageFailed(prof,msgCnt);
        return;
    }

    bool hasLocal2 = (centre == 98 && longValue != 0);

    //For ECMWF bufrs with section 2 we read the lat/lon from the
    //header when there are no subsets
    if (hasLocal2 && subsetNum == 1) {
        double latVal, lonVal;
        bool hasLocalPos    = true;
        std::string latName = "localLatitude";
        std::string lonName = "localLongitude";
        if (MV_CODES_CHECK(codes_get_double(ch, latName.c_str(), &latVal), 0) == false) {
            hasLocalPos = false;
        }
        if (MV_CODES_CHECK(codes_get_double(ch, lonName.c_str(), &lonVal), 0) == false) {
            hasLocalPos = false;
        }

        if (hasLocalPos) {
            collector->add(msgCnt, 1, -1, latVal, lonVal);
            return;
        }
    }

    //Read the position from the data
    size_t latAllocLen = 0, lonAllocLen = 0;
    size_t latLen = 0, lonLen = 0;
    double* lonArray = 0;
    double* latArray = 0;

    //expand the message
    codes_set_long(ch, "skipExtraKeyAttributes", 1);
    if (codes_set_long(ch, "unpack", 1) != 0) {
        return;
    }

    //compressed data

    if (compressed) {
        int maxRank = 1000000;
        int ir      = 1;
        while (ir < maxRank) {
            double latVal, lonVal;
            latLen = 0;
            lonLen = 0;

            std::string latName = "#" + std::to_string(ir) + "#latitude";
            std::string lonName = "#" + std::to_string(ir) + "#longitude";

            //MV_CODES_CHECK(codes_get_size(ch, latName.c_str(), &latLen),0);
            //MV_CODES_CHECK(codes_get_size(ch, lonName.c_str(), &lonLen),0);

            codes_get_size(ch, latName.c_str(), &latLen);
            codes_get_size(ch, lonName.c_str(), &lonLen);

            if (latLen == 0 || lonLen == 0)
                break;

            //Single value
            if (latLen == 1) {
                MV_CODES_CHECK(codes_get_double(ch, latName.c_str(), &latVal), 0);
            }
            //Array
            else {
                readDoubleArray(ch, latName, latLen, latAllocLen, &latArray);
            }

            //Single value
            if (lonLen == 1) {
                MV_CODES_CHECK(codes_get_double(ch, lonName.c_str(), &lonVal), 0);
            }
            //Array
            else {
                readDoubleArray(ch, lonName, lonLen, lonAllocLen, &lonArray);
            }

            if (latLen != 1 && lonLen != 1 &&
                (latLen != lonLen || latLen != static_cast<size_t>(subsetNum)))
                break;

            for (int i = 0; i < subsetNum; i++) {
                collector->add(msgCnt, i + 1, ir,
                               (latLen == 1) ? latVal : latArray[i],
                               (lonLen == 1) ? lonVal : lonArray[i]);
            }
            ir++;
        }
    }

    //uncompressed
    else {
        //loop for the subsets
        for (int i = 0; i < subsetNum; i++) {
            latLen = 0;
            lonLen = 0;

            std::string latName = "/subsetNumber=" + std::to_string(i + 1) + "/latitude";
            std::string lonName = "/subsetNumber=" + std::to_string(i + 1) + "/longitude";

            //MV_CODES_CHECK(codes_get_size(ch, latName.c_str(), &latLen),0);
            //MV_CODES_CHECK(codes_get_size(ch, lonName.c_str(), &lonLen),0);

            codes_get_size(ch, latName.c_str(), &latLen);
            codes_get_size(ch, lonName.c_str(), &lonLen);

            if (latLen == lonLen && latLen > 0) {
                //Single value
                if (latLen == 1) {
                    double latVal, lonVal;
                    MV_CODES_CHECK(codes_get_double(ch, latName.c_str(), &latVal), 0);
                    MV_CODES_CHECK(codes_get_double(ch, lonName.c_str(), &lonVal), 0);

                    collector->add(msgCnt, i + 1, 1, latVal, lonVal);
                }
                //array
                else {
                    readDoubleArray(ch, latName, latLen, latAllocLen, &latArray);
                    readDoubleArray(ch, lonName, latLen, lonAllocLen, &lonArray);

                    for (size_t j = 0; j < latLen; j++) {
                        collector->add(msgCnt, i + 1, j + 1, latArray[j], lonArray[j]);
                    }
                }
            }
        }
    }

    if (latAllocLen > 0) {
        free(latArray);
    }
    if (lonAllocLen > 0) {
        free(lonArray);
    }
}


void BufrMetaData::readDoubleArray(codes_handle* ch, const std::string& key, std::size_t len, std::size_t& allocLen, double** data)
{
    if (len > allocLen) {
        if (*data) {
            free(*data);
        }
        allocLen = len;
        *data    = (double*)malloc(len * sizeof(double));
    }

    MV_CODES_CHECK(codes_get_double_array(ch, key.c_str(), *data, &len), 0);
}

void BufrMetaData::checkStringValue(std::string& t)
{
    static const std::string mStr = "missing";
    static const char chFF        = static_cast<char>(0xFF);

    if (t.length() > 0) {
        for (char i : t) {
            if (i != chFF) {
                return;
            }
        }
    }
    t = mStr;
}
