/*
    This file is part of Msc-generator.
    Copyright (C) 2008-2024 Zoltan Turanyi
    Distributed under GNU Affero General Public License.

    Msc-generator is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    Msc-generator is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with Msc-generator.  If not, see <http://www.gnu.org/licenses/>.
*/
/** @file utf8utils.cpp Utilities for UTF
 * @ingroup libcgencommon_files  */


#include <cstdint>
#include "utf8utils.h"
#include "contour_basics.h" //for FALLTHROUGH

/*UTF conversion routines were copied from llvm, the license is below.
*
* Copyright 2001-2004 Unicode, Inc.
*
* Disclaimer
*
* This source code is provided as is by Unicode, Inc. No claims are
* made as to fitness for any particular purpose. No warranties of any
* kind are expressed or implied. The recipient agrees to determine
* applicability of information provided. If this file has been
* purchased on magnetic or optical media from Unicode, Inc., the
* sole remedy for any claim will be exchange of defective media
* within 90 days of receipt.
*
* Limitations on Rights to Redistribute This Code
*
* Unicode, Inc. hereby grants the right to freely use the information
* supplied in this file in the creation of products supporting the
* Unicode Standard, and to make copies of this file in any form
* for internal or external distribution as long as this notice
* remains attached.
*/

/* Or we can replace them with a windows specific version
 * http://download.microsoft.com/download/4/8/4/484CE61B-A33A-4C62-B6BE-EB8DF1F07D5A/Code_DicanioUnicode0916.zip
 * or from https://msdn.microsoft.com/magazine/0916magcode
 */

/* Some fundamental constants */
#define UNI_REPLACEMENT_CHAR (uint32_t)0x0000FFFD
#define UNI_MAX_BMP (uint32_t)0x0000FFFF
#define UNI_MAX_UTF16 (uint32_t)0x0010FFFF

#define UNI_MAX_UTF8_BYTES_PER_CODE_POINT 4

#define UNI_UTF16_BYTE_ORDER_MARK_NATIVE  0xFEFF
#define UNI_UTF16_BYTE_ORDER_MARK_SWAPPED 0xFFFE


static const uint32_t halfBase = 0x0010000UL;
static const uint32_t halfMask = 0x3FFUL;
static const int halfShift = 10; /* used for shifting by 10 bits */

#define UNI_SUR_HIGH_START  0xD800UL
#define UNI_SUR_HIGH_END    0xDBFFUL
#define UNI_SUR_LOW_START   0xDC00UL
#define UNI_SUR_LOW_END     0xDFFFUL

/** Index into the table below with the first byte of a UTF-8 sequence to
 * get the number of trailing bytes that are supposed to follow it.
 * Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is
 * left as-is for anyone who may want to do such conversion, which was
 * allowed in earlier algorithms.*/
const unsigned char numTrailingBytesForUTF8[256] = {
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
    2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5
};

/** Magic values subtracted from a buffer value during UTF8 conversion.
* This table contains as many values as there might be trailing bytes
* in a UTF-8 sequence.
*/
static const uint32_t offsetsFromUTF8[6] = {0x00000000UL, 0x00003080UL, 0x000E2080UL,
0x03C82080UL, 0xFA082080UL, 0x82082080UL};

/** Once the bits are split out into bytes of UTF-8, this is a mask OR-ed
* into the first byte, depending on how many bytes follow.  There are
* as many entries in this table as there are UTF-8 sequence types.
* (I.e., one byte sequence, two byte... etc.). Remember that sequencs
* for *legal* UTF-8 will be 4 or fewer bytes total.
*/
static const unsigned char firstByteMark[7] = {0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC};


/** Utility routine to tell whether a sequence of bytes is legal UTF-8.
* This must be called with the length pre-determined by the first byte.
* The sequence is illegal right away if there aren't that many bytes
* available.
* If presented with a length > 4, this returns false.  The Unicode
* definition of UTF-8 goes up to 4-byte sequences*/
bool isLegalUTF8Char(const uint8_t* source, int length) noexcept
{
    uint8_t a;
    const uint8_t *srcptr = source+length;
    switch (length) {
    default: return false;
        /* Everything else falls through when "true"... */
    case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; FALLTHROUGH;
    case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; FALLTHROUGH;
    case 2: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; //FALLTHROUGH

        switch (*source) {
            /* no fall-through in this inner switch */
        case 0xE0: if (a < 0xA0) return false; break;
        case 0xED: if (a > 0x9F) return false; break;
        case 0xF0: if (a < 0x90) return false; break;
        case 0xF4: if (a > 0x8F) return false; break;
        default:   if (a < 0x80) return false;
        }
        FALLTHROUGH;

    case 1: if (*source >= 0x80 && *source < 0xC2) return false;
    }
    if (*source > 0xF4) return false;
    return true;
}

bool IsValidUTF8(std::string_view utf8) noexcept
{
    for (auto i = utf8.begin(); i!=utf8.end(); /*nope*/) {
        int length = UTF8TrailingBytes(*i)+1;
        if (utf8.end() - i < length)
            return false;
        if (!isLegalUTF8Char((const uint8_t*)&*i, length))
            return false;
        i += length;
    }
    return true;
}

std::string ConvertFromUTF16_to_UTF8(std::string_view utf16)
{
    const uint16_t* source = (const uint16_t *)utf16.data();
    //truncate odd length
    size_t len = utf16.length()/2;
    //see if we start with a byte-order mark
    bool swap = false;
    if (len) {
        if (*source == UNI_UTF16_BYTE_ORDER_MARK_NATIVE) {
            source++;
            len--;
        } else if (*source == UNI_UTF16_BYTE_ORDER_MARK_SWAPPED) {
            source++;
            len--;
            swap = true;
        }
    }
    const uint16_t* sourceEnd = source+len;
    std::string ret;
    ret.reserve(4*len+1); //at most 4 bytes per char

    while (source < sourceEnd) {
        uint32_t ch;
        unsigned short bytesToWrite = 0;
        const uint32_t byteMask = 0xBF;
        const uint32_t byteMark = 0x80;
        ch = *source++;
        if (swap)
            ch = 0xffff & ((ch>>8) | (ch<<8)); //swap endianness
                                               /* If we have a surrogate pair, convert to uint32_t first. */
        if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) {
            /* If the 16 bits following the high surrogate are in the source buffer... */
            if (source < sourceEnd) {
                uint32_t ch2 = *source;
                /* If it's a low surrogate, convert to uint32_t. */
                if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {
                    ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
                        + (ch2 - UNI_SUR_LOW_START) + halfBase;
                    ++source;
                }
            } else  /* We don't have the 16 bits following the high surrogate. */
                break;
        }
        /* Figure out how many bytes the result will require */
        if (ch < (uint32_t)0x80) {
            bytesToWrite = 1;
        } else if (ch < (uint32_t)0x800) {
            bytesToWrite = 2;
        } else if (ch < (uint32_t)0x10000) {
            bytesToWrite = 3;
        } else if (ch < (uint32_t)0x110000) {
            bytesToWrite = 4;
        } else {
            bytesToWrite = 3;
            ch = UNI_REPLACEMENT_CHAR;
        }

        char buff[4];
        switch (bytesToWrite) { /* note: everything falls through. */
        case 4: buff[3] = (uint8_t)((ch | byteMark) & byteMask); ch >>= 6; FALLTHROUGH;
        case 3: buff[2] = (uint8_t)((ch | byteMark) & byteMask); ch >>= 6; FALLTHROUGH;
        case 2: buff[1] = (uint8_t)((ch | byteMark) & byteMask); ch >>= 6; FALLTHROUGH;
        case 1: buff[0] = (uint8_t)(ch | firstByteMark[bytesToWrite]);
        }
        ret.insert(ret.end(), buff, buff+bytesToWrite);
    }
    return ret;
}

std::wstring ConvertFromUTF8_to_UTF16(std::string_view utf8, bool include_BOM)
{
    const uint8_t* source = (const uint8_t*)utf8.data();
    const uint8_t* sourceEnd = source+utf8.length();
    std::wstring ret;
    ret.reserve(utf8.length()+2); //add two chars (4 bytes) for BOM and trailing zero
                                
    if (include_BOM)
        ret.push_back(UNI_UTF16_BYTE_ORDER_MARK_NATIVE);

    while (source < sourceEnd) {
        uint32_t ch = 0;
        unsigned short extraBytesToRead = UTF8TrailingBytes(*source);
        if (extraBytesToRead >= sourceEnd - source)
            break;
        if (!isLegalUTF8Char(source, extraBytesToRead+1)) {
            //replace with UNI_REPLACEMENT_CHAR
            source += extraBytesToRead;
            ret.push_back(UNI_REPLACEMENT_CHAR);
            continue; //attempt to read on
        }
        /*
        * The cases all fall through. See "Note A" below.
        */
        switch (extraBytesToRead) {
        case 5: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */ FALLTHROUGH;
        case 4: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */ FALLTHROUGH;
        case 3: ch += *source++; ch <<= 6; FALLTHROUGH;
        case 2: ch += *source++; ch <<= 6; FALLTHROUGH;
        case 1: ch += *source++; ch <<= 6; FALLTHROUGH;
        case 0: ch += *source++;
        }
        ch -= offsetsFromUTF8[extraBytesToRead];

        if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */
                                 /* UTF-16 surrogate values are illegal in UTF-32 */
            if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END)
                ret.push_back(UNI_REPLACEMENT_CHAR);
            else
                ret.push_back((wchar_t)ch); /* normal case */
        } else if (ch > UNI_MAX_UTF16) {
            ret.push_back(UNI_REPLACEMENT_CHAR);
        } else {
            /* to is a character in range 0xFFFF - 0x10FFFF. */
            ch -= halfBase;
            ret.push_back(wchar_t((ch >> halfShift) + UNI_SUR_HIGH_START));
            ret.push_back(wchar_t((ch & halfMask) + UNI_SUR_LOW_START));
        }
    }
    return ret;
}

bool ConvertToUTF8(std::string &text, bool force_as_unicode)
{
    if (force_as_unicode || !IsValidUTF8(text)) {
        text = ConvertFromUTF16_to_UTF8(text);
        return true;
    }
    return false;
}



