/*
 *  $Id: sensolytics.c 29611 2026-03-17 10:31:06Z yeti-dn $
 *  Copyright (C) 2009-2025 David Necas (Yeti).
 *
 *  This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public
 *  License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any
 *  later version.
 *
 *  This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
 *  warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
 *  details.
 *
 *  You should have received a copy of the GNU General Public License along with this program; if not, write to the
 *  Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 */

/**
 * [FILE-MAGIC-FREEDESKTOP]
 * <mime-type type="application/x-sensolytics-spm">
 *   <comment>Sensolytics SPM data</comment>
 *   <magic priority="70">
 *     <match type="string" offset="0" value="# Sensolytics: "/>
 *   </magic>
 * </mime-type>
 **/

/**
 * [FILE-MAGIC-FILEMAGIC]
 * # Sensolytics
 * # Apparently any kind of EOL can appear.
 * 0 string \x23\x20Sensolytics:\x20
 * >&0 string 1.1\x0d Sensolytics line scan data
 * >&0 string 1.1\x0a Sensolytics line scan data
 * >&0 string 1.2\x0d Sensolytics image data
 * >&0 string 1.2\x0a Sensolytics image data
 * >&0 string 1.3\x0d Sensolytics potentiostat data
 * >&0 string 1.3\x0a Sensolytics potentiostat data
 * >&0 string 1.4\x0d Sensolytics frequency spectra data
 * >&0 string 1.4\x0a Sensolytics frequency spectra data
 * >&0 string 3.3\x0d Sensolytics linear sweep voltametry data
 * >&0 string 3.3\x0a Sensolytics linear sweep voltametry data
 * >&0 string 3.4\x0d Sensolytics differential pulse voltametry data
 * >&0 string 3.4\x0a Sensolytics differential pulse voltametry data
 * >&0 string 3.6\x0d Sensolytics pulse data
 * >&0 string 3.6\x0a Sensolytics pulse data
 * >&0 string 3.7\x0d Sensolytics amperometry data
 * >&0 string 3.7\x0a Sensolytics amperometry data
 * >&0 string 3.8\x0d Sensolytics AC measurement data
 * >&0 string 3.8\x0a Sensolytics AC measurement data
 * >&0 string 3.11\x0d Sensolytics frequency scan data
 * >&0 string 3.11\x0a Sensolytics frequency scan data
 * >&0 string 3.12\x0d Sensolytics line scan data
 * >&0 string 3.12\x0a Sensolytics line scan data
 * >&0 string 3.13\x0d Sensolytics 4D array scan data
 * >&0 string 3.13\x0a Sensolytics 4D array scan data
 **/

/**
 * [FILE-MAGIC-USERGUIDE]
 * Sensolytics DAT
 * .dat
 * Read Curvemap
 **/

#include "config.h"
#include <glib/gi18n-lib.h>
#include <string.h>
#include <stdlib.h>
#include <gwy.h>

#include "err.h"

/* Not a real magic header, but should catch the stuff */
#define MAGIC "# Sensolytics: "
#define MAGIC_SIZE (sizeof(MAGIC)-1)
#define EXTENSION ".dat"

#define Micrometer 1e-6

typedef struct {
    gint xres, yres;
    gdouble xreal, yreal;
    gdouble dx, dy;
    const gchar *xyunit;
} Dimensions;

typedef struct {
    gint *pixel_index;
    GwyNield *mask;
    gint ndata;
    gboolean add_mask;
} SensolyticsGridMap;

typedef struct {
    GwyField *dfield;
    GwyLawn *lawn;
    gdouble *data;
    gchar **strings;
    gint n;
    gint nalloc;
    const gchar *name;
    const gchar *unit;
    /* Only for lawns. */
    gchar *title;
    GwyDict *specmeta;
} SensolyticsChannel;

static gboolean       module_register         (void);
static gint           detect_file             (const GwyFileDetectInfo *fileinfo,
                                               gboolean only_name);
static GwyFile*       load_file               (const gchar *filename,
                                               GwyRunModeFlags mode,
                                               GError **error);
static GHashTable*    read_header             (gchar **p,
                                               GArray *channels,
                                               GError **error);
static gboolean       parse_channels          (GHashTable *hash,
                                               GArray *channels,
                                               gchar *line,
                                               GError **error);
static gboolean       find_image_dimensions   (GHashTable *hash,
                                               Dimensions *dimensions,
                                               GError **error);
static gboolean       read_data               (GArray *channels,
                                               gchar **p,
                                               GError **error);
static void           map_to_grid             (GArray *channels,
                                               const Dimensions *dimensions,
                                               SensolyticsGridMap *map);
static void           create_fields           (GArray *channels,
                                               const Dimensions *dimensions,
                                               SensolyticsGridMap *map);
static void           create_lawns            (GArray *channels,
                                               const Dimensions *dimensions,
                                               SensolyticsGridMap *map,
                                               const gchar *parent_filename);
static GwyGraphModel* create_graph_model      (SensolyticsChannel *abscissa,
                                               SensolyticsChannel *ordinate);
static GwyDict*  get_meta                (GHashTable *hash);
static void           clone_field_meta        (GwyFile *file,
                                               GwyDict *meta,
                                               gint id);
static void           clone_lawn_meta         (GwyFile *file,
                                               GwyDict *meta,
                                               GwyDict *specmeta,
                                               gint id);
static gboolean       check_coordinate_columns(GArray *channels,
                                               gint required_value_cols,
                                               GError **error);
static void           clear_channels          (GArray *channels);

static GwyModuleInfo module_info = {
    GWY_MODULE_ABI_VERSION,
    &module_register,
    N_("Imports Sensolytics text files."),
    "Yeti <yeti@gwyddion.net>",
    "0.5",
    "David Nečas (Yeti)",
    "2009",
};

GWY_MODULE_QUERY2(module_info, sensolytics)

static gboolean
module_register(void)
{
    gwy_file_func_register("sensolytics",
                           N_("Sensolytics text files (.dat)"),
                           detect_file, load_file, NULL, NULL);

    return TRUE;
}

static gint
detect_file(const GwyFileDetectInfo *fileinfo, gboolean only_name)
{
    guint i, iver, len = fileinfo->buffer_len;
    const guchar *head = fileinfo->head;

    if (only_name)
        return g_str_has_suffix(fileinfo->name_lowercase, EXTENSION) ? 10 : 0;

    if (len < MAGIC_SIZE + 2 || memcmp(head, MAGIC, MAGIC_SIZE) != 0)
        return 0;

    /* Allow variable white space after the colon. */
    iver = MAGIC_SIZE;
    while (iver < len && head[iver] == ' ')
        iver++;
    /* Then accept a sequence of digits and dots. */
    i = iver;
    if (i == len || !g_ascii_isdigit(head[i]))
        return 0;
    while (i < len && (g_ascii_isdigit(head[i]) || head[i] == '.'))
        i++;
    /* Then there must be white space. */
    if (i == iver || i == len || !g_ascii_isspace(head[i]))
        return 0;

    return 100;
}

static gdouble
lawn_reduce_avg(gint ncurves, gint curvelength, const gdouble *curvedata, gpointer user_data)
{
    guint i, idx = GPOINTER_TO_UINT(user_data);
    gdouble s = 0.0;

    g_return_val_if_fail(idx < ncurves, 0.0);
    if (!curvelength)
        return s;

    curvedata += idx*curvelength;
    for (i = 0; i < curvelength; i++)
        s += curvedata[i];
    return s/curvelength;
}

static GwyFile*
load_file(const gchar *filename,
          G_GNUC_UNUSED GwyRunModeFlags mode,
          GError **error)
{
    GwyFile *file = NULL;
    GwyDict *meta = NULL;
    gchar *buffer = NULL;
    GError *err = NULL;
    GHashTable *hash = NULL;
    gchar *p, *experiment_type;
    GArray *channels = NULL;
    SensolyticsChannel *channel, *channel2;
    SensolyticsGridMap map;
    GwyGraphModel *gmodel;
    GwyField *preview;
    Dimensions dimensions;
    guint i, abscissa, startfrom;
    gint j, id;

    if (!g_file_get_contents(filename, &buffer, NULL, &err)) {
        err_GET_FILE_CONTENTS(error, &err);
        return NULL;
    }

    gwy_clear1(map);
    channels = g_array_new(FALSE, TRUE, sizeof(SensolyticsChannel));
    p = buffer;
    if (!(hash = read_header(&p, channels, error)))
        goto end;
    if (!channels->len) {
        err_NO_DATA(error);
        goto end;
    }
    if (!read_data(channels, &p, error))
        goto end;

    id = 0;
    experiment_type = g_hash_table_lookup(hash, "Experiment code");
    if (gwy_stramong(experiment_type, "1.2", "3.13", NULL)) {
        if (!find_image_dimensions(hash, &dimensions, error))
            goto end;
        if (!check_coordinate_columns(channels, 6, error))
            goto end;
        map_to_grid(channels, &dimensions, &map);
        create_fields(channels, &dimensions, &map);
        create_lawns(channels, &dimensions, &map, filename);
        meta = get_meta(hash);

        file = gwy_file_new_in_construction();
        for (i = 0; i < channels->len; i++) {
            channel = &g_array_index(channels, SensolyticsChannel, i);
            if (!channel->dfield)
                continue;
            gwy_file_set_image(file, id, channel->dfield);
            gwy_check_nonsquare_image(file, id);
            if (map.add_mask)
                gwy_file_pass_image_mask(file, id, gwy_nield_copy(map.mask));
            if (channel->name)
                gwy_file_set_title(file, GWY_FILE_IMAGE, id, channel->name, FALSE);
            else
                gwy_image_title_fall_back(file, id);
            gwy_log_add_import(file, GWY_FILE_IMAGE, id, NULL, filename);
            clone_field_meta(file, meta, id);
            id++;
        }

        id = 0;
        for (i = 0; i < channels->len; i++) {
            channel = &g_array_index(channels, SensolyticsChannel, i);
            if (!channel->lawn)
                continue;
            gwy_file_set_cmap(file, id, channel->lawn);
            if (channel->title)
                gwy_file_set_title(file, GWY_FILE_CMAP, id, channel->title, FALSE);
            preview = gwy_field_new(dimensions.xres, dimensions.yres, dimensions.xreal, dimensions.yreal, FALSE);
            j = gwy_lawn_get_n_curves(channel->lawn) - 1;
            gwy_lawn_reduce_to_plane(channel->lawn, preview, lawn_reduce_avg, GINT_TO_POINTER(j));
            gwy_unit_assign(gwy_field_get_unit_z(preview), gwy_lawn_get_unit_curve(channel->lawn, j));
            gwy_dict_pass_object(GWY_DICT(file), gwy_file_key_cmap_picture(id), preview);
            gwy_log_add_import(file, GWY_FILE_CMAP, id, NULL, filename);
            clone_lawn_meta(file, meta, channel->specmeta, id);
            id++;
        }

        g_object_unref(meta);
    }
    else {
        if (!check_coordinate_columns(channels, 1, error))
            goto end;

        abscissa = 0;
        startfrom = 1;
        if (channels->len > 2) {
            channel = &g_array_index(channels, SensolyticsChannel, 1);
            if (channel->data && gwy_strequal(channel->name, "Position rel.")) {
                abscissa = 1;
                startfrom = 2;
            }
        }
        channel = &g_array_index(channels, SensolyticsChannel, abscissa);
        if (!channel->data) {
            err_NO_DATA(error);
            goto end;
        }

        file = gwy_file_new_in_construction();
        for (i = startfrom; i < channels->len; i++) {
            channel2 = &g_array_index(channels, SensolyticsChannel, i);
            if (!channel2->data)
                continue;
            gmodel = create_graph_model(channel, channel2);
            gwy_file_pass_graph(file, id, gmodel);
            gwy_log_add_import(file, GWY_FILE_GRAPH, id, NULL, filename);
            id++;
        }
    }

end:
    g_free(buffer);
    if (hash)
        g_hash_table_destroy(hash);
    g_clear_object(&map.mask);
    g_free(map.pixel_index);
    if (channels) {
        clear_channels(channels);
        g_array_free(channels, TRUE);
    }

    return file;
}

static GHashTable*
read_header(gchar **p, GArray *channels, GError **error)
{
    gchar *line, *value, *opening_bracket, *experiment_type;
    GHashTable *hash;
    GString *str;

    line = gwy_str_next_line(p);
    g_strstrip(line);
    if (!g_str_has_prefix(line, MAGIC)) {
        err_FILE_TYPE(error, "Sensolytics");
        return NULL;
    }
    experiment_type = line + MAGIC_SIZE;
    g_strstrip(experiment_type);

    /* From all the 1.x types, we only know how to read images. */
    if (g_str_has_prefix(experiment_type, "1.")) {
        if (!gwy_strequal(experiment_type, "1.2")) {
            err_NO_DATA(error);
            return NULL;
        }
    }
    else if (!g_str_has_prefix(experiment_type, "3.")) {
        err_NO_DATA(error);
        return NULL;
    }

    str = g_string_new(NULL);
    hash = g_hash_table_new(g_str_hash, g_str_equal);
    g_hash_table_insert(hash, "Experiment code", experiment_type);
    for (line = gwy_str_next_line(p); line; line = gwy_str_next_line(p)) {
        g_strstrip(line);
        if (!line[0])
            continue;

        /* FIXME: If we miss the column headers this will produce a warning for every data line in the file. */
        if (line[0] != '#') {
            g_warning("Comment line does not start with #.");
            continue;
        }

        do {
            line++;
        } while (g_ascii_isspace(*line));

        /* The channel line looks like X [µm], Y [µm], etc.
         * XXX: We might als look for the initial ‘X [’ because it should be the first column. */
        if (strstr(line, "],")) {
            if (!parse_channels(hash, channels, line, error)) {
                g_hash_table_destroy(hash);
                hash = NULL;
            }
            /* In any case, that concludes the header. */
            goto end;
        }

        value = strchr(line, ':');
        if (!value) {
            value = line;
            line = "Experiment type";
        }
        else {
            /* Move units from "Key [units]: value" to value to make it easier for us to parse later. */
            if (value > line
                && *(value - 1) == ']'
                && (opening_bracket = strstr(line, " ["))
                && opening_bracket+2 <= value-1) {
                gwy_debug("before transformation <%s>", line);
                g_string_assign(str, line);
                g_string_erase(str, opening_bracket - line, value - opening_bracket);
                g_string_append_c(str, ' ');
                g_string_append_len(str, opening_bracket+2, (value-1) - (opening_bracket+2));
                /* We removed [ and ] so str must be shorter and can be safely copied back. */
                memcpy(line, str->str, str->len+1);
                value = strchr(line, ':');
                gwy_debug("after transformation <%s>", line);
                g_assert(value);
            }
            *value = '\0';
            g_strchomp(line);
            do {
                value++;
            } while (g_ascii_isspace(*value));

            if (gwy_strequal(line, "Warning"))
                continue;
        }

        gwy_debug("<%s>=<%s>", line, value);
        g_hash_table_insert(hash, line, value);
    }

end:
    g_string_free(str, TRUE);

    return hash;
}

/* This function makes sure channels->len is the real number of channels (including the ignored like X and Y),
 * no matter the file format version. */
static gboolean
parse_channels(GHashTable *hash, GArray *channels, gchar *line,
               GError **error)
{
    SensolyticsChannel channel;
    GString *str;
    gchar *p = line;
    const gchar *experiment_type;
    gint i, ncols;

    /* Parse the channel header line. But note that it is rubbish for 1.x files for which we read the standalone
     * header lines like ‘Channel 1 Name’ and ‘Channel 1 Unit’. We simply do afterwards, overwriting the rubbish
     * from the header. */
    do {
        gchar *unit, *end, *comma;

        if ((comma = strchr(p, ',')))
            *comma = '\0';

        if ((unit = strchr(p, '['))) {
            *unit = '\0';
            unit++;
            if ((end = strchr(unit+1, ']')))
                *end = '\0';
            g_strstrip(unit);
        }
        g_strstrip(p);

        gwy_clear1(channel);
        channel.name = p;
        channel.unit = unit;
        g_array_append_val(channels, channel);

        p = comma ? comma + 1 : NULL;
    } while (p);

    /* Old 1.x file? */
    experiment_type = g_hash_table_lookup(hash, "Experiment code");
    if (!g_str_has_prefix(experiment_type, "1."))
        return TRUE;

    if (!require_keys(hash, error, "Channels", NULL))
        return FALSE;
    ncols = atoi(g_hash_table_lookup(hash, "Channels"));
    if (ncols < 1 || channels->len <= 6) {
        err_NO_DATA(error);
        return FALSE;
    }
    if (ncols > 1024) {
        err_INVALID(error, "Channels");
        return FALSE;
    }

    /* Scrape it and start again with the real channels. */
    g_array_set_size(channels, 6);
    str = g_string_new(NULL);
    for (i = 0; i < ncols; i++) {
        gwy_clear1(channel);

        g_string_printf(str, "Channel %d Unit", i+1);
        channel.unit = g_hash_table_lookup(hash, str->str);
        g_string_printf(str, "Channel %d Name", i+1);
        channel.name = g_hash_table_lookup(hash, str->str);
        g_array_append_val(channels, channel);
    }
    g_string_free(str, TRUE);

    return TRUE;
}

static gdouble
parse_number_with_unit(GHashTable *hash, const gchar *key, const gchar **unitstr)
{
    const gchar *s;
    GwyUnit *siunit;
    gchar *end;
    gint power10;
    gdouble v;

    s = g_hash_table_lookup(hash, key);
    g_return_val_if_fail(s, 1.0);
    v = g_ascii_strtod(s, &end);
    sanitise_real_size(&v, key);
    while (g_ascii_isspace(*end))
        end++;
    if (unitstr)
        *unitstr = end;
    if (*end) {
        siunit = gwy_unit_new_parse(end, &power10);
        v *= gwy_exp10(power10);
        g_object_unref(siunit);
    }

    return v;
}

static gboolean
find_image_dimensions(GHashTable *hash, Dimensions *dimensions,
                      GError **error)
{
    const gchar *experiment_type;

    if (!require_keys(hash, error, "Experiment code", "X-Length", "Y-Length", NULL))
        return FALSE;

    experiment_type = g_hash_table_lookup(hash, "Experiment code");
    if (g_str_has_prefix(experiment_type, "1.")) {
        /* NB: If we ever wanted to read the increments for 1.x files, they are spelled Inkrement. */
        if (!require_keys(hash, error, "Lines", "Rows", NULL))
            return FALSE;

        dimensions->xreal = parse_number_with_unit(hash, "X-Length", NULL);
        dimensions->yreal = parse_number_with_unit(hash, "Y-Length", NULL);
        dimensions->dx = parse_number_with_unit(hash, "X-Inkrement", NULL);
        dimensions->dy = parse_number_with_unit(hash, "Y-Inkrement", NULL);
        /* Units are not given explicitly in 1.x, assume the default µm. */
        dimensions->xyunit = "m";
        dimensions->xreal *= Micrometer;
        dimensions->yreal *= Micrometer;
        dimensions->dx *= Micrometer;
        dimensions->dy *= Micrometer;

        dimensions->yres = atoi(g_hash_table_lookup(hash, "Lines"));
        /* XXX: When the file says Rows, it actually means Columns-1.  Bite me. */
        dimensions->xres = atoi(g_hash_table_lookup(hash, "Rows")) + 1;
    }
    else if (g_str_has_prefix(experiment_type, "3.")) {
        /* There are no array dimensions in 3.x files. Guess them using the increments. */
        if (!require_keys(hash, error, "X-Increment", "Y-Increment", NULL))
            return FALSE;

        /* We should check all the units are compatible. But if they are not it is not really our problem. */
        dimensions->xreal = parse_number_with_unit(hash, "X-Length", &dimensions->xyunit);
        dimensions->yreal = parse_number_with_unit(hash, "Y-Length", NULL);
        dimensions->dx = parse_number_with_unit(hash, "X-Increment", NULL);
        dimensions->dy = parse_number_with_unit(hash, "Y-Increment", NULL);

        gwy_debug("guessing array size from lengths (%g, %g) and steps (%g, %g)",
                  dimensions->xreal, dimensions->yreal, dimensions->dx, dimensions->dy);
        dimensions->xres = GWY_ROUND(fabs(dimensions->xreal/dimensions->dx)) + 1;
        dimensions->yres = GWY_ROUND(fabs(dimensions->yreal/dimensions->dy)) + 1;
    }
    else {
        g_assert_not_reached();
    }

    if (err_DIMENSION(error, dimensions->yres))
        return FALSE;
    if (err_DIMENSION(error, dimensions->xres))
        return FALSE;

    return TRUE;
}

static gboolean
err_MALFORMED_DATA(GError **error, guint k)
{
    g_set_error(error, GWY_MODULE_FILE_ERROR, GWY_MODULE_FILE_ERROR_DATA,
                _("Malformed data encountered when reading sample #%u"), k);
    return FALSE;
}

static gboolean
read_data(GArray *channels, gchar **p, GError **error)
{
    SensolyticsChannel *channel;
    gint j, G_GNUC_UNUSED lineno, power10, ncols = channels->len;
    gchar *line, *end;
    GwyUnit *unit;
    gdouble *q;

    unit = gwy_unit_new(NULL);
    q = g_new(gdouble, ncols);
    for (j = 0; j < ncols; j++) {
        channel = &g_array_index(channels, SensolyticsChannel, j);
        power10 = gwy_unit_set_from_string(unit, channel->unit);
        q[j] = gwy_exp10(power10);
    }
    g_object_unref(unit);

    lineno = 0;
    while ((line = gwy_str_next_line(p))) {
        lineno++;
        g_strstrip(line);
        if (!line[0] || line[0] == '#')
            continue;

        /* Analyse the first row. There does not seem to be any way to tell in advance which columns are data and
         * which are references to files. */
        for (j = 0; j < ncols; j++) {
            channel = &g_array_index(channels, SensolyticsChannel, j);
            /* String references to files (or other, weirder stuff). The first columns must be coordinates. However,
             * the number of coordinate columns differs between 1D data (1 or 2) and 2D data (6). We must do a sanity
             * check later. */
            if (line[0] == '"') {
                line++;
                end = strchr(line, '"');
                if (!end)
                    return err_MALFORMED_DATA(error, channel->n+1);
                *end = '\0';
                if (!channel->n) {
                    gwy_debug("column %u is string", j);
                    channel->nalloc = 16;
                    channel->strings = g_new(gchar*, channel->nalloc);
                }
                if (!channel->strings)
                    return err_MALFORMED_DATA(error, channel->n+1);
                if (channel->n == channel->nalloc) {
                    channel->nalloc *= 2;
                    channel->strings = g_renew(gchar*, channel->strings, channel->nalloc);
                }
                channel->strings[channel->n++] = line;
                line = end+1;
            }
            else if (g_ascii_isdigit(line[0]) || line[0] == '-' || line[0] == '+' || line[0] == '.') {
                if (!channel->n) {
                    gwy_debug("column %u is data", j);
                    channel->nalloc = 16;
                    channel->data = g_new(gdouble, channel->nalloc);
                }
                if (!channel->data)
                    return err_MALFORMED_DATA(error, channel->n+1);
                if (channel->n == channel->nalloc) {
                    channel->nalloc *= 2;
                    channel->data = g_renew(gdouble, channel->data, channel->nalloc);
                }
                channel->data[channel->n++] = q[j]*g_ascii_strtod(line, &end);
                line = end;
            }
            else
                return err_MALFORMED_DATA(error, channel->n+1);

            while (line[0] == ',' || g_ascii_isspace(line[0]))
                line++;
        }
    }
    g_free(q);

    if (!g_array_index(channels, SensolyticsChannel, 0).n) {
        err_NO_DATA(error);
        return FALSE;
    }

    return TRUE;
}

static void
map_to_grid(GArray *channels, const Dimensions *dimensions,
            SensolyticsGridMap *map)
{
    SensolyticsChannel *xrel, *yrel;
    gint k, ndata, col, row;
    gint *pixel_index;

    /* Skip X, Y and Z, each being two values. Can something useful be in Z? */
    ndata = map->ndata = g_array_index(channels, SensolyticsChannel, 0).n;

    /* XXX: Hardcoded X and Y channel numbers. */
    xrel = &g_array_index(channels, SensolyticsChannel, 1);
    if (!xrel->name || !gwy_strequal(xrel->name, "X rel."))
        g_warning("Column 2 is not \"X rel.\", mapping to pixels will be nonsense.");

    yrel = &g_array_index(channels, SensolyticsChannel, 3);
    if (!yrel->name || !gwy_strequal(yrel->name, "Y rel."))
        g_warning("Column 4 is not \"Y rel.\", mapping to pixels will be nonsense.");

    map->pixel_index = pixel_index = g_new(gint, ndata);
    map->mask = gwy_nield_new(dimensions->xres, dimensions->yres);
    gint *m = gwy_nield_get_data(map->mask);
    /* Figure out which data row corresponds to which pixel in the image.
     * NB: The increments and lengths can be negative. If the step is negative the relative coordinates seem to
     * indeed go into negative. */
    for (k = 0; k < ndata; k++) {
        col = GWY_ROUND(fabs(xrel->data[k]/dimensions->dx));
        col = CLAMP(col, 0, dimensions->xres-1);
        row = GWY_ROUND(fabs(yrel->data[k]/dimensions->dy));
        row = CLAMP(row, 0, dimensions->yres-1);
        pixel_index[k] = row*dimensions->xres + col;
        gwy_debug("index %d = (%d, %d) :: %g %g :: %g %g",
                  k, row, col, xrel->data[k], dimensions->dx, yrel->data[k], dimensions->dy);
        m[pixel_index[k]] += 1;
    }
}

static void
create_fields(GArray *channels, const Dimensions *dimensions, SensolyticsGridMap *map)
{
    SensolyticsChannel *channel;
    GwyField *field;
    gint k, ndata = map->ndata;
    guint i, ncols = channels->len;
    gint *pixel_index = map->pixel_index;

    gint n = dimensions->xres * dimensions->yres;
    gint *m = gwy_nield_get_data(map->mask);
    for (i = 5; i < ncols; i++) {
        channel = &g_array_index(channels, SensolyticsChannel, i);
        /* Skip string channels for now, we need to create Lawns from them later. */
        if (!channel->data)
            continue;

        field = gwy_field_new(dimensions->xres, dimensions->yres,
                              fabs(dimensions->xreal), fabs(dimensions->yreal), TRUE);
        /* The unit may be given with prefix, but we have already taken care of it. Do not multiply it by some power
         * of 10 again. */
        gwy_unit_set_from_string(gwy_field_get_unit_xy(field), dimensions->xyunit);
        gwy_unit_set_from_string(gwy_field_get_unit_z(field), channel->unit);
        gdouble *d = gwy_field_get_data(field);
        for (k = 0; k < ndata; k++)
            d[pixel_index[k]] += channel->data[k];
        for (k = 0; k < n; k++)
            d[k] = m[k] > 0 ? d[k]/m[k] : 0.0;

        if (gwy_field_rms(field) == 0.0) {
            g_object_unref(field);
            continue;
        }

        gwy_field_flip(field, TRUE, FALSE);
        channel->dfield = field;
    }
    if (gwy_nield_count(map->mask) != n) {
        gwy_nield_invert(map->mask);
        for (i = 5; i < ncols; i++) {
            channel = &g_array_index(channels, SensolyticsChannel, i);
            if (channel->dfield)
                gwy_field_laplace_solve(channel->dfield, map->mask, GWY_LAPLACE_MASKED, 0.5);
        }
        map->add_mask = TRUE;
    }
}

static gchar*
resolve_complex_data_datafile(gchar *datafilename, const gchar *dirname, const gchar *basename)
{
    gchar *filename, *s;

    /* References to some serialised .NET binary monstrosity representing user procedures. We know we cannot read
     * these. */
    if (g_str_has_suffix(datafilename, ".nox") || g_str_has_suffix(datafilename, ".NOX"))
        return NULL;

    g_strdelimit(datafilename, "\\", '/');
    if (datafilename[0] == '/')
        datafilename++;

    /* There are appaently different ways it can refer to the file. Sometimes without subdirectory and
     * extension, sometimes with. */
    filename = g_build_filename(dirname, datafilename, NULL);
    if (g_file_test(filename, G_FILE_TEST_IS_REGULAR))
        return filename;

    s = g_strconcat(filename, ".dat", NULL);
    g_free(filename);
    filename = s;
    if (g_file_test(filename, G_FILE_TEST_IS_REGULAR))
        return filename;

    g_free(filename);
    filename = g_build_filename(dirname, basename, datafilename, NULL);
    if (g_file_test(filename, G_FILE_TEST_IS_REGULAR))
        return filename;

    s = g_strconcat(filename, ".dat", NULL);
    g_free(filename);
    filename = s;
    if (g_file_test(filename, G_FILE_TEST_IS_REGULAR))
        return filename;

    gwy_debug("can NOT resolve %s", datafilename);
    g_free(filename);
    return NULL;
}

static void
create_lawns(GArray *channels, const Dimensions *dimensions, SensolyticsGridMap *map,
             const gchar *parent_filename)
{
    SensolyticsChannel *channel, *specchannel;
    GArray *specchannels, *curvedata;
    GwyLawn *lawn;
    gdouble *d;
    gchar *dirname, *basename, *filename, *buffer, *p, *s;
    GHashTable *spechash = NULL;
    gint *pixel_index = map->pixel_index;
    guint i, j, ncurves, ncols = channels->len;
    gint k;

    dirname = g_path_get_dirname(parent_filename);
    /* The basename without .dat suffix is the name of the subdirectory with curve data. */
    basename = g_path_get_basename(parent_filename);
    if (g_str_has_suffix(basename, ".dat") || g_str_has_suffix(basename, ".DAT"))
        basename[strlen(basename)-4] = '\0';

    specchannels = g_array_new(FALSE, TRUE, sizeof(SensolyticsChannel));
    curvedata = g_array_new(FALSE, FALSE, sizeof(gdouble));
    for (i = 5; i < ncols; i++) {
        channel = &g_array_index(channels, SensolyticsChannel, i);
        /* Skip channels with data; they have been handled already. */
        if (!channel->strings)
            continue;

        lawn = NULL;
        for (k = 0; k < channel->n; k++) {
            if (!(filename = resolve_complex_data_datafile(channel->strings[k], dirname, basename)))
                continue;

            gwy_debug("[%d] OK %s", k, filename);
            buffer = NULL;
            if (!g_file_get_contents(filename, &buffer, NULL, NULL)) {
                gwy_debug("file %s seemed OK but we cannot read it", filename);
                goto end;
            }

            p = buffer;
            if (!(spechash = read_header(&p, specchannels, NULL))
                || !specchannels->len
                || !read_data(specchannels, &p, NULL)
                || !check_coordinate_columns(specchannels, 1, NULL))
                goto end;

            ncurves = 0;
            g_array_set_size(curvedata, 0);
            for (j = 0; j < specchannels->len; j++) {
                specchannel = &g_array_index(specchannels, SensolyticsChannel, j);
                if ((d = specchannel->data)) {
                    g_array_append_vals(curvedata, d, specchannel->n);
                    ncurves++;
                }
            }
            if (!ncurves)
                goto end;

            if (!lawn) {
                /* We cannot represent metadata for each curve, but try to incorporate at least the first one into
                 * the lawn's metadata. */
                if (!channel->specmeta)
                    channel->specmeta = get_meta(spechash);
                lawn = gwy_lawn_new(dimensions->xres, dimensions->yres, dimensions->xreal, dimensions->yreal,
                                    ncurves, 0);
                gwy_unit_set_from_string(gwy_lawn_get_unit_xy(lawn), dimensions->xyunit);
                if ((s = g_hash_table_lookup(spechash, "Experiment type")))
                    gwy_assign_string(&channel->title, s);

                ncurves = 0;
                for (j = 0; j < specchannels->len; j++) {
                    specchannel = &g_array_index(specchannels, SensolyticsChannel, j);
                    if ((d = specchannel->data)) {
                        if (specchannel->name)
                            gwy_lawn_set_curve_label(lawn, ncurves, specchannel->name);
                        gwy_unit_set_from_string(gwy_lawn_get_unit_curve(lawn, ncurves), specchannel->unit);
                        ncurves++;
                    }
                }
            }
            else {
                if (ncurves != gwy_lawn_get_n_curves(lawn)) {
                    g_warning("Inconsistent content of curve data files.");
                    g_clear_object(&lawn);
                    /* Terminate the loop, but allow the end cleanup to run. */
                    k = channel->n;
                    goto end;
                }
            }

            gwy_lawn_set_curves(lawn,
                                pixel_index[k] % dimensions->xres, pixel_index[k]/dimensions->xres,
                                curvedata->len/ncurves, &g_array_index(curvedata, gdouble, 0), NULL);

end:
            g_free(filename);
            g_free(buffer);
            clear_channels(specchannels);
            if (spechash) {
                g_hash_table_unref(spechash);
                spechash = NULL;
            }
        }
        channel->lawn = lawn;
        lawn = NULL;
    }
    g_array_free(specchannels, TRUE);
    g_array_free(curvedata, TRUE);
    g_free(basename);
    g_free(dirname);
}

static GwyGraphModel*
create_graph_model(SensolyticsChannel *abscissa, SensolyticsChannel *ordinate)
{
    GwyGraphModel *gmodel = gwy_graph_model_new();
    GwyGraphCurveModel *cmodel;
    GwyUnit *xunit, *yunit;

    g_return_val_if_fail(abscissa->n == ordinate->n, gmodel);

    xunit = gwy_unit_new(abscissa->unit);
    yunit = gwy_unit_new(ordinate->unit);
    g_object_set(gmodel, "unit-x", xunit, "unit-y", yunit, NULL);
    g_object_unref(xunit);
    g_object_unref(yunit);

    if (abscissa->name)
        g_object_set(gmodel, "axis-label-bottom", abscissa->name, NULL);
    if (ordinate->name)
        g_object_set(gmodel, "axis-label-left", ordinate->name, "title", ordinate->name, NULL);

    cmodel = gwy_graph_curve_model_new();
    g_object_set(cmodel, "mode", GWY_GRAPH_CURVE_LINE, NULL);
    gwy_graph_curve_model_set_data(cmodel, abscissa->data, ordinate->data, abscissa->n);
    gwy_graph_curve_model_enforce_order(cmodel);
    gwy_graph_model_add_curve(gmodel, cmodel);
    g_object_unref(cmodel);


    return gmodel;
}

/* Some files are ISO-8859-1, some are UTF-8, … */
static void
add_meta(gpointer hkey, gpointer hvalue, gpointer user_data)
{
    GwyDict *meta = (GwyDict*)user_data;
    gchar *key = (gchar*)hkey, *value = (gchar*)hvalue;
    gchar *storage = NULL;

    if (!g_utf8_validate(key, -1, NULL)) {
        key = storage = gwy_convert_to_utf8(key, -1, "ISO-8859-1");
        if (!key)
            return;
    }

    if (g_utf8_validate(value, -1, NULL))
        value = g_strdup(value);
    else
        value = gwy_convert_to_utf8(value, -1, "ISO-8859-1");

    if (value)
        gwy_dict_pass_string_by_name(meta, key, value);
    g_free(storage);
}

static GwyDict*
get_meta(GHashTable *hash)
{
    GwyDict *meta = gwy_dict_new_in_construction();

    g_hash_table_foreach(hash, add_meta, meta);

    return meta;
}

static void
clone_field_meta(GwyFile *file, GwyDict *meta, gint id)
{
    if (gwy_dict_get_n_items(meta))
        gwy_file_pass_meta(file, GWY_FILE_IMAGE, id, gwy_dict_copy(meta));
}

static void
clone_lawn_meta(GwyFile *file, GwyDict *meta, GwyDict *specmeta, gint id)
{
    GwyDict *metacopy = gwy_dict_copy(meta);
    gwy_dict_transfer(specmeta, metacopy, NULL, NULL, FALSE, FALSE);
    if (gwy_dict_get_n_items(metacopy))
        gwy_file_pass_meta(file, GWY_FILE_CMAP, id, metacopy);
    else
        g_object_unref(metacopy);
}

static gboolean
check_coordinate_columns(GArray *channels, gint required_value_cols, GError **error)
{
    gint i;

    if ((gint)channels->len < required_value_cols) {
        g_set_error(error, GWY_MODULE_FILE_ERROR, GWY_MODULE_FILE_ERROR_DATA,
                    _("Wrong number of initial data columns."));
        return FALSE;
    }

    for (i = 0; i < MIN(required_value_cols, (gint)channels->len); i++) {
        SensolyticsChannel *channel = &g_array_index(channels, SensolyticsChannel, i);
        if (!channel->data) {
            g_set_error(error, GWY_MODULE_FILE_ERROR, GWY_MODULE_FILE_ERROR_DATA,
                        _("Wrong number of initial data columns."));
            return FALSE;
        }
    }
    return TRUE;
}

static void
clear_channels(GArray *channels)
{
    guint i;

    for (i = 0; i < channels->len; i++) {
        SensolyticsChannel *channel = &g_array_index(channels, SensolyticsChannel, i);
        g_clear_object(&channel->dfield);
        g_clear_object(&channel->lawn);
        g_clear_object(&channel->specmeta);
        GWY_FREE(channel->data);
        GWY_FREE(channel->strings);
        GWY_FREE(channel->title);
    }
    g_array_set_size(channels, 0);
}

/* vim: set cin columns=120 tw=118 et ts=4 sw=4 cino=>1s,e0,n0,f0,{0,}0,^0,\:1s,=0,g1s,h0,t0,+1s,c3,(0,u0 : */
