// This file is a part of Julia. License is MIT: https://julialang.org/license

/*
  saving and restoring system images
*/
#include <stdlib.h>
#include <string.h>
#include <stdio.h> // printf

#include "julia.h"
#include "julia_internal.h"
#include "builtin_proto.h"
#include "processor.h"

#ifndef _OS_WINDOWS_
#include <dlfcn.h>
#endif

#ifndef _COMPILER_MICROSOFT_
#include "valgrind.h"
#else
#define RUNNING_ON_VALGRIND 0
#endif
#include "julia_assert.h"

#ifdef __cplusplus
extern "C" {
#endif

// TODO: put WeakRefs on the weak_refs list during deserialization
// TODO: handle finalizers

// An array of references that need to be restored from the sysimg
// This is a manually constructed dual of the gvars array, which would be produced by codegen for Julia code, for C.
static void *const _tags[] = {
         // builtin types
         &jl_any_type, &jl_symbol_type, &jl_ssavalue_type, &jl_datatype_type, &jl_slotnumber_type,
         &jl_simplevector_type, &jl_array_type, &jl_typedslot_type,
         &jl_expr_type, &jl_globalref_type, &jl_string_type,
         &jl_module_type, &jl_tvar_type, &jl_method_instance_type, &jl_method_type, &jl_code_instance_type,
         &jl_linenumbernode_type, &jl_lineinfonode_type,
         &jl_gotonode_type, &jl_quotenode_type,
         &jl_pinode_type, &jl_phinode_type, &jl_phicnode_type, &jl_upsilonnode_type,
         &jl_type_type, &jl_bottom_type, &jl_ref_type, &jl_pointer_type, &jl_llvmpointer_type,
         &jl_vararg_type, &jl_abstractarray_type,
         &jl_densearray_type, &jl_nothing_type, &jl_function_type, &jl_typeofbottom_type,
         &jl_unionall_type, &jl_typename_type, &jl_builtin_type, &jl_code_info_type,
         &jl_task_type, &jl_uniontype_type, &jl_typetype_type, &jl_abstractstring_type,
         &jl_array_any_type, &jl_intrinsic_type, &jl_abstractslot_type,
         &jl_methtable_type, &jl_typemap_level_type, &jl_typemap_entry_type,
         &jl_voidpointer_type, &jl_newvarnode_type,
         &jl_anytuple_type_type, &jl_anytuple_type, &jl_namedtuple_type, &jl_emptytuple_type,
         &jl_array_symbol_type, &jl_array_uint8_type, &jl_array_int32_type,
         &jl_int32_type, &jl_int64_type, &jl_bool_type, &jl_uint8_type,
         &jl_uint32_type, &jl_uint64_type, &jl_char_type, &jl_weakref_type,
         &jl_int8_type, &jl_int16_type, &jl_uint16_type,
         &jl_float16_type, &jl_float32_type, &jl_float64_type, &jl_floatingpoint_type,
         &jl_number_type, &jl_signed_type,
         // special typenames
         &jl_tuple_typename, &jl_pointer_typename, &jl_llvmpointer_typename, &jl_array_typename, &jl_type_typename,
         &jl_vararg_typename, &jl_namedtuple_typename,
         &jl_vecelement_typename,
         // special exceptions
         &jl_errorexception_type, &jl_argumenterror_type, &jl_typeerror_type,
         &jl_methoderror_type, &jl_loaderror_type, &jl_initerror_type,
         &jl_undefvarerror_type, &jl_stackovf_exception, &jl_diverror_exception,
         &jl_interrupt_exception, &jl_boundserror_type, &jl_memory_exception,
         &jl_undefref_exception, &jl_readonlymemory_exception,
#ifdef SEGV_EXCEPTION
         &jl_segv_exception,
#endif
         // other special values
         &jl_emptysvec, &jl_emptytuple, &jl_false, &jl_true, &jl_nothing,
         &jl_an_empty_string, &jl_an_empty_vec_any,

         &jl_module_init_order, &jl_core_module, &jl_base_module, &jl_main_module, &jl_top_module,
         &jl_typeinf_func, &jl_type_type_mt, &jl_nonfunction_mt,
         // some Core.Builtin Functions that we want to be able to reference:
         &jl_builtin_throw, &jl_builtin_is, &jl_builtin_typeof, &jl_builtin_sizeof,
         &jl_builtin_issubtype, &jl_builtin_isa, &jl_builtin_typeassert, &jl_builtin__apply,
         &jl_builtin__apply_iterate,
         &jl_builtin_isdefined, &jl_builtin_nfields, &jl_builtin_tuple, &jl_builtin_svec,
         &jl_builtin_getfield, &jl_builtin_setfield, &jl_builtin_fieldtype, &jl_builtin_arrayref,
         &jl_builtin_const_arrayref, &jl_builtin_arrayset, &jl_builtin_arraysize,
         &jl_builtin_apply_type, &jl_builtin_applicable, &jl_builtin_invoke,
         &jl_builtin__expr, &jl_builtin_ifelse, &jl_builtin__typebody,
         NULL };
static jl_value_t **const*const tags = (jl_value_t**const*const)_tags;

// hash of definitions for predefined tagged object
static htable_t symbol_table;
static uintptr_t nsym_tag;
// array of definitions for the predefined tagged object types
// (reverse of symbol_table)
static arraylist_t deser_sym;

// table of all objects that are serialized
static htable_t backref_table;
static int backref_table_numel;
static arraylist_t layout_table;
static arraylist_t builtin_typenames;

// list of (size_t pos, (void *f)(jl_value_t*)) entries
// for the serializer to mark values in need of rework by function f
// during deserialization later
static arraylist_t reinit_list;

// @ccallable entry points to install
static arraylist_t ccallable_list;

// hash of definitions for predefined function pointers
static htable_t fptr_to_id;
void *native_functions;

// array of definitions for the predefined function pointers
// (reverse of fptr_to_id)
// This is a manually constructed dual of the fvars array, which would be produced by codegen for Julia code, for C.
static const jl_fptr_args_t id_to_fptrs[] = {
    &jl_f_throw, &jl_f_is, &jl_f_typeof, &jl_f_issubtype, &jl_f_isa,
    &jl_f_typeassert, &jl_f__apply, &jl_f__apply_iterate, &jl_f__apply_pure, &jl_f__apply_latest, &jl_f_isdefined,
    &jl_f_tuple, &jl_f_svec, &jl_f_intrinsic_call, &jl_f_invoke_kwsorter,
    &jl_f_getfield, &jl_f_setfield, &jl_f_fieldtype, &jl_f_nfields,
    &jl_f_arrayref, &jl_f_const_arrayref, &jl_f_arrayset, &jl_f_arraysize, &jl_f_apply_type,
    &jl_f_applicable, &jl_f_invoke, &jl_f_sizeof, &jl_f__expr, &jl_f__typevar,
    &jl_f_ifelse, &jl_f__structtype, &jl_f__abstracttype, &jl_f__primitivetype,
    &jl_f__typebody, &jl_f__setsuper, &jl_f__equiv_typedef,
    NULL };

typedef struct {
    ios_t *s;
    ios_t *const_data;
    ios_t *symbols;
    ios_t *relocs;
    ios_t *gvar_record;
    ios_t *fptr_record;
    arraylist_t relocs_list;
    arraylist_t gctags_list;
    jl_ptls_t ptls;
} jl_serializer_state;

static jl_value_t *jl_idtable_type = NULL;
static jl_typename_t *jl_idtable_typename = NULL;
static jl_value_t *jl_bigint_type = NULL;
static int gmp_limb_size = 0;

enum RefTags {
    DataRef,
    ConstDataRef,
    TagRef,
    SymbolRef,
    BindingRef,
    FunctionRef,
    BuiltinFunctionRef
};

// calling conventions for internal entry points.
// this is used to set the method-instance->invoke field
typedef enum {
    JL_API_NULL,
    JL_API_BOXED,
    JL_API_CONST,
    JL_API_WITH_PARAMETERS,
    JL_API_INTERPRETED,
    JL_API_BUILTIN,
    JL_API_MAX
} jl_callingconv_t;


// this supports up to 8 RefTags, 512MB of pointer data, and 4/2 (64/32-bit) GB of constant data.
// if a larger size is required, will need to add support for writing larger relocations in many cases below
#define RELOC_TAG_OFFSET 29


/* read and write in host byte order */

#define write_uint8(s, n) ios_putc((n), (s))
#define read_uint8(s) ((uint8_t)ios_getc((s)))

static void write_uint32(ios_t *s, uint32_t i) JL_NOTSAFEPOINT
{
    ios_write(s, (char*)&i, 4);
}

static uint32_t read_uint32(ios_t *s) JL_NOTSAFEPOINT
{
    uint32_t x = 0;
    ios_read(s, (char*)&x, 4);
    return x;
}


// --- Static Compile ---

static void *jl_sysimg_handle = NULL;
static uint64_t sysimage_base = 0;
static uintptr_t *sysimg_gvars_base = NULL;
static const int32_t *sysimg_gvars_offsets = NULL;
static jl_sysimg_fptrs_t sysimg_fptrs;

static inline uintptr_t *sysimg_gvars(uintptr_t *base, size_t idx)
{
    return base + sysimg_gvars_offsets[idx] / sizeof(base[0]);
}

JL_DLLEXPORT int jl_running_on_valgrind(void)
{
    return RUNNING_ON_VALGRIND;
}

static void jl_load_sysimg_so(void)
{
    int imaging_mode = jl_generating_output() && !jl_options.incremental;
    // in --build mode only use sysimg data, not precompiled native code
    if (!imaging_mode && jl_options.use_sysimage_native_code==JL_OPTIONS_USE_SYSIMAGE_NATIVE_CODE_YES) {
        jl_dlsym(jl_sysimg_handle, "jl_sysimg_gvars_base", (void **)&sysimg_gvars_base, 1);
        jl_dlsym(jl_sysimg_handle, "jl_sysimg_gvars_offsets", (void **)&sysimg_gvars_offsets, 1);
        sysimg_gvars_offsets += 1;
        assert(sysimg_fptrs.base);
        uintptr_t *tls_getter_slot;
        jl_dlsym(jl_sysimg_handle, "jl_get_ptls_states_slot", (void **)&tls_getter_slot, 1);
        *tls_getter_slot = (uintptr_t)jl_get_ptls_states_getter();
        size_t *tls_offset_idx;
        jl_dlsym(jl_sysimg_handle, "jl_tls_offset", (void **)&tls_offset_idx, 1);
        *tls_offset_idx = (uintptr_t)(jl_tls_offset == -1 ? 0 : jl_tls_offset);

#ifdef _OS_WINDOWS_
        sysimage_base = (intptr_t)jl_sysimg_handle;
#else
        Dl_info dlinfo;
        if (dladdr((void*)sysimg_gvars_base, &dlinfo) != 0) {
            sysimage_base = (intptr_t)dlinfo.dli_fbase;
        }
        else {
            sysimage_base = 0;
        }
#endif
    }
    else {
        memset(&sysimg_fptrs, 0, sizeof(sysimg_fptrs));
    }
    const char *sysimg_data;
    jl_dlsym(jl_sysimg_handle, "jl_system_image_data", (void **)&sysimg_data, 1);
    size_t *plen;
    jl_dlsym(jl_sysimg_handle, "jl_system_image_size", (void **)&plen, 1);
    jl_restore_system_image_data(sysimg_data, *plen);
}


// --- serializer ---

static uintptr_t jl_fptr_id(void *fptr)
{
    void **pbp = ptrhash_bp(&fptr_to_id, fptr);
    if (*pbp == HT_NOTFOUND || fptr == NULL)
        return 0;
    else
        return *(uintptr_t*)pbp;
}

#define jl_serialize_value(s, v) jl_serialize_value_(s,(jl_value_t*)(v))
static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v);


static void jl_serialize_module(jl_serializer_state *s, jl_module_t *m)
{
    jl_serialize_value(s, m->name);
    jl_serialize_value(s, m->parent);
    size_t i;
    void **table = m->bindings.table;
    for (i = 1; i < m->bindings.size; i += 2) {
        if (table[i] != HT_NOTFOUND) {
            jl_binding_t *b = (jl_binding_t*)table[i];
            jl_serialize_value(s, b->name);
            jl_serialize_value(s, b->value);
            jl_serialize_value(s, b->globalref);
            jl_serialize_value(s, b->owner);
        }
    }

    for (i = 0; i < m->usings.len; i++) {
        jl_serialize_value(s, (jl_value_t*)m->usings.items[i]);
    }
}


#define NBOX_C 1024

static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v)
{
    // ignore items that are given a special representation
    if (v == NULL || jl_is_symbol(v)) {
        return;
    }
    else if (jl_typeis(v, jl_task_type)) {
        if (v == (jl_value_t*)s->ptls->root_task) {
            jl_serialize_value(s, ((jl_task_t*)v)->tls);
            return;
        }
    }
    else if (jl_typeis(v, jl_int64_type)) {
        int64_t i64 = *(int64_t*)v + NBOX_C / 2;
        if ((uint64_t)i64 < NBOX_C)
            return;
    }
    else if (jl_typeis(v, jl_int32_type)) {
        int32_t i32 = *(int32_t*)v + NBOX_C / 2;
        if ((uint32_t)i32 < NBOX_C)
            return;
    }
    else if (jl_typeis(v, jl_uint8_type)) {
        return;
    }

    void **bp = ptrhash_bp(&backref_table, v);
    if (*bp != HT_NOTFOUND) {
        return;
    }

    size_t item = ++backref_table_numel;
    assert(item < ((uintptr_t)1 << RELOC_TAG_OFFSET) && "too many items to serialize");
    char *pos = (char*)HT_NOTFOUND + item;
    *bp = (void*)pos;

    // some values have special representations
    jl_datatype_t *t = (jl_datatype_t*)jl_typeof(v);
    jl_serialize_value(s, t);

    if (t->layout->npointers == 0) {
        // skip it
    }
    else if (jl_is_svec(v)) {
        size_t i, l = jl_svec_len(v);
        jl_value_t **data = jl_svec_data(v);
        for (i = 0; i < l; i++) {
            jl_serialize_value(s, data[i]);
        }
    }
    else if (jl_is_array(v)) {
        jl_array_t *ar = (jl_array_t*)v;
        jl_serialize_value(s, jl_typeof(ar));
        if (ar->flags.ptrarray) {
            size_t i, l = jl_array_len(ar);
            for (i = 0; i < l; i++) {
                jl_serialize_value(s, jl_array_ptr_ref(ar, i));
            }
        }
        else if (ar->flags.hasptr) {
            const char *data = (const char*)jl_array_data(ar);
            uint16_t elsz = ar->elsize;
            size_t i, l = jl_array_len(ar);
            jl_datatype_t *et = (jl_datatype_t*)jl_tparam0(jl_typeof(ar));
            size_t j, np = et->layout->npointers;
            for (i = 0; i < l; i++) {
                for (j = 0; j < np; j++) {
                    uint32_t ptr = jl_ptr_offset(et, j);
                    jl_value_t *fld = ((jl_value_t**)data)[ptr];
                    JL_GC_PROMISE_ROOTED(fld);
                    jl_serialize_value(s, fld);
                }
                data += elsz;
            }
        }
    }
    else if (jl_typeis(v, jl_module_type)) {
        jl_serialize_module(s, (jl_module_t*)v);
    }
    else if (t->layout->nfields > 0) {
        char *data = (char*)jl_data_ptr(v);
        size_t i, np = t->layout->npointers;
        for (i = 0; i < np; i++) {
            uint32_t ptr = jl_ptr_offset(t, i);
            jl_value_t *fld = ((jl_value_t* const*)data)[ptr];
            jl_serialize_value(s, fld);
        }
    }
}

static void ios_ensureroom(ios_t *s, size_t newsize) JL_NOTSAFEPOINT
{
    size_t prevsize = s->size;
    if (prevsize < newsize) {
        ios_trunc(s, newsize);
        assert(s->size == newsize);
        memset(&s->buf[prevsize], 0, newsize - prevsize);
    }
}

static void record_gvar(jl_serializer_state *s, int gid, uintptr_t reloc_id) JL_NOTSAFEPOINT
{
    if (gid == 0)
        return;
    ios_ensureroom(s->gvar_record, gid * sizeof(uint32_t));
    ios_seek(s->gvar_record, (gid - 1) * sizeof(uint32_t));
    assert(reloc_id < UINT32_MAX);
    write_uint32(s->gvar_record, reloc_id);
}


static void write_padding(ios_t *s, size_t nb) JL_NOTSAFEPOINT
{
    static const char zeros[16] = {0};
    while (nb > 16) {
        ios_write(s, zeros, 16);
        nb -= 16;
    }
    if (nb != 0)
        ios_write(s, zeros, nb);
}


static void write_pointer(ios_t *s) JL_NOTSAFEPOINT
{
    assert((ios_pos(s) & (sizeof(void*) - 1)) == 0 && "stream misaligned for writing a word-sized value");
    write_padding(s, sizeof(void*));
}


#define backref_id(s, v) _backref_id(s, (jl_value_t*)(v))
static uintptr_t _backref_id(jl_serializer_state *s, jl_value_t *v) JL_NOTSAFEPOINT
{
    assert(v != NULL && "cannot get backref to NULL object");
    void *idx = HT_NOTFOUND;
    if (jl_is_symbol(v)) {
        void **pidx = ptrhash_bp(&symbol_table, v);
        idx = *pidx;
        if (idx == HT_NOTFOUND) {
            size_t l = strlen(jl_symbol_name((jl_sym_t*)v));
            write_uint32(s->symbols, l);
            ios_write(s->symbols, jl_symbol_name((jl_sym_t*)v), l + 1);
            size_t offset = ++nsym_tag;
            assert(offset < ((uintptr_t)1 << RELOC_TAG_OFFSET) && "too many symbols");
            idx = (void*)((char*)HT_NOTFOUND + ((uintptr_t)SymbolRef << RELOC_TAG_OFFSET) + offset);
            *pidx = idx;
        }
    }
    else if (v == (jl_value_t*)s->ptls->root_task) {
        return (uintptr_t)TagRef << RELOC_TAG_OFFSET;
    }
    else if (jl_typeis(v, jl_int64_type)) {
        int64_t i64 = *(int64_t*)v + NBOX_C / 2;
        if ((uint64_t)i64 < NBOX_C)
            return ((uintptr_t)TagRef << RELOC_TAG_OFFSET) + i64 + 1;
    }
    else if (jl_typeis(v, jl_int32_type)) {
        int32_t i32 = *(int32_t*)v + NBOX_C / 2;
        if ((uint32_t)i32 < NBOX_C)
            return ((uintptr_t)TagRef << RELOC_TAG_OFFSET) + i32 + 1 + NBOX_C;
    }
    else if (jl_typeis(v, jl_uint8_type)) {
        uint8_t u8 = *(uint8_t*)v;
        return ((uintptr_t)TagRef << RELOC_TAG_OFFSET) + u8 + 1 + NBOX_C + NBOX_C;
    }
    if (idx == HT_NOTFOUND) {
        idx = ptrhash_get(&backref_table, v);
        assert(idx != HT_NOTFOUND && "object missed during jl_serialize_value pass");
    }
    return (char*)idx - 1 - (char*)HT_NOTFOUND;
}


static void write_pointerfield(jl_serializer_state *s, jl_value_t *fld) JL_NOTSAFEPOINT
{
    if (fld != NULL) {
        arraylist_push(&s->relocs_list, (void*)(uintptr_t)ios_pos(s->s));
        arraylist_push(&s->relocs_list, (void*)backref_id(s, fld));
    }
    write_pointer(s->s);
}

static void write_gctaggedfield(jl_serializer_state *s, uintptr_t ref) JL_NOTSAFEPOINT
{
    arraylist_push(&s->gctags_list, (void*)(uintptr_t)ios_pos(s->s));
    arraylist_push(&s->gctags_list, (void*)ref);
    write_pointer(s->s);
}


static void jl_write_module(jl_serializer_state *s, uintptr_t item, jl_module_t *m)
{
    size_t reloc_offset = ios_pos(s->s);
    size_t tot = sizeof(jl_module_t);
    ios_write(s->s, (char*)m, tot);

    jl_module_t *newm = (jl_module_t*)&s->s->buf[reloc_offset];
    newm->name = NULL;
    arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_module_t, name)));
    arraylist_push(&s->relocs_list, (void*)backref_id(s, m->name));
    newm->parent = NULL;
    arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_module_t, parent)));
    arraylist_push(&s->relocs_list, (void*)backref_id(s, m->parent));
    newm->primary_world = jl_world_counter;

    // write out the bindings table as a list
    // immediately after jl_module_t
    // (the ptrhash will need to be recreated on load)
    size_t count = 0;
    size_t i;
    void **table = m->bindings.table;
    for (i = 1; i < m->bindings.size; i += 2) {
        if (table[i] != HT_NOTFOUND) {
            jl_binding_t *b = (jl_binding_t*)table[i];
            write_gctaggedfield(s, (uintptr_t)BindingRef << RELOC_TAG_OFFSET);
            tot += sizeof(void*);
            size_t binding_reloc_offset = ios_pos(s->s);
            record_gvar(s, jl_get_llvm_gv(native_functions, (jl_value_t*)b),
                    ((uintptr_t)DataRef << RELOC_TAG_OFFSET) + binding_reloc_offset);
            write_pointerfield(s, (jl_value_t*)b->name);
            write_pointerfield(s, b->value);
            write_pointerfield(s, b->globalref);
            write_pointerfield(s, (jl_value_t*)b->owner);
            size_t flag_offset = offsetof(jl_binding_t, owner) + sizeof(b->owner);
            ios_write(s->s, (char*)b + flag_offset, sizeof(*b) - flag_offset);
            tot += sizeof(jl_binding_t);
            count += 1;
        }
    }
    assert(ios_pos(s->s) - reloc_offset == tot);
    newm = (jl_module_t*)&s->s->buf[reloc_offset]; // buf might have been reallocated
    newm->bindings.size = count; // stash the count in newm->size
    newm->bindings.table = NULL;
    memset(&newm->bindings._space, 0, sizeof(newm->bindings._space));

    // write out the usings list
    memset(&newm->usings._space, 0, sizeof(newm->usings._space));
    if (m->usings.items == &m->usings._space[0]) {
        newm->usings.items = (void**)offsetof(jl_module_t, usings._space);
        arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_module_t, usings.items)));
        arraylist_push(&s->relocs_list, (void*)(((uintptr_t)DataRef << RELOC_TAG_OFFSET) + item));
        size_t i;
        for (i = 0; i < m->usings.len; i++) {
            arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_module_t, usings._space[i])));
            arraylist_push(&s->relocs_list, (void*)backref_id(s, m->usings._space[i]));
        }
    }
    else {
        newm->usings.items = (void**)tot;
        arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_module_t, usings.items)));
        arraylist_push(&s->relocs_list, (void*)(((uintptr_t)DataRef << RELOC_TAG_OFFSET) + item));
        size_t i;
        for (i = 0; i < m->usings.len; i++) {
            write_pointerfield(s, (jl_value_t*)m->usings.items[i]);
            tot += sizeof(void*);
        }
        for (; i < m->usings.max; i++) {
            write_pointer(s->s);
            tot += sizeof(void*);
        }
    }
}

#if 0
static size_t jl_sort_size(jl_datatype_t *dt)
{
    if (dt == jl_simplevector_type)
        return SIZE_MAX - 5;
    if (dt == jl_string_type)
        return SIZE_MAX - 4;
    if (dt->name == jl_array_typename)
        return SIZE_MAX - 3;
    if (dt == jl_datatype_type)
        return SIZE_MAX - 2;
    if (dt == jl_module_type)
        return SIZE_MAX - 1;
    return jl_datatype_size(dt);
}
#endif

static int sysimg_sort_order(const void *pa, const void *pb)
{
    uintptr_t sa = ((uintptr_t*)pa)[1];
    uintptr_t sb = ((uintptr_t*)pb)[1];
    return (sa > sb ? 1 : (sa < sb ? -1 : 0));
#if 0
    jl_value_t *a = *(jl_value_t**)pa;
    jl_datatype_t *tya = (jl_datatype_t*)jl_typeof(a);
    size_t sa = jl_sort_size(tya);
    jl_value_t *b = *(jl_value_t**)pb;
    jl_datatype_t *tyb = (jl_datatype_t*)jl_typeof(b);
    size_t sb = jl_sort_size(tyb);
    if (sa == sb) {
        sa = tya->uid;
        sb = tyb->uid;
    }
    return (sa > sb ? 1 : (sa < sb ? -1 : 0));
#endif
}

jl_value_t *jl_find_ptr = NULL;
static void jl_write_values(jl_serializer_state *s)
{
    arraylist_t objects_list;
    arraylist_new(&objects_list, backref_table_numel * 2);

    arraylist_new(&layout_table, 0);
    arraylist_grow(&layout_table, backref_table_numel);
    memset(layout_table.items, 0, backref_table_numel * sizeof(void*));

    size_t i, len = backref_table.size;
    void **p = backref_table.table;
    for (i = 0; i < len; i += 2) {
        char *reloc_id = (char*)p[i + 1];
        if (reloc_id != HT_NOTFOUND) {
            jl_value_t *v = (jl_value_t*)p[i];
            uintptr_t item = reloc_id - 1 - (char*)HT_NOTFOUND;
            objects_list.items[objects_list.len++] = (void*)v;
            objects_list.items[objects_list.len++] = (void*)item;
        }
    }
    assert(backref_table_numel * 2 == objects_list.len);
    qsort(objects_list.items, backref_table_numel, sizeof(void*) * 2, sysimg_sort_order);

    for (i = 0, len = backref_table_numel * 2; i < len; i += 2) {
        jl_value_t *v = (jl_value_t*)objects_list.items[i];
        JL_GC_PROMISE_ROOTED(v);
        uintptr_t item = (uintptr_t)objects_list.items[i + 1];
        jl_datatype_t *t = (jl_datatype_t*)jl_typeof(v);
        assert((t->instance == NULL || t->instance == v) && "detected singleton construction corruption");
        // realign stream to expected gc alignment (16 bytes)
        uintptr_t skip_header_pos = ios_pos(s->s) + sizeof(jl_taggedvalue_t);
        write_padding(s->s, LLT_ALIGN(skip_header_pos, 16) - skip_header_pos);
        // write header
        write_gctaggedfield(s, backref_id(s, t));
        size_t reloc_offset = ios_pos(s->s);
        assert(item < layout_table.len && layout_table.items[item] == NULL);
        layout_table.items[item] = (void*)reloc_offset;
        record_gvar(s, jl_get_llvm_gv(native_functions, v), ((uintptr_t)DataRef << RELOC_TAG_OFFSET) + reloc_offset);

        // write data
        if (jl_is_cpointer(v)) {
            write_pointer(s->s);
        }
        else if (jl_is_array(v)) {
#define JL_ARRAY_ALIGN(jl_value, nbytes) LLT_ALIGN(jl_value, nbytes)
            jl_array_t *ar = (jl_array_t*)v;
            jl_value_t *et = jl_tparam0(jl_typeof(v));
            int ndimwords = jl_array_ndimwords(ar->flags.ndims);
            size_t tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords * sizeof(size_t), JL_CACHE_BYTE_ALIGNMENT);
            // copy header
            ios_write(s->s, (char*)v, tsz);
            // make some header modifications in-place
            jl_array_t *newa = (jl_array_t*)&s->s->buf[reloc_offset];
            size_t alen = jl_array_len(ar);
            size_t tot = alen * ar->elsize;
            if (newa->flags.ndims == 1)
                newa->maxsize = alen;
            newa->offset = 0;
            newa->flags.how = 0;
            newa->flags.pooled = 0;
            newa->flags.isshared = 0;

            // write data
            if (!ar->flags.ptrarray && !ar->flags.hasptr) {
                uintptr_t data = LLT_ALIGN(ios_pos(s->const_data), 16);
                // realign stream to max(data-align(array), sizeof(void*))
                write_padding(s->const_data, data - ios_pos(s->const_data));
                // write data and relocations
                newa->data = NULL; // relocation offset
                data /= sizeof(void*);
                assert(data < ((uintptr_t)1 << RELOC_TAG_OFFSET) && "offset to constant data too large");
                arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_array_t, data))); // relocation location
                arraylist_push(&s->relocs_list, (void*)(((uintptr_t)ConstDataRef << RELOC_TAG_OFFSET) + data)); // relocation target
                if (jl_is_cpointer_type(et)) {
                    // reset Ptr elements to C_NULL
                    size_t i;
                    for (i = 0; i < alen; i++)
                        write_pointer(s->const_data);
                }
                else {
                    int isbitsunion = jl_array_isbitsunion(ar);
                    if (ar->elsize == 1 && !isbitsunion)
                        tot += 1;
                    ios_write(s->const_data, (char*)jl_array_data(ar), tot);
                    if (isbitsunion)
                        ios_write(s->const_data, jl_array_typetagdata(ar), alen);
                }
            }
            else {
                newa->data = (void*)tsz; // relocation offset
                arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_array_t, data))); // relocation location
                arraylist_push(&s->relocs_list, (void*)(((uintptr_t)DataRef << RELOC_TAG_OFFSET) + item)); // relocation target
                if (ar->flags.hasptr) {
                    // copy all of the data first
                    const char *data = (const char*)jl_array_data(ar);
                    ios_write(s->s, data, tot);
                    // the rewrite all of the embedded pointers to null+relocation
                    uint16_t elsz = ar->elsize;
                    size_t j, np = ((jl_datatype_t*)et)->layout->npointers;
                    size_t i;
                    for (i = 0; i < alen; i++) {
                        for (j = 0; j < np; j++) {
                            size_t offset = i * elsz + jl_ptr_offset(((jl_datatype_t*)et), j) * sizeof(jl_value_t*);
                            jl_value_t *fld = *(jl_value_t**)&data[offset];
                            if (fld != NULL) {
                                arraylist_push(&s->relocs_list, (void*)(uintptr_t)(reloc_offset + tsz + offset)); // relocation location
                                arraylist_push(&s->relocs_list, (void*)backref_id(s, fld)); // relocation target
                                memset(&s->s->buf[reloc_offset + tsz + offset], 0, sizeof(fld)); // relocation offset (none)
                            }
                            else {
                                assert(*(jl_value_t**)&s->s->buf[reloc_offset + tsz + offset] == NULL);
                            }
                        }
                    }
                }
                else {
                    size_t i;
                    for (i = 0; i < alen; i++) {
                        jl_value_t *e = jl_array_ptr_ref(v, i);
                        write_pointerfield(s, e);
                    }
                }
            }
        }
        else if (jl_typeis(v, jl_module_type)) {
            jl_write_module(s, item, (jl_module_t*)v);
            // will need to recreate the binding table for this
            arraylist_push(&reinit_list, (void*)item);
            arraylist_push(&reinit_list, (void*)2);
        }
        else if (jl_typeis(v, jl_task_type)) {
            jl_error("Task cannot be serialized");
        }
        else if (jl_is_svec(v)) {
            ios_write(s->s, (char*)v, sizeof(void*));
            size_t i, l = jl_svec_len(v);
            assert(l > 0 || (jl_svec_t*)v == jl_emptysvec);
            for (i = 0; i < l; i++) {
                write_pointerfield(s, jl_svecref(v, i));
            }
        }
        else if (jl_is_string(v)) {
            ios_write(s->s, (char*)v, sizeof(void*) + jl_string_len(v));
            write_uint8(s->s, '\0'); // null-terminated strings for easier C-compatibility
        }
        else if (jl_datatype_nfields(t) == 0) {
            assert(t->layout->npointers == 0);
            if (t->size > 0)
                ios_write(s->s, (char*)v, t->size);
        }
        else if (jl_bigint_type && jl_typeis(v, jl_bigint_type)) {
            jl_value_t *sizefield = jl_get_nth_field(v, 1);
            int32_t sz = jl_unbox_int32(sizefield);
            int32_t nw = (sz == 0 ? 1 : (sz < 0 ? -sz : sz));
            size_t nb = nw * gmp_limb_size;
            ios_write(s->s, (char*)&nw, sizeof(int32_t));
            ios_write(s->s, (char*)&sz, sizeof(int32_t));
            uintptr_t data = LLT_ALIGN(ios_pos(s->const_data), 8);
            write_padding(s->const_data, data - ios_pos(s->const_data));
            data /= sizeof(void*);
            assert(data < ((uintptr_t)1 << RELOC_TAG_OFFSET) && "offset to constant data too large");
            arraylist_push(&s->relocs_list, (void*)(reloc_offset + 8)); // relocation location
            arraylist_push(&s->relocs_list, (void*)(((uintptr_t)ConstDataRef << RELOC_TAG_OFFSET) + data)); // relocation target
            void *pdata = jl_unbox_voidpointer(jl_get_nth_field(v, 2));
            ios_write(s->const_data, (char*)pdata, nb);
            write_pointer(s->s);
        }
        else {
            const char *data = (const char*)v;
            size_t i, nf = jl_datatype_nfields(t);
            size_t tot = 0;
            for (i = 0; i < nf; i++) {
                size_t offset = jl_field_offset(t, i);
                const char *slot = data + offset;
                write_padding(s->s, offset - tot);
                tot = offset;
                size_t fsz = jl_field_size(t, i);
                if (t->mutabl && jl_is_cpointer_type(jl_field_type(t, i))) {
                    // reset Ptr fields to C_NULL
                    assert(!jl_field_isptr(t, i));
                    write_pointer(s->s);
                }
                else if (fsz > 0) {
                    ios_write(s->s, slot, fsz);
                }
                tot += fsz;
            }

            size_t np = t->layout->npointers;
            for (i = 0; i < np; i++) {
                size_t offset = jl_ptr_offset(t, i) * sizeof(jl_value_t*);
                jl_value_t *fld = *(jl_value_t**)&data[offset];
                if (fld != NULL) {
                    arraylist_push(&s->relocs_list, (void*)(uintptr_t)(offset + reloc_offset)); // relocation location
                    arraylist_push(&s->relocs_list, (void*)backref_id(s, fld)); // relocation target
                    memset(&s->s->buf[offset + reloc_offset], 0, sizeof(fld)); // relocation offset (none)
                }
            }

            if (jl_is_method(v)) {
                write_padding(s->s, sizeof(jl_method_t) - tot);
                if (((jl_method_t*)v)->ccallable) {
                    arraylist_push(&ccallable_list, (void*)item);
                    arraylist_push(&ccallable_list, (void*)3);
                }
            }
            else if (jl_is_code_instance(v)) {
                jl_code_instance_t *m = (jl_code_instance_t*)v;
                jl_code_instance_t *newm = (jl_code_instance_t*)&s->s->buf[reloc_offset];

                newm->invoke = NULL;
                newm->isspecsig = 0;
                newm->specptr.fptr = NULL;
                int8_t fptr_id = JL_API_NULL;
                int8_t builtin_id = 0;
                if (m->invoke == jl_fptr_const_return) {
                    fptr_id = JL_API_CONST;
                }
                else {
                    if (jl_is_method(m->def->def.method)) {
                        builtin_id = jl_fptr_id(m->specptr.fptr);
                        if (builtin_id) { // found in the table of builtins
                            assert(builtin_id >= 2);
                            fptr_id = JL_API_BUILTIN;
                        }
                        else {
                            int32_t invokeptr_id = 0;
                            int32_t specfptr_id = 0;
                            jl_get_function_id(native_functions, m, &invokeptr_id, &specfptr_id); // see if we generated code for it
                            if (invokeptr_id) {
                                if (invokeptr_id == -1) {
                                    fptr_id = JL_API_BOXED;
                                }
                                else if (invokeptr_id == -2) {
                                    fptr_id = JL_API_WITH_PARAMETERS;
                                }
                                else {
                                    assert(invokeptr_id > 0);
                                    ios_ensureroom(s->fptr_record, invokeptr_id * sizeof(void*));
                                    ios_seek(s->fptr_record, (invokeptr_id - 1) * sizeof(void*));
                                    write_uint32(s->fptr_record, ~reloc_offset);
#ifdef _P64
                                    write_padding(s->fptr_record, 4);
#endif
                                }
                                if (specfptr_id) {
                                    assert(specfptr_id > invokeptr_id && specfptr_id > 0);
                                    ios_ensureroom(s->fptr_record, specfptr_id * sizeof(void*));
                                    ios_seek(s->fptr_record, (specfptr_id - 1) * sizeof(void*));
                                    write_uint32(s->fptr_record, reloc_offset);
#ifdef _P64
                                    write_padding(s->fptr_record, 4);
#endif
                                }
                            }
                        }
                    }
                }
                newm->invoke = NULL; // relocation offset
                if (fptr_id != JL_API_NULL) {
                    arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_code_instance_t, invoke))); // relocation location
                    arraylist_push(&s->relocs_list, (void*)(((uintptr_t)FunctionRef << RELOC_TAG_OFFSET) + fptr_id)); // relocation target
                }
                if (builtin_id >= 2) {
                    arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_code_instance_t, specptr.fptr))); // relocation location
                    arraylist_push(&s->relocs_list, (void*)(((uintptr_t)BuiltinFunctionRef << RELOC_TAG_OFFSET) + builtin_id - 2)); // relocation target
                }
            }
            else if (jl_is_datatype(v)) {
                jl_datatype_t *dt = (jl_datatype_t*)v;
                jl_datatype_t *newdt = (jl_datatype_t*)&s->s->buf[reloc_offset];
                if (dt->layout != NULL) {
                    size_t nf = dt->layout->nfields;
                    size_t np = dt->layout->npointers;
                    size_t fieldsize = jl_fielddesc_size(dt->layout->fielddesc_type);
                    char *flddesc = (char*)dt->layout;
                    size_t fldsize = sizeof(jl_datatype_layout_t) + nf * fieldsize;
                    if (dt->layout->first_ptr != -1)
                        fldsize += np << dt->layout->fielddesc_type;
                    uintptr_t layout = LLT_ALIGN(ios_pos(s->const_data), sizeof(void*));
                    write_padding(s->const_data, layout - ios_pos(s->const_data)); // realign stream
                    newdt->layout = NULL; // relocation offset
                    layout /= sizeof(void*);
                    arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_datatype_t, layout))); // relocation location
                    arraylist_push(&s->relocs_list, (void*)(((uintptr_t)ConstDataRef << RELOC_TAG_OFFSET) + layout)); // relocation target
                    ios_write(s->const_data, flddesc, fldsize);
                }
            }
            else if (((jl_datatype_t*)(jl_typeof(v)))->name == jl_idtable_typename) {
                // will need to rehash this, later (after types are fully constructed)
                arraylist_push(&reinit_list, (void*)item);
                arraylist_push(&reinit_list, (void*)1);
            }
            else {
                write_padding(s->s, t->size - tot);
            }
        }
    }
}


static void jl_write_gv_syms(jl_serializer_state *s, jl_sym_t *v)
{
    // since symbols are static, they might not have had a
    // reference anywhere in the code image other than here
    int32_t gv = jl_get_llvm_gv(native_functions, (jl_value_t*)v);
    if (gv != 0) {
        uintptr_t item = backref_id(s, v);
        assert(item >> RELOC_TAG_OFFSET == SymbolRef);
        record_gvar(s, gv, item);
    }
    if (v->left)
        jl_write_gv_syms(s, v->left);
    if (v->right)
        jl_write_gv_syms(s, v->right);
}

static void jl_write_gv_int(jl_serializer_state *s, jl_value_t *v)
{
    int32_t gv = jl_get_llvm_gv(native_functions, (jl_value_t*)v);
    if (gv != 0) {
        uintptr_t item = backref_id(s, v);
        assert(item >> RELOC_TAG_OFFSET == TagRef);
        record_gvar(s, gv, item);
    }
}
static void jl_write_gv_ints(jl_serializer_state *s)
{
    // this also ensures all objects referenced in the code have
    // references in the system image to their global variable
    // since codegen knows that some integer boxes are static,
    // they might not have had a reference anywhere in the code
    // image other than here
    size_t i;
    for (i = 0; i < NBOX_C; i++) {
        jl_write_gv_int(s, jl_box_int32((int32_t)i - NBOX_C / 2));
        jl_write_gv_int(s, jl_box_int64((int64_t)i - NBOX_C / 2));
    }
    for (i = 0; i < 256; i++) {
        jl_write_gv_int(s, jl_box_uint8(i));
    }
}

static inline uint32_t load_uint32(uintptr_t *base)
{
    uint32_t v = jl_load_unaligned_i32((void*)*base);
    *base += 4;
    return v;
}


static void jl_read_symbols(jl_serializer_state *s)
{
    assert(deser_sym.len == nsym_tag);
    uintptr_t base = (uintptr_t)&s->symbols->buf[0];
    uintptr_t end = base + s->symbols->size;
    while (base < end) {
        uint32_t len = load_uint32(&base);
        const char *str = (const char*)base;
        base += len + 1;
        //printf("symbol %3d: %s\n", len, str);
        jl_sym_t *sym = jl_symbol_n(str, len);
        arraylist_push(&deser_sym, (void*)sym);
    }
}


static uintptr_t get_reloc_for_item(uintptr_t reloc_item, size_t reloc_offset)
{
    enum RefTags tag = (enum RefTags)(reloc_item >> RELOC_TAG_OFFSET);
    if (tag == DataRef) {
        // need to compute the final relocation offset via the layout table
        assert(reloc_item < layout_table.len);
        uintptr_t reloc_base = (uintptr_t)layout_table.items[reloc_item];
        assert(reloc_base != 0 && "layout offset missing for relocation item");
        // write reloc_offset into s->s at pos
        return reloc_base + reloc_offset;
    }
    else {
        // just write the item reloc_id directly
#ifndef JL_NDEBUG
        assert(reloc_offset == 0 && "offsets for relocations to builtin objects should be precomposed in the reloc_item");
        size_t offset = (reloc_item & (((uintptr_t)1 << RELOC_TAG_OFFSET) - 1));
        switch (tag) {
        case ConstDataRef:
            break;
        case SymbolRef:
            assert(offset < nsym_tag && "corrupt relocation item id");
            break;
        case TagRef:
            assert(offset < 2 * NBOX_C + 257 && "corrupt relocation item id");
            break;
        case BindingRef:
            assert(offset == 0 && "corrupt relocation offset");
            break;
        case BuiltinFunctionRef:
            assert(offset < sizeof(id_to_fptrs) / sizeof(*id_to_fptrs) && "unknown function pointer id");
            break;
        case FunctionRef:
            assert(offset < JL_API_MAX && "unknown function pointer id");
            break;
        case DataRef:
        default:
            assert(0 && "corrupt relocation item id");
            abort();
        }
#endif
        return reloc_item; // pre-composed relocation + offset
    }
}


static inline uintptr_t get_item_for_reloc(jl_serializer_state *s, uintptr_t base, size_t size, uint32_t reloc_id)
{
    enum RefTags tag = (enum RefTags)(reloc_id >> RELOC_TAG_OFFSET);
    size_t offset = (reloc_id & (((uintptr_t)1 << RELOC_TAG_OFFSET) - 1));
    switch (tag) {
    case DataRef:
        assert(offset <= size);
        return base + offset;
    case ConstDataRef:
        return (uintptr_t)s->const_data->buf + (offset * sizeof(void*));
    case SymbolRef:
        assert(offset < deser_sym.len && deser_sym.items[offset] && "corrupt relocation item id");
        return (uintptr_t)deser_sym.items[offset];
    case BindingRef:
        return jl_buff_tag | GC_OLD_MARKED;
    case TagRef:
        if (offset == 0)
            return (uintptr_t)s->ptls->root_task;
        offset -= 1;
        if (offset < NBOX_C)
            return (uintptr_t)jl_box_int64((int64_t)offset - NBOX_C / 2);
        offset -= NBOX_C;
        if (offset < NBOX_C)
            return (uintptr_t)jl_box_int32((int32_t)offset - NBOX_C / 2);
        offset -= NBOX_C;
        if (offset < 256)
            return (uintptr_t)jl_box_uint8(offset);
        // offset -= 256;
        assert(0 && "corrupt relocation item id");
    case BuiltinFunctionRef:
        assert(offset < sizeof(id_to_fptrs) / sizeof(*id_to_fptrs) && "unknown function pointer ID");
        return (uintptr_t)id_to_fptrs[offset];
    case FunctionRef:
        switch ((jl_callingconv_t)offset) {
        case JL_API_BOXED:
            if (sysimg_fptrs.base)
                return (uintptr_t)jl_fptr_args;
            JL_FALLTHROUGH;
        case JL_API_WITH_PARAMETERS:
            if (sysimg_fptrs.base)
                return (uintptr_t)jl_fptr_sparam;
            return (uintptr_t)NULL;
        case JL_API_CONST:
            return (uintptr_t)jl_fptr_const_return;
        case JL_API_INTERPRETED:
            return (uintptr_t)jl_fptr_interpret_call;
        case JL_API_BUILTIN:
            return (uintptr_t)jl_fptr_args;
        case JL_API_NULL:
        case JL_API_MAX:
        //default:
            assert("corrupt relocation item id");
        }
    }
    abort();
}


static void jl_write_skiplist(ios_t *s, char *base, size_t size, arraylist_t *list)
{
    size_t i;
    for (i = 0; i < list->len; i += 2) {
        size_t pos = (size_t)list->items[i];
        size_t item = (size_t)list->items[i + 1];
        uintptr_t *pv = (uintptr_t*)(base + pos);
        assert(pos < size && pos != 0);
        *pv = get_reloc_for_item(item, *pv);
        // record pos in relocations list
        // TODO: save space by using delta-compression
        assert(pos < UINT32_MAX);
        write_uint32(s, pos);
    }
    write_uint32(s, 0);
}


static void jl_write_relocations(jl_serializer_state *s)
{
    char *base = &s->s->buf[0];
    jl_write_skiplist(s->relocs, base, s->s->size, &s->gctags_list);
    jl_write_skiplist(s->relocs, base, s->s->size, &s->relocs_list);
}


static void jl_read_relocations(jl_serializer_state *s, uint8_t bits)
{
    uintptr_t base = (uintptr_t)&s->s->buf[0];
    size_t size = s->s->size;
    while (1) {
        uintptr_t val = (uintptr_t)&s->relocs->buf[s->relocs->bpos];
        uint32_t offset = load_uint32(&val);
        s->relocs->bpos += sizeof(uint32_t);
        if (offset == 0)
            break;
        uintptr_t *pv = (uintptr_t*)(base + offset);
        uintptr_t v = *pv;
        v = get_item_for_reloc(s, base, size, v);
        *pv = v | bits;
    }
}

static char* sysimg_base;
static char* sysimg_relocs;
void gc_sweep_sysimg(void)
{
    uintptr_t base = (uintptr_t)sysimg_base;
    uintptr_t relocs = (uintptr_t)sysimg_relocs;
    if (relocs == 0)
        return;
    while (1) {
        uint32_t offset = load_uint32(&relocs);
        if (offset == 0)
            break;
        jl_taggedvalue_t *o = (jl_taggedvalue_t*)(base + offset);
        o->bits.gc = GC_OLD;
    }
}

#define jl_write_value(s, v) _jl_write_value((s), (jl_value_t*)(v))
static void _jl_write_value(jl_serializer_state *s, jl_value_t *v)
{
    if (v == NULL) {
        write_uint32(s->s, 0);
        return;
    }
    uintptr_t item = backref_id(s, v);
    uintptr_t reloc = get_reloc_for_item(item, 0);
    assert(reloc < UINT32_MAX);
    write_uint32(s->s, reloc);
}


static jl_value_t *jl_read_value(jl_serializer_state *s)
{
    uintptr_t base = (uintptr_t)&s->s->buf[0];
    size_t size = s->s->size;
    uintptr_t val = base + s->s->bpos;
    uint32_t offset = load_uint32(&val);
    s->s->bpos += sizeof(uint32_t);
    if (offset == 0)
        return NULL;
    return (jl_value_t*)get_item_for_reloc(s, base, size, offset);
}


static void jl_update_all_fptrs(jl_serializer_state *s)
{
    jl_sysimg_fptrs_t fvars = sysimg_fptrs;
    // make these NULL now so we skip trying to restore GlobalVariable pointers later
    sysimg_gvars_base = NULL;
    sysimg_fptrs.base = NULL;
    if (fvars.base == NULL)
        return;
    int sysimg_fvars_max = s->fptr_record->size / sizeof(void*);
    size_t i;
    uintptr_t base = (uintptr_t)&s->s->buf[0];
    jl_method_instance_t **linfos = (jl_method_instance_t**)&s->fptr_record->buf[0];
    uint32_t clone_idx = 0;
    for (i = 0; i < sysimg_fvars_max; i++) {
        uintptr_t val = (uintptr_t)&linfos[i];
        uint32_t offset = load_uint32(&val);
        linfos[i] = NULL;
        if (offset != 0) {
            int specfunc = 1;
            if (offset & ((uintptr_t)1 << (8 * sizeof(uint32_t) - 1))) {
                // if high bit is set, this is the func wrapper, not the specfunc
                specfunc = 0;
                offset = ~offset;
            }
            jl_code_instance_t *codeinst = (jl_code_instance_t*)(base + offset);
            uintptr_t base = (uintptr_t)fvars.base;
            assert(jl_is_method(codeinst->def->def.method) && codeinst->invoke != jl_fptr_const_return);
            assert(specfunc ? codeinst->invoke != NULL : codeinst->invoke == NULL);
            linfos[i] = codeinst->def;
            int32_t offset = fvars.offsets[i];
            for (; clone_idx < fvars.nclones; clone_idx++) {
                uint32_t idx = fvars.clone_idxs[clone_idx] & jl_sysimg_val_mask;
                if (idx < i)
                    continue;
                if (idx == i)
                    offset = fvars.clone_offsets[clone_idx];
                break;
            }
            void *fptr = (void*)(base + offset);
            if (specfunc) {
                codeinst->specptr.fptr = fptr;
                codeinst->isspecsig = 1; // TODO: set only if confirmed to be true
            }
            else {
                codeinst->invoke = (jl_callptr_t)fptr;
            }
            jl_fptr_to_llvm(fptr, codeinst, specfunc);
        }
    }
    jl_register_fptrs(sysimage_base, &fvars, linfos, sysimg_fvars_max);
}


static void jl_update_all_gvars(jl_serializer_state *s)
{
    if (sysimg_gvars_base == NULL)
        return;
    size_t gvname_index = 0;
    uintptr_t base = (uintptr_t)&s->s->buf[0];
    size_t size = s->s->size;
    uintptr_t gvars = (uintptr_t)&s->gvar_record->buf[0];
    uintptr_t end = gvars + s->gvar_record->size;
    while (gvars < end) {
        uint32_t offset = load_uint32(&gvars);
        if (offset) {
            uintptr_t v = get_item_for_reloc(s, base, size, offset);
            *sysimg_gvars(sysimg_gvars_base, gvname_index) = v;
        }
        gvname_index += 1;
    }
}


static void jl_finalize_serializer(jl_serializer_state *s, arraylist_t *list)
{
    size_t i, l;

    // record list of reinitialization functions
    l = list->len;
    for (i = 0; i < l; i += 2) {
        size_t item = (size_t)list->items[i];
        size_t reloc_offset = (size_t)layout_table.items[item];
        assert(reloc_offset != 0);
        write_uint32(s->s, (uint32_t)reloc_offset);
        write_uint32(s->s, (uint32_t)((uintptr_t)list->items[i + 1]));
    }
    write_uint32(s->s, 0);
}


static void jl_reinit_item(jl_value_t *v, int how)
{
    switch (how) {
        case 1: { // rehash IdDict
            jl_array_t **a = (jl_array_t**)v;
            assert(jl_is_array(*a));
            // Assume *a don't need a write barrier
            *a = jl_idtable_rehash(*a, jl_array_len(*a));
            jl_gc_wb(v, *a);
            break;
        }
        case 2: { // rebuild the binding table for module v
            jl_module_t *mod = (jl_module_t*)v;
            assert(jl_is_module(mod));
            size_t nbindings = mod->bindings.size;
            htable_new(&mod->bindings, nbindings);
            struct binding {
                uintptr_t tag;
                jl_binding_t b;
            } *b;
            b = (struct binding*)&mod[1];
            while (nbindings > 0) {
                ptrhash_put(&mod->bindings, (char*)b->b.name, &b->b);
                b += 1;
                nbindings -= 1;
            }
            if (mod->usings.items != &mod->usings._space[0]) {
                void **newitems = (void**)malloc_s(mod->usings.max * sizeof(void*));
                memcpy(newitems, mod->usings.items, mod->usings.len * sizeof(void*));
                mod->usings.items = newitems;
            }
            break;
        }
        case 3: { // install ccallable entry point in JIT
            jl_svec_t *sv = ((jl_method_t*)v)->ccallable;
            jl_compile_extern_c(NULL, NULL, jl_sysimg_handle, jl_svecref(sv, 0), jl_svecref(sv, 1));
            break;
        }
        default:
            assert(0 && "corrupt deserialization state");
            abort();
    }
}


static void jl_finalize_deserializer(jl_serializer_state *s)
{
    // run reinitialization functions
    uintptr_t base = (uintptr_t)&s->s->buf[0];
    while (1) {
        size_t offset = read_uint32(s->s);
        if (offset == 0)
            break;
        jl_value_t *v = (jl_value_t*)(base + offset);
        jl_reinit_item(v, read_uint32(s->s));
    }
}



// --- helper functions ---

// remove cached types not referenced in the stream
static void jl_prune_type_cache(jl_svec_t *cache)
{
    size_t l = jl_svec_len(cache), ins = 0, i;
    for (i = 0; i < l; i++) {
        jl_value_t *ti = jl_svecref(cache, i);
        if (ti == NULL)
            break;
        if (ptrhash_get(&backref_table, ti) != HT_NOTFOUND || jl_get_llvm_gv(native_functions, ti) != 0)
            jl_svecset(cache, ins++, ti);
        else if (jl_is_datatype(ti)) {
            jl_value_t *singleton = ((jl_datatype_t*)ti)->instance;
            if (singleton && (ptrhash_get(&backref_table, singleton) != HT_NOTFOUND ||
                        jl_get_llvm_gv(native_functions, singleton) != 0))
                jl_svecset(cache, ins++, ti);
        }
    }
    if (i > ins) {
        memset(&jl_svec_data(cache)[ins], 0, (i - ins) * sizeof(jl_value_t*));
    }
}


// --- entry points ---

static void jl_init_serializer2(int);
static void jl_cleanup_serializer2(void);

static void jl_save_system_image_to_stream(ios_t *f)
{
    jl_gc_collect(JL_GC_FULL);
    jl_gc_collect(JL_GC_INCREMENTAL);   // sweep finalizers
    JL_TIMING(SYSIMG_DUMP);
    int en = jl_gc_enable(0);
    jl_init_serializer2(1);
    htable_reset(&backref_table, 250000);
    arraylist_new(&reinit_list, 0);
    arraylist_new(&ccallable_list, 0);
    backref_table_numel = 0;
    ios_t sysimg, const_data, symbols, relocs, gvar_record, fptr_record;
    ios_mem(&sysimg,     1000000);
    ios_mem(&const_data,  100000);
    ios_mem(&symbols,     100000);
    ios_mem(&relocs,      100000);
    ios_mem(&gvar_record, 100000);
    ios_mem(&fptr_record, 100000);
    jl_serializer_state s;
    s.s = &sysimg;
    s.const_data = &const_data;
    s.symbols = &symbols;
    s.relocs = &relocs;
    s.gvar_record = &gvar_record;
    s.fptr_record = &fptr_record;
    s.ptls = jl_get_ptls_states();
    arraylist_new(&s.relocs_list, 0);
    arraylist_new(&s.gctags_list, 0);

    // empty!(Core.ARGS)
    if (jl_core_module != NULL) {
        jl_array_t *args = (jl_array_t*)jl_get_global(jl_core_module, jl_symbol("ARGS"));
        if (args != NULL) {
            jl_array_del_end(args, jl_array_len(args));
        }
    }

    jl_idtable_type = jl_base_module ? jl_get_global(jl_base_module, jl_symbol("IdDict")) : NULL;
    jl_idtable_typename = jl_base_module ? ((jl_datatype_t*)jl_unwrap_unionall((jl_value_t*)jl_idtable_type))->name : NULL;
    jl_bigint_type = jl_base_module ? jl_get_global(jl_base_module, jl_symbol("BigInt")) : NULL;
    if (jl_bigint_type) {
        gmp_limb_size = jl_unbox_long(jl_get_global((jl_module_t*)jl_get_global(jl_base_module, jl_symbol("GMP")),
                                                    jl_symbol("BITS_PER_LIMB"))) / 8;
    }

    { // step 1: record values (recursively) that need to go in the image
        size_t i;
        for (i = 0; tags[i] != NULL; i++) {
            jl_value_t *tag = *tags[i];
            jl_serialize_value(&s, tag);
        }
        for (i = 0; i < builtin_typenames.len; i++) {
            jl_typename_t *tn = (jl_typename_t*)builtin_typenames.items[i];
            jl_prune_type_cache(tn->cache);
            jl_prune_type_cache(tn->linearcache);
        }
        for (i = 0; i < builtin_typenames.len; i++) {
            jl_typename_t *tn = (jl_typename_t*)builtin_typenames.items[i];
            jl_serialize_value(&s, tn->cache);
            jl_serialize_value(&s, tn->linearcache);
        }
    }

    { // step 2: build all the sysimg sections
        write_padding(&sysimg, sizeof(uint32_t));
        jl_write_values(&s);
        jl_write_relocations(&s);
        jl_write_gv_syms(&s, jl_get_root_symbol());
        jl_write_gv_ints(&s);
    }

    if (sysimg.size > ((uintptr_t)1 << RELOC_TAG_OFFSET) ||
        const_data.size > ((uintptr_t)1 << RELOC_TAG_OFFSET)*sizeof(void*)) {
        jl_printf(JL_STDERR, "ERROR: system image too large\n");
        jl_exit(1);
    }

    // step 3: combine all of the sections into one file
    write_uint32(f, sysimg.size - sizeof(uint32_t));
    ios_seek(&sysimg, sizeof(uint32_t));
    ios_copyall(f, &sysimg);
    ios_close(&sysimg);

    write_uint32(f, const_data.size);
    // realign stream to max-alignment for data
    write_padding(f, LLT_ALIGN(ios_pos(f), 16) - ios_pos(f));
    ios_seek(&const_data, 0);
    ios_copyall(f, &const_data);
    ios_close(&const_data);

    write_uint32(f, symbols.size);
    ios_seek(&symbols, 0);
    ios_copyall(f, &symbols);
    ios_close(&symbols);

    write_uint32(f, relocs.size);
    ios_seek(&relocs, 0);
    ios_copyall(f, &relocs);
    ios_close(&relocs);

    write_uint32(f, gvar_record.size);
    ios_seek(&gvar_record, 0);
    ios_copyall(f, &gvar_record);
    ios_close(&gvar_record);

    write_uint32(f, fptr_record.size);
    ios_seek(&fptr_record, 0);
    ios_copyall(f, &fptr_record);
    ios_close(&fptr_record);

    { // step 4: record locations of special roots
        s.s = f;
        size_t i;
        for (i = 0; tags[i] != NULL; i++) {
            jl_value_t *tag = *tags[i];
            jl_write_value(&s, tag);
        }
        jl_write_value(&s, s.ptls->root_task->tls);
        write_uint32(f, jl_get_gs_ctr());
        write_uint32(f, jl_world_counter);
        write_uint32(f, jl_typeinf_world);
        jl_finalize_serializer(&s, &reinit_list);
        jl_finalize_serializer(&s, &ccallable_list);
    }

    arraylist_free(&layout_table);
    arraylist_free(&reinit_list);
    arraylist_free(&ccallable_list);
    arraylist_free(&s.relocs_list);
    arraylist_free(&s.gctags_list);
    jl_cleanup_serializer2();

    jl_gc_enable(en);
}

JL_DLLEXPORT ios_t *jl_create_system_image(void *_native_data)
{
    ios_t *f = (ios_t*)malloc_s(sizeof(ios_t));
    ios_mem(f, 0);
    native_functions = _native_data;
    jl_save_system_image_to_stream(f);
    return f;
}

JL_DLLEXPORT size_t ios_write_direct(ios_t *dest, ios_t *src);
JL_DLLEXPORT void jl_save_system_image(const char *fname)
{
    ios_t f;
    if (ios_file(&f, fname, 1, 1, 1, 1) == NULL) {
        jl_errorf("cannot open system image file \"%s\" for writing", fname);
    }
    JL_SIGATOMIC_BEGIN();
    jl_save_system_image_to_stream(&f);
    ios_close(&f);
    JL_SIGATOMIC_END();
}

extern void jl_init_int32_int64_cache(void);
extern void jl_gc_set_permalloc_region(void *start, void *end);

// Takes in a path of the form "usr/lib/julia/sys.so" (jl_restore_system_image should be passed the same string)
JL_DLLEXPORT void jl_preload_sysimg_so(const char *fname)
{
    if (jl_sysimg_handle)
        return; // embedded target already called jl_set_sysimg_so

    char *dot = (char*) strrchr(fname, '.');
    int is_ji = (dot && !strcmp(dot, ".ji"));

    // Get handle to sys.so
    if (!is_ji) // .ji extension => load .ji file only
        jl_set_sysimg_so(jl_load_dynamic_library(fname, JL_RTLD_LOCAL | JL_RTLD_NOW, 1));
}

// Allow passing in a module handle directly, rather than a path
JL_DLLEXPORT void jl_set_sysimg_so(void *handle)
{
    void* *jl_RTLD_DEFAULT_handle_pointer;
    int symbol_found = jl_dlsym(handle, "jl_RTLD_DEFAULT_handle_pointer", (void **)&jl_RTLD_DEFAULT_handle_pointer, 0);
    if (!symbol_found || (void*)&jl_RTLD_DEFAULT_handle != *jl_RTLD_DEFAULT_handle_pointer)
        jl_error("System image file failed consistency check: maybe opened the wrong version?");
    if (jl_options.cpu_target == NULL)
        jl_options.cpu_target = "native";
    jl_sysimg_handle = handle;
    sysimg_fptrs = jl_init_processor_sysimg(handle);
}

static void jl_restore_system_image_from_stream(ios_t *f)
{
    JL_TIMING(SYSIMG_LOAD);
    int en = jl_gc_enable(0);
    jl_init_serializer2(0);
    ios_t sysimg, const_data, symbols, relocs, gvar_record, fptr_record;
    jl_serializer_state s;
    s.s = NULL;
    s.const_data = &const_data;
    s.symbols = &symbols;
    s.relocs = &relocs;
    s.gvar_record = &gvar_record;
    s.fptr_record = &fptr_record;
    s.ptls = jl_get_ptls_states();
    arraylist_new(&s.relocs_list, 0);
    arraylist_new(&s.gctags_list, 0);

    // step 1: read section map
    assert(ios_pos(f) == 0 && f->bm == bm_mem);
    size_t sizeof_sysimg = read_uint32(f);
    ios_static_buffer(&sysimg, f->buf, sizeof_sysimg + sizeof(uint32_t));
    ios_skip(f, sizeof_sysimg);

    size_t sizeof_constdata = read_uint32(f);
    // realign stream to max-alignment for data
    ios_seek(f, LLT_ALIGN(ios_pos(f), 16));
    ios_static_buffer(&const_data, f->buf + f->bpos, sizeof_constdata);
    ios_skip(f, sizeof_constdata);

    size_t sizeof_symbols = read_uint32(f);
    ios_static_buffer(&symbols, f->buf + f->bpos, sizeof_symbols);
    ios_skip(f, sizeof_symbols);

    size_t sizeof_relocations = read_uint32(f);
    assert(!ios_eof(f));
    ios_static_buffer(&relocs, f->buf + f->bpos, sizeof_relocations);
    ios_skip(f, sizeof_relocations);

    size_t sizeof_gvar_record = read_uint32(f);
    assert(!ios_eof(f));
    ios_static_buffer(&gvar_record, f->buf + f->bpos, sizeof_gvar_record);
    ios_skip(f, sizeof_gvar_record);

    size_t sizeof_fptr_record = read_uint32(f);
    assert(!ios_eof(f));
    ios_static_buffer(&fptr_record, f->buf + f->bpos, sizeof_fptr_record);
    ios_skip(f, sizeof_fptr_record);

    // step 2: get references to special values
    s.s = f;
    size_t i;
    for (i = 0; tags[i] != NULL; i++) {
        jl_value_t **tag = tags[i];
        *tag = jl_read_value(&s);
    }
    s.ptls->root_task = (jl_task_t*)jl_gc_alloc(s.ptls, sizeof(jl_task_t), jl_task_type);
    memset(s.ptls->root_task, 0, sizeof(jl_task_t));
    s.ptls->root_task->tls = jl_read_value(&s);
    jl_init_int32_int64_cache();
    jl_init_box_caches();

    uint32_t gs_ctr = read_uint32(f);
    jl_world_counter = read_uint32(f);
    jl_typeinf_world = read_uint32(f);
    jl_set_gs_ctr(gs_ctr);
    s.s = NULL;

    // step 3: apply relocations
    assert(!ios_eof(f));
    jl_read_symbols(&s);
    ios_close(&symbols);

    sysimg_base = &sysimg.buf[0];
    sysimg_relocs = &relocs.buf[0];
    jl_gc_set_permalloc_region((void*)sysimg_base, (void*)(sysimg_base + sysimg.size));

    s.s = &sysimg;
    jl_read_relocations(&s, GC_OLD_MARKED); // gctags
    size_t sizeof_tags = ios_pos(&relocs);
    (void)sizeof_tags;
    jl_read_relocations(&s, 0); // general relocs
    ios_close(&relocs);
    ios_close(&const_data);
    jl_update_all_gvars(&s); // gvars relocs
    ios_close(&gvar_record);
    s.s = NULL;

    s.s = f;
    // reinit items except ccallables
    jl_finalize_deserializer(&s);
    s.s = NULL;

    if (0) {
        printf("sysimg size breakdown:\n"
               "     sys data: %8u\n"
               "  isbits data: %8u\n"
               "      symbols: %8u\n"
               "    tags list: %8u\n"
               "   reloc list: %8u\n"
               "    gvar list: %8u\n"
               "    fptr list: %8u\n",
            (unsigned)sizeof_sysimg,
            (unsigned)sizeof_constdata,
            (unsigned)sizeof_symbols,
            (unsigned)sizeof_tags,
            (unsigned)(sizeof_relocations - sizeof_tags),
            (unsigned)sizeof_gvar_record,
            (unsigned)sizeof_fptr_record);
    }

    s.s = &sysimg;
    jl_init_codegen();
    jl_update_all_fptrs(&s); // fptr relocs and registration
    // reinit ccallables, which require codegen to be initialized
    s.s = f;
    jl_finalize_deserializer(&s);

    ios_close(&fptr_record);
    ios_close(&sysimg);
    s.s = NULL;

    jl_gc_reset_alloc_count();
    jl_gc_enable(en);
    jl_cleanup_serializer2();
}

// TODO: need to enforce that the alignment of the buffer is suitable for vectors
JL_DLLEXPORT void jl_restore_system_image(const char *fname)
{
#ifndef JL_NDEBUG
    char *dot = fname ? (char*)strrchr(fname, '.') : NULL;
    int is_ji = (dot && !strcmp(dot, ".ji"));
    assert((is_ji || jl_sysimg_handle) && "System image file not preloaded");
#endif

    if (jl_sysimg_handle) {
        // load the pre-compiled sysimage from jl_sysimg_handle
        jl_load_sysimg_so();
    }
    else {
        ios_t f;
        if (ios_file(&f, fname, 1, 0, 0, 0) == NULL)
            jl_errorf("System image file \"%s\" not found.", fname);
        ios_bufmode(&f, bm_none);
        JL_SIGATOMIC_BEGIN();
        ios_seek_end(&f);
        size_t len = ios_pos(&f);
        char *sysimg = (char*)jl_gc_perm_alloc(len, 0, 64, 0);
        ios_seek(&f, 0);
        if (ios_readall(&f, sysimg, len) != len)
            jl_errorf("Error reading system image file.");
        ios_close(&f);
        ios_static_buffer(&f, sysimg, len);
        jl_restore_system_image_from_stream(&f);
        ios_close(&f);
        JL_SIGATOMIC_END();
    }
}

JL_DLLEXPORT void jl_restore_system_image_data(const char *buf, size_t len)
{
    ios_t f;
    JL_SIGATOMIC_BEGIN();
    ios_static_buffer(&f, (char*)buf, len);
    jl_restore_system_image_from_stream(&f);
    ios_close(&f);
    JL_SIGATOMIC_END();
}

// --- init ---

static void jl_init_serializer2(int for_serialize)
{
    if (for_serialize) {
        htable_new(&symbol_table, 0);
        htable_new(&fptr_to_id, sizeof(id_to_fptrs) / sizeof(*id_to_fptrs));
        htable_new(&backref_table, 0);
        arraylist_new(&builtin_typenames, 0);
        uintptr_t i;
        for (i = 0; id_to_fptrs[i] != NULL; i++) {
            ptrhash_put(&fptr_to_id, (void*)(uintptr_t)id_to_fptrs[i], (void*)(i + 2));
        }
    }
    else {
        arraylist_new(&deser_sym, 0);
    }
    nsym_tag = 0;
}

static void jl_cleanup_serializer2(void)
{
    htable_reset(&symbol_table, 0);
    htable_reset(&fptr_to_id, 0);
    htable_reset(&backref_table, 0);
    arraylist_free(&deser_sym);
    arraylist_free(&builtin_typenames);
}

#ifdef __cplusplus
}
#endif
