/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*
 *  (C) 2001 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 *
 */

#ifndef MPIR_HANDLEMEM_H_INCLUDED
#define MPIR_HANDLEMEM_H_INCLUDED

#include "mpichconfconst.h"
#include "mpichconf.h"
#include "mpir_objects.h"
#include "mpir_thread.h"

/* style: allow:printf:5 sig:0 */
#ifdef MPICH_DEBUG_HANDLEALLOC

/* The following is a handler that may be added to finalize to test whether
   handles remain allocated, including those from the direct blocks.

   When adding memory checking, this routine should be invoked as

   MPIR_Add_finalize(MPIR_check_handles_on_finalize, objmem, 1);

   as part of the object intialization.

   The algorithm follows the following approach:

   The memory allocation approach manages a list of available objects.
   These objects are allocated from several places:
      "direct" - this is a block of preallocated space
      "indirect" - this is a block of blocks that are allocated as necessary.
                   E.g., objmem_ptr->indirect[0..objmem_ptr->indirect_size-1]
		   are pointers (or null) to a block of memory.  This block is
		   then divided into objects that are added to the avail list.

   To provide information on the handles that are still in use, we must
   "repatriate" all of the free objects, at least virtually.  To give
   the best information, for each free item, we determine to which block
   it belongs.
*/
static inline int MPIR_check_handles_on_finalize(void *objmem_ptr)
{
    MPIR_Object_alloc_t *objmem = (MPIR_Object_alloc_t *) objmem_ptr;
    int i;
    MPIR_Handle_common *ptr;
    int leaked_handles = FALSE;
    int directSize = objmem->direct_size;
    char *direct = (char *) objmem->direct;
    char *directEnd = (char *) direct + directSize * objmem->size - 1;
    int nDirect = 0;
    int *nIndirect = 0;

    /* Return immediately if this object has not allocated any space */
    if (!objmem->initialized) {
        return 0;
    }

    if (objmem->indirect_size > 0) {
        nIndirect = (int *) MPL_calloc(objmem->indirect_size, sizeof(int));
    }
    /* Count the number of items in the avail list.  These include
     * all objects, whether direct or indirect allocation */
    ptr = objmem->avail;
    while (ptr) {
        /* printf("Looking at %p\n", ptr); */
        /* Find where this object belongs */
        if ((char *) ptr >= direct && (char *) ptr < directEnd) {
            nDirect++;
        }
        else {
            void **indirect = (void **) objmem->indirect;
            for (i = 0; i < objmem->indirect_size; i++) {
                char *start = indirect[i];
                char *end = start + HANDLE_NUM_INDICES * objmem->size;
                if ((char *) ptr >= start && (char *) ptr < end) {
                    nIndirect[i]++;
                    break;
                }
            }
            if (i == objmem->indirect_size) {
                /* Error - could not find the owning memory */
                /* Temp */
                printf("Could not place object at %p in handle memory for type %s\n", ptr,
                       MPIR_Handle_get_kind_str(objmem->kind));
                printf("direct block is [%p,%p]\n", direct, directEnd);
                if (objmem->indirect_size) {
                    printf("indirect block is [%p,%p]\n", indirect[0],
                           (char *) indirect[0] + HANDLE_NUM_INDICES * objmem->size);
                }
            }
        }
        ptr = ptr->next;
    }

    if (0) {
        /* Produce a report */
        printf("Object handles:\n\ttype  \t%s\n\tsize  \t%d\n\tdirect size\t%d\n\
\tindirect size\t%d\n", MPIR_Handle_get_kind_str(objmem->kind), objmem->size, objmem->direct_size, objmem->indirect_size);
    }
    if (nDirect != directSize) {
        leaked_handles = TRUE;
        printf("In direct memory block for handle type %s, %d handles are still allocated\n",
               MPIR_Handle_get_kind_str(objmem->kind), directSize - nDirect);
    }
    for (i = 0; i < objmem->indirect_size; i++) {
        if (nIndirect[i] != HANDLE_NUM_INDICES) {
            leaked_handles = TRUE;
            printf
                ("In indirect memory block %d for handle type %s, %d handles are still allocated\n",
                 i, MPIR_Handle_get_kind_str(objmem->kind), HANDLE_NUM_INDICES - nIndirect[i]);
        }
    }

    if (nIndirect) {
        MPL_free(nIndirect);
    }

    if (leaked_handles && MPIR_CVAR_ABORT_ON_LEAKED_HANDLES) {
        /* comm_world has been (or should have been) destroyed by this point,
         * pass comm=NULL */
        MPID_Abort(NULL, MPI_ERR_OTHER, 1, "ERROR: leaked handles detected, aborting");
        MPIR_Assert(0);
    }

    return 0;
}

#endif

/* This is the utility file for info that contains routines used to
   manage the arrays used to store handle objects.

   To use these routines, allocate the following in a utility file
   used by each object (e.g., info, datatype, comm, group, ...).
   (The comment format // is used in this example but the usual
   C comment convention should be used of course.)  The usage is described
   below.

   // Declarations begin here
   // Static declaration of the information about the block
   // Define the number of preallocated entries # omitted)
   define MPID_<OBJ>_PREALLOC 256
   MPIR_Object_alloc_t MPID_<obj>_mem = { 0, 0, 0, 0, MPID_<obj>,
				      sizeof(MPID_<obj>), MPID_<obj>_direct,
                                      MPID_<OBJ>_PREALLOC, };

   // Preallocated objects
   MPID_<obj> MPID_<obj>_direct[MPID_<OBJ>_PREALLOC];
   static int initialized = 0;

   // Next available object
   static int MPID_<obj> *avail = 0;

   // Extension (indirect) objects
   static MPID_<obj> *(*MPID_<obj>_indirect)[] = 0;
   static int MPID_<obj>_indirect_size = 0;
    // Declarations end here

   These provide for lazy initialization; applications that do not need a
   particular object will not include any of the code or even reference
   the space.

   Note that these routines are designed for the MPI objects, and include the
   creation of a "handle" that is appropriate for the MPI object value.

   None of these routines is thread-safe.  Any routine that uses them
   must ensure that only one thread at a time may call them.
*/

/* This routine is called by finalize when MPI exits */
static inline int MPIR_Handle_free(void *((*indirect)[]), int indirect_size)
{
    int i;

    /* Remove any allocated storage */
    for (i = 0; i < indirect_size; i++) {
        MPL_free((*indirect)[i]);
    }
    if (indirect) {
        MPL_free(indirect);
    }
    /* This does *not* remove any objects that the user created
     * and then did not destroy */
    return 0;
}

#if defined(MPL_VG_AVAILABLE)
#define HANDLE_VG_LABEL(objptr_, objsize_, handle_type_, is_direct_)                             \
    do {                                                                                         \
        if (MPL_VG_RUNNING_ON_VALGRIND()) {                                                      \
            char desc_str[256];                                                                  \
            MPL_snprintf(desc_str, sizeof(desc_str)-1,                                           \
                          "[MPICH handle: objptr=%p handle=0x%x %s/%s]",                         \
                          (objptr_), (objptr_)->handle,                                          \
                          ((is_direct_) ? "DIRECT" : "INDIRECT"),                                \
                          MPIR_Handle_get_kind_str(handle_type_));                               \
            /* we don't keep track of the block descriptor because the handle */                 \
            /* values never change once allocated */                                             \
            MPL_VG_CREATE_BLOCK((objptr_), (objsize_), desc_str);                                \
        }                                                                                        \
    } while (0)
#else
#define HANDLE_VG_LABEL(objptr_, objsize_, handle_type_, is_direct_) do {} while (0)
#endif

static inline void *MPIR_Handle_direct_init(void *direct,
                                            int direct_size, int obj_size, int handle_type)
{
    int i;
    MPIR_Handle_common *hptr = 0;
    char *ptr = (char *) direct;

    for (i = 0; i < direct_size; i++) {
        /* printf("Adding %p in %d\n", ptr, handle_type); */
        /* First cast to (void*) to avoid false warnings about alignment
         * (consider that a requirement of the input parameters) */
        hptr = (MPIR_Handle_common *) (void *) ptr;
        ptr = ptr + obj_size;
        hptr->next = ptr;
        hptr->handle = ((unsigned) HANDLE_KIND_DIRECT << HANDLE_KIND_SHIFT) |
            (handle_type << HANDLE_MPI_KIND_SHIFT) | i;

        HANDLE_VG_LABEL(hptr, obj_size, handle_type, 1);
    }

    if (hptr)
        hptr->next = 0;
    return direct;
}

/* indirect is really a pointer to a pointer to an array of pointers */
static inline void *MPIR_Handle_indirect_init(void *(**indirect)[],
                                              int *indirect_size,
                                              int indirect_num_blocks,
                                              int indirect_num_indices,
                                              int obj_size, int handle_type)
{
    void *block_ptr;
    MPIR_Handle_common *hptr = 0;
    char *ptr;
    int i;

    /* Must create new storage for dynamically allocated objects */
    /* Create the table */
    if (!*indirect) {
        /* printf("Creating indirect table with %d pointers to blocks in it\n", indirect_num_blocks); */
        *indirect = (void *) MPL_calloc(indirect_num_blocks, sizeof(void *));
        if (!*indirect) {
            return 0;
        }
        *indirect_size = 0;
    }

    /* See if we can allocate another block */
    if (*indirect_size >= indirect_num_blocks) {
        /* printf("Out of space in indirect table\n"); */
        return 0;
    }

    /* Create the next block */
    /* printf("Creating indirect block number %d with %d objects in it\n", *indirect_size, indirect_num_indices); */
    block_ptr = (void *) MPL_calloc(indirect_num_indices, obj_size);
    if (!block_ptr) {
        return 0;
    }
    ptr = (char *) block_ptr;
    for (i = 0; i < indirect_num_indices; i++) {
        /* Cast to (void*) to avoid false warning about alignment */
        hptr = (MPIR_Handle_common *) (void *) ptr;
        ptr = ptr + obj_size;
        hptr->next = ptr;
        hptr->handle = ((unsigned) HANDLE_KIND_INDIRECT << HANDLE_KIND_SHIFT) |
            (handle_type << HANDLE_MPI_KIND_SHIFT) | (*indirect_size << HANDLE_INDIRECT_SHIFT) | i;
        /* printf("handle=%#x handle_type=%x *indirect_size=%d i=%d\n", hptr->handle, handle_type, *indirect_size, i); */

        HANDLE_VG_LABEL(hptr, obj_size, handle_type, 0);
    }
    hptr->next = 0;
    /* We're here because avail is null, so there is no need to set
     * the last block ptr to avail */
    /* printf("loc of update is %x\n", &(**indirect)[*indirect_size]);  */
    (**indirect)[*indirect_size] = block_ptr;
    *indirect_size = *indirect_size + 1;
    return block_ptr;
}

static inline int MPIR_Handle_finalize(void *objmem_ptr)
{
    MPIR_Object_alloc_t *objmem = (MPIR_Object_alloc_t *) objmem_ptr;

    (void) MPIR_Handle_free(objmem->indirect, objmem->indirect_size);
    /* This does *not* remove any Info objects that the user created
     * and then did not destroy */

    /* at this point we are done with the memory pool, inform valgrind */
    MPL_VG_DESTROY_MEMPOOL(objmem_ptr);

    return 0;
}

/*+
  MPIR_Handle_obj_alloc - Create an object using the handle allocator

Input Parameters:
. objmem - Pointer to object memory block.

  Return Value:
  Pointer to new object.  Null if no more objects are available or can
  be allocated.

  Notes:
  In addition to returning a pointer to a new object, this routine may
  allocate additional space for more objects.

  This routine is thread-safe.

  This routine is performance-critical (it may be used to allocate
  MPI_Requests) and should not call any other routines in the common
  case.

  Threading: The 'MPID_THREAD_CS_ENTER/EXIT(POBJ, MPIR_THREAD_POBJ_HANDLE_MUTEX)' enables both
  finer-grain
  locking with a single global mutex and with a mutex specific for handles.

  +*/
#undef FUNCNAME
#define FUNCNAME MPIR_Handle_obj_alloc
#undef FCNAME
#define FCNAME MPL_QUOTE(FUNCNAME)
static inline void *MPIR_Handle_obj_alloc(MPIR_Object_alloc_t * objmem)
{
    void *ret;
    MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_POBJ_HANDLE_MUTEX);
    ret = MPIR_Handle_obj_alloc_unsafe(objmem);
    MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_POBJ_HANDLE_MUTEX);
    return ret;
}

#undef FUNCNAME
#define FUNCNAME MPIR_Handle_obj_alloc_unsafe
#undef FCNAME
#define FCNAME MPL_QUOTE(FUNCNAME)
static inline void *MPIR_Handle_obj_alloc_unsafe(MPIR_Object_alloc_t * objmem)
{
    MPIR_Handle_common *ptr;

    if (objmem->avail) {
        ptr = objmem->avail;
        objmem->avail = objmem->avail->next;
        /* We do not clear ptr->next as we set it to an invalid pattern
         * when doing memory debugging and we don't need to set it
         * for the production/default case */
        /* ptr points to object to allocate */
    }
    else {
        int objsize, objkind;

        objsize = objmem->size;
        objkind = objmem->kind;

        if (!objmem->initialized) {
            MPL_VG_CREATE_MEMPOOL(objmem, 0 /*rzB */ , 0 /*is_zeroed */);

            /* Setup the first block.  This is done here so that short MPI
             * jobs do not need to include any of the Info code if no
             * Info-using routines are used */
            objmem->initialized = 1;
            ptr = MPIR_Handle_direct_init(objmem->direct, objmem->direct_size, objsize, objkind);
            if (ptr) {
                objmem->avail = ptr->next;
            }

#ifdef MPICH_DEBUG_HANDLEALLOC
            /* The priority of these callbacks must be greater than
             * the priority of the callback that frees the objmem direct and
             * indirect storage. */
            MPIR_Add_finalize(MPIR_check_handles_on_finalize, objmem,
                              MPIR_FINALIZE_CALLBACK_HANDLE_CHECK_PRIO);
#endif
            MPIR_Add_finalize(MPIR_Handle_finalize, objmem, 0);
            /* ptr points to object to allocate */
        }
        else {
            /* no space left in direct block; setup the indirect block. */

            ptr = MPIR_Handle_indirect_init(&objmem->indirect,
                                            &objmem->indirect_size,
                                            HANDLE_NUM_BLOCKS,
                                            HANDLE_NUM_INDICES, objsize, objkind);
            if (ptr) {
                objmem->avail = ptr->next;
            }

            /* ptr points to object to allocate */
        }
    }

    if (ptr) {
#ifdef USE_MEMORY_TRACING
        /* We set the object to an invalid pattern.  This is similar to
         * what is done by MPL_trmalloc by default (except that trmalloc uses
         * 0xda as the byte in the memset)
         */
        /* if the object was previously freed then MEMPOOL_FREE marked it as
         * NOACCESS, so we need to make it addressable again before memsetting
         * it */
        /* save and restore the handle -- it's a more robust method than
         * encoding the layout of the structure */
        int tmp_handle;
        MPL_VG_MAKE_MEM_DEFINED(ptr, objmem->size);
        tmp_handle = ptr->handle;
        memset(ptr, 0xef, objmem->size);
        ptr->handle = tmp_handle;
#endif /* USE_MEMORY_TRACING */
        /* mark the mem as addressable yet undefined if valgrind is available */
        MPL_VG_MEMPOOL_ALLOC(objmem, ptr, objmem->size);
        /* the handle value is always valid at return from this function */
        MPL_VG_MAKE_MEM_DEFINED(&ptr->handle, sizeof(ptr->handle));

        /* necessary to prevent annotations from being misinterpreted.  HB/HA
         * arcs will be drawn between a req object in across a free/alloc
         * boundary otherwise */
        /* NOTE: basically causes DRD's --trace-addr option to be useless for
         * handlemem-allocated objects. Consider one of the trace-inducing
         * annotations instead. */
        MPL_VG_ANNOTATE_NEW_MEMORY(ptr, objmem->size);

        MPL_DBG_MSG_FMT(MPIR_DBG_HANDLE, TYPICAL, (MPL_DBG_FDEST,
                                                   "Allocating object ptr %p (handle val 0x%08x)",
                                                   ptr, ptr->handle));
    }

    return ptr;
}

/*+
  MPIR_Handle_obj_free - Free an object allocated with MPID_Handle_obj_new

Input Parameters:
+ objmem - Pointer to object block
- object - Object to delete

  Notes:
  This routine assumes that only a single thread calls it at a time; this
  is true for the SINGLE_CS approach to thread safety
  +*/
static inline void MPIR_Handle_obj_free(MPIR_Object_alloc_t * objmem, void *object)
{
    MPIR_Handle_common *obj = (MPIR_Handle_common *) object;

    MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_POBJ_HANDLE_MUTEX);

    MPL_DBG_MSG_FMT(MPIR_DBG_HANDLE, TYPICAL, (MPL_DBG_FDEST,
                                               "Freeing object ptr %p (0x%08x kind=%s) refcount=%d",
                                               (obj),
                                               (obj)->handle,
                                               MPIR_Handle_get_kind_str(HANDLE_GET_MPI_KIND
                                                                        ((obj)->handle)),
                                               MPIR_Object_get_ref(obj)));

#ifdef USE_MEMORY_TRACING
    {
        /* set the object memory to an invalid value (0xec), except for the handle field */
        int tmp_handle;
        tmp_handle = obj->handle;
        memset(obj, 0xec, objmem->size);
        obj->handle = tmp_handle;
    }
#endif

    if (MPL_VG_RUNNING_ON_VALGRIND()) {
        int tmp_handle = obj->handle;

        MPL_VG_MEMPOOL_FREE(objmem, obj);
        /* MEMPOOL_FREE marks the object NOACCESS, so we have to make the
         * MPIR_Handle_common area that is used for internal book keeping
         * addressable again. */
        MPL_VG_MAKE_MEM_DEFINED(&obj->handle, sizeof(obj->handle));
        MPL_VG_MAKE_MEM_UNDEFINED(&obj->next, sizeof(obj->next));

        /* tt#1626: MEMPOOL_FREE also uses the value of `--free-fill=0x..` to
         * memset the object, so we need to restore the handle */
        obj->handle = tmp_handle;

        /* Necessary to prevent annotations from being misinterpreted.  HB/HA arcs
         * will be drawn between a req object in across a free/alloc boundary
         * otherwise.  Specifically, stores to obj->next when obj is actually an
         * MPIR_Request falsely look like a race to DRD and Helgrind because of the
         * other lockfree synchronization used with requests. */
        MPL_VG_ANNOTATE_NEW_MEMORY(obj, objmem->size);
    }

    obj->next = objmem->avail;
    objmem->avail = obj;
    MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_POBJ_HANDLE_MUTEX);
}

/*
 * Get an pointer to dynamically allocated storage for objects.
 */
static inline void *MPIR_Handle_get_ptr_indirect(int handle, MPIR_Object_alloc_t * objmem)
{
    int block_num, index_num;

    /* Check for a valid handle type */
    if (HANDLE_GET_MPI_KIND(handle) != objmem->kind) {
        return 0;
    }

    /* Find the block */
    block_num = HANDLE_BLOCK(handle);
    if (block_num >= objmem->indirect_size) {
        return 0;
    }

    /* Find the entry */
    index_num = HANDLE_BLOCK_INDEX(handle);
    /* If we could declare the blocks to a known size object, we
     * could do something like
     * return &((MPIR_Info**)*MPIU_Info_mem.indirect)[block_num][index_num];
     * since we cannot, we do the calculation by hand.
     */
    /* Get the pointer to the block of addresses.  This is an array of
     * void * */
    {
        char *block_ptr;
        /* Get the pointer to the block */
        block_ptr = (char *) (*(objmem->indirect))[block_num];
        /* Get the item */
        block_ptr += index_num * objmem->size;
        return block_ptr;
    }
}


#endif /* MPIR_HANDLEMEM_H_INCLUDED */
