/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/* 
 *
 *   Copyright (C) 1997 University of Chicago. 
 *   See COPYRIGHT notice in top-level directory.
 */

#include <assert.h>
#include "adio.h"
#include "adio_extern.h"
#ifdef AGGREGATION_PROFILE
#include "mpe.h"
#endif
#include "heap-sort.h"

/*
#define DEBUG1
#define DEBUG2
#define DEBUG3
*/
/* #define DEBUG_HEAP */

#define DTYPE_SKIP

#ifdef DEBUG3
static char *off_type_name[MAX_OFF_TYPE] = {"TEMP_OFFSETS",
				     "REAL_OFFSETS"};
#endif

/* Simple function to return the size of the view_state. */
static inline ADIO_Offset view_state_get_cur_sz(view_state *tmp_view_state_p,
                                                int op_type)
{
    flatten_state *tmp_state_p = NULL;
    switch(op_type)
    {   
        case TEMP_OFF:
            tmp_state_p = &(tmp_view_state_p->tmp_state);
            break;
        case REAL_OFF:
            tmp_state_p = &(tmp_view_state_p->cur_state);
            break;
        default:
            fprintf(stderr, "op_type invalid\n");
    }
    return tmp_state_p->cur_sz;
}

/* Simple function to return the len of the next piece of the view_state. */
static inline ADIO_Offset view_state_get_next_len(view_state *tmp_view_state_p,
						  int op_type)
{
    flatten_state *tmp_state_p = NULL;
    switch(op_type)
    {
	case TEMP_OFF:
	    tmp_state_p = &(tmp_view_state_p->tmp_state);
	    break;
	case REAL_OFF:
	    tmp_state_p = &(tmp_view_state_p->cur_state);
	    break;
	default:
	    fprintf(stderr, "op_type invalid\n");
    }
    return (ADIO_Offset) 
	tmp_view_state_p->flat_type_p->blocklens[tmp_state_p->idx] -
	tmp_state_p->cur_reg_off;
}

/* Add up to a region of a file view and no larger than a max size.
 * The view_state is always consistent with the abs_off and where the
 * index and cur_reg_off point to.  The regions should be coalesced if
 * possible later on. */
static inline int view_state_add_region(
    ADIO_Offset max_sz,
    view_state *tmp_view_state_p, 
    ADIO_Offset *st_reg_p,
    ADIO_Offset *tmp_reg_sz_p,
    int op_type)
{
    ADIOI_Flatlist_node *tmp_flat_type_p = NULL;
    flatten_state *tmp_state_p = NULL;
    int64_t data_sz = 0;

#ifdef AGGREGATION_PROFILE
    /* MPE_Log_event (5020, 0, NULL); */
#endif

    switch(op_type)
    {
	case TEMP_OFF:
	    tmp_state_p = &(tmp_view_state_p->tmp_state);
	    break;
	case REAL_OFF:
	    tmp_state_p = &(tmp_view_state_p->cur_state);
	    break;
	default:
	    fprintf(stderr, "op_type invalid\n");
    }

    tmp_flat_type_p = tmp_view_state_p->flat_type_p;

    *st_reg_p = tmp_state_p->abs_off;

    /* Should be looking at some data (or it's a zero len blocklens
     * (i.e. placeholder). */
    assert(tmp_state_p->cur_reg_off != 
	   tmp_flat_type_p->blocklens[tmp_state_p->idx]);
    /* Shouldn't have been called if the view_state is done. */
    assert(tmp_state_p->cur_sz != tmp_view_state_p->sz);

    /* Make sure we are not in a non-zero region in the flat_type */
    assert(tmp_flat_type_p->blocklens[tmp_state_p->idx] != 0);
    
#ifdef DEBUG3
    fprintf(stderr, "view_state:(blocklens[%Ld]=%d,cur_reg_off=%Ld,"
	    "max_sz=%Ld)\n", tmp_state_p->idx, 
	    tmp_flat_type_p->blocklens[tmp_state_p->idx], 
	    tmp_state_p->cur_reg_off, max_sz);
#endif

    /* Can it add the whole piece? */
    if (tmp_flat_type_p->blocklens[tmp_state_p->idx] - 
	tmp_state_p->cur_reg_off <= max_sz)
    {
	data_sz = tmp_flat_type_p->blocklens[tmp_state_p->idx] -
            tmp_state_p->cur_reg_off;

	tmp_state_p->cur_sz += data_sz;

	/* Advance the abs_off to the beginning of the next piece */
	if (tmp_flat_type_p->count == 1)
	{
	    assert(tmp_flat_type_p->blocklens[tmp_state_p->idx] != 0);
	    tmp_state_p->abs_off += data_sz;
#ifdef DEBUG3 
	    fprintf(stderr, "view_state_add_region: %s contig type "
		    "(old abs_off=%Ld,abs_off=%Ld,cur_sz=%Ld,reg size=%Ld)\n", 
		    off_type_name[op_type], tmp_state_p->abs_off - data_sz, 
		    tmp_state_p->abs_off, tmp_state_p->cur_sz, data_sz);
#endif
	}
	else
	{ 
	    /* Is this the last region in the datatype? */
	    if (tmp_state_p->idx == (tmp_flat_type_p->count - 1))
	    {
		tmp_state_p->abs_off += data_sz -
		    tmp_flat_type_p->indices[tmp_flat_type_p->count-1] -
		    tmp_flat_type_p->blocklens[tmp_flat_type_p->count-1] +
		    tmp_view_state_p->ext;
#ifdef DEBUG3
	    fprintf(stderr, "view_state_add_region: %s last region for type "
		    "(old abs_off=%Ld,abs_off=%Ld,cur_sz=%Ld,reg size=%Ld)\n", 
		    off_type_name[op_type], tmp_state_p->abs_off - data_sz, 
		    tmp_state_p->abs_off, tmp_state_p->cur_sz, data_sz);
#endif
	    }
	    else
	    {
		tmp_state_p->abs_off += 
		    tmp_flat_type_p->indices[tmp_state_p->idx + 1] -
		    (tmp_flat_type_p->indices[tmp_state_p->idx] +
		     tmp_state_p->cur_reg_off);
#ifdef DEBUG3
	    fprintf(stderr, "view_state_add_region: %s inner region type "
		    "(old abs_off=%Ld,abs_off=%Ld,cur_sz=%Ld,reg size=%Ld)\n", 
		    off_type_name[op_type], tmp_state_p->abs_off - 
		    (tmp_flat_type_p->indices[tmp_state_p->idx + 1] -
                    (tmp_flat_type_p->indices[tmp_state_p->idx] +
                     tmp_state_p->cur_reg_off)), tmp_state_p->abs_off, 
		    tmp_state_p->cur_sz, data_sz);
#endif
	    }
	    /* Increment idx to next non-zero region in the flat_type */
	    do {
		tmp_state_p->idx = 
		    (tmp_state_p->idx + 1) % tmp_flat_type_p->count;
	    } while (tmp_flat_type_p->blocklens[tmp_state_p->idx] == 0);
	}
	tmp_state_p->cur_reg_off = 0;
    }
    else /* Add part of the piece */
    {
	data_sz = max_sz;
	tmp_state_p->cur_reg_off += data_sz;
	tmp_state_p->abs_off += data_sz;
	tmp_state_p->cur_sz += data_sz;
#ifdef DEBUG3 
	    fprintf(stderr, "view_state_add_region: %s partial region type "
		    "(cur_reg_off=%Ld,abs_off=%Ld,cur_sz=%Ld,reg size=%Ld\n", 
		    off_type_name[op_type], tmp_state_p->cur_reg_off, 
		    tmp_state_p->abs_off, tmp_state_p->cur_sz, data_sz);
#endif
    }

    *tmp_reg_sz_p = data_sz;
#ifdef AGGREGATION_PROFILE
    /* MPE_Log_event (5021, 0, NULL); */
#endif
    return 0;
}

/* Set up the abs_off, idx, and cur_reg_off of a view_state for the
 * tmp_state or the cur_state. */
int ADIOI_init_view_state(int file_ptr_type,
		    int nprocs, 
		    view_state *view_state_arr,
		    int op_type)
{
    ADIOI_Flatlist_node *tmp_flat_type_p = NULL;
    ADIO_Offset tmp_off_used = 0, st_reg = 0, tmp_reg_sz = 0;
    int i;
    flatten_state *tmp_state_p = NULL;
    view_state *tmp_view_p = NULL;

    for (i = 0; i < nprocs; i++)
    {
	switch(op_type)
	{
	    case TEMP_OFF:
		tmp_state_p = &(view_state_arr[i].tmp_state);
		break;
	    case REAL_OFF:
		tmp_state_p = &(view_state_arr[i].cur_state);
		break;
	    default:
		fprintf(stderr, "op_type invalid\n");
	}
	
	tmp_view_p = &(view_state_arr[i]);
	tmp_flat_type_p = tmp_view_p->flat_type_p;

	if (file_ptr_type == ADIO_INDIVIDUAL)
	    tmp_state_p->abs_off = tmp_view_p->fp_ind;
	else
	    tmp_state_p->abs_off = tmp_view_p->disp;
	
	tmp_off_used = 0;

	/* initialize tmp_state idx */
	while (tmp_flat_type_p->blocklens[tmp_state_p->idx] == 0)
	    tmp_state_p->idx = (tmp_state_p->idx + 1) % tmp_flat_type_p->count;
	if (file_ptr_type == ADIO_EXPLICIT_OFFSET)
	    tmp_state_p->abs_off += tmp_flat_type_p->indices[tmp_state_p->idx];

	/* Initialize the abs_off by moving into the datatype 
	 * byte_off bytes.  Since we only do this in the beginning, we
	 * make the assumption that pieces are added whole until the last
	 * piece which MAY be partial. */
	while (tmp_off_used != tmp_view_p->byte_off)
	{
	    view_state_add_region(
		tmp_view_p->byte_off - tmp_off_used,
		&(view_state_arr[i]), &st_reg, &tmp_reg_sz, 
		op_type);
	}
	
	/* Re-initialize the cur_size so that the abs_off was set to
	 * the proper position while the actual size = 0.*/
	tmp_state_p->cur_sz = 0;
#ifdef DEBUG1
	fprintf(stderr, "init_view_state: %s (idx=%d,byte_off=%Ld,"
		"abs_off=%Ld,reg_off=%Ld,sz=%Ld)\n", off_type_name[op_type], 
		i, tmp_view_p->byte_off, tmp_state_p->abs_off,
		tmp_state_p->cur_reg_off, tmp_view_p->sz);
#endif	

    }
    return 0;
}

/* Return the next file realm offset and length for this datatype state
 * within a particular file realm. */
static inline int get_next_fr_off(ADIO_File fd,
				  ADIO_Offset off,
				  ADIO_Offset fr_st_off,
				  MPI_Datatype *fr_type_p,
				  ADIO_Offset *fr_next_off_p,
				  ADIO_Offset *fr_max_len_p) 
{
    MPI_Aint fr_extent = -1;
    ADIO_Offset tmp_off, off_rem;
    ADIOI_Flatlist_node *fr_node_p;
    int i = -1, fr_dtype_ct = 0;

    /* Should have already been flattened in calc_file_realms() */
    fr_node_p = ADIOI_Flatten_and_find(*fr_type_p);
    assert(fr_node_p != NULL);

    /* Did we get to the first region of the file realm? */
    if (off - fr_st_off < 0)
    {
	*fr_next_off_p = fr_st_off + fr_node_p->indices[0];
	*fr_max_len_p = fr_node_p->blocklens[0];
	return 0;
    }

    /* Calculate how many times to loop through the fr_type 
     * and where the next fr_off is. */
    MPI_Type_extent(*fr_type_p, &fr_extent);
    tmp_off = off - fr_st_off;
    fr_dtype_ct = tmp_off / fr_extent;
    off_rem = tmp_off % fr_extent;
    for (i = 0; i < fr_node_p->count; i++)
    {
	if (off_rem < fr_node_p->indices[i])
	{
	    *fr_next_off_p = fr_st_off +
		(fr_dtype_ct * fr_extent) + fr_node_p->indices[i];
	    *fr_max_len_p = fr_node_p->blocklens[i];
	    return 0;
	}
	else if (off_rem < fr_node_p->indices[i] + fr_node_p->blocklens[i])
	{
	    *fr_next_off_p = off;
	    *fr_max_len_p = fr_node_p->blocklens[i] - 
		(off_rem - fr_node_p->indices[i]);
	    return off;
	}
    }
    
    /* Shouldn't get here. */
    fprintf(stderr, "get_next_fr_off: Couldn't find the correct "
	    "location of the next offset for this file realm.\n");
    return -1;
}

/* Look in all the view states for the first offset within a given
 * file realm.  Report the end of a contiguous region within the file
 * realm (possibly more than the actual view state may be able to
 * process contiguously). */
static inline int find_next_off(ADIO_File fd,
				view_state *view_state_p,
				ADIO_Offset fr_st_off,
				MPI_Datatype *fr_type_p,
				int op_type,
				ADIO_Offset *cur_off_p,
				ADIO_Offset *cur_reg_max_len_p)
{
    ADIOI_Flatlist_node *tmp_flat_type_p = NULL;
    ADIO_Offset tmp_off = -1, fr_next_off = -1, fr_max_len = -1, 
	tmp_fr_max_len = -1;
    int ret = 0;
    flatten_state *tmp_state_p = NULL;
    ADIO_Offset tmp_st_off = 0, tmp_reg_sz = 0;
#ifdef DTYPE_SKIP
    int skip_type_ct;
#endif

#ifdef AGGREGATION_PROFILE
    /* MPE_Log_event (5022, 0, NULL); */
#endif

    switch(op_type)
    {
	case TEMP_OFF:
	    tmp_state_p = &(view_state_p->tmp_state);
	    break;
	case REAL_OFF:
	    tmp_state_p = &(view_state_p->cur_state);
	    break;
	default:
	    fprintf(stderr, "op_type invalid\n");
    }
	
    tmp_flat_type_p = view_state_p->flat_type_p;

    /* Can we use this proc? */
    if (tmp_state_p->cur_sz < view_state_p->sz) {
	tmp_st_off = 0;
	tmp_reg_sz = 0;
	/* If the current region is not within the file realm, advance
	 * the state until it is and calculate the end of the next file 
	 * realm in fr_max_len. */
	ret = get_next_fr_off(fd,
			      tmp_state_p->abs_off, 
			      fr_st_off,
			      fr_type_p,
			      &fr_next_off,
			      &fr_max_len);
	
	while ((tmp_state_p->abs_off < fr_next_off) &&
	       (tmp_state_p->cur_sz != view_state_p->sz))
	{
	    
	/* While this might appear to be erroneous at first,
	 * view_state_add_region can only add a single piece at a
	 * time.  Therefore, it will never overshoot the beginning
	 * of the next file realm.  When it finally does enter the
	 * next file realm it will not be able to go beyond its
	 * first piece. */
	    
#ifdef DTYPE_SKIP
	    if (tmp_flat_type_p->count > 1) {
		/* let's see if we can skip whole datatypes */
		skip_type_ct = (fr_next_off - tmp_state_p->abs_off) /
		    view_state_p->ext;
		if (skip_type_ct > 0) {
		    /* before we go on, let's check if we've actually
		     * finished up already */
		    tmp_state_p->cur_sz += skip_type_ct *
			view_state_p->type_sz;
		    if (tmp_state_p->cur_sz >= view_state_p->sz) {
			tmp_state_p->cur_sz = view_state_p->sz;
			break;
		    }
		    tmp_state_p->abs_off += skip_type_ct * view_state_p->ext;
		}
	    }
#endif
	    view_state_add_region(
		fr_next_off - tmp_state_p->abs_off,
		view_state_p,
		&tmp_st_off,
		&tmp_reg_sz,
		op_type);

	    ret = get_next_fr_off(fd,
				  tmp_state_p->abs_off, 
				  fr_st_off,
				  fr_type_p,
				  &fr_next_off,
				  &fr_max_len);
	}

	if (tmp_state_p->cur_sz != view_state_p->sz) {
	    tmp_off = tmp_state_p->abs_off;
	    /* Calculate how much of the remaining file realm there is from the
	     * current offset */
	    tmp_fr_max_len = fr_next_off + fr_max_len - tmp_off;
	}
    }

    *cur_off_p = tmp_off;
    *cur_reg_max_len_p = tmp_fr_max_len;
#ifdef AGGREGATION_PROFILE
    /* MPE_Log_event (5023, 0, NULL); */
#endif
    return ret;
}

/* Upon completion of a full collective buffer, end of a file realm
 * region (data sieving), or the end of all I/O for an aggregator, we
 * should return a list of MPI_Datatypes that correspond to client
 * communication into a collective buffer, a list of corresponding
 * sizes, and an aggregate MPI_Datatype which will be used as a
 * filetype in MPI_File_write/read on the aggregator. */ 
int ADIOI_Build_agg_reqs(ADIO_File fd, int rw_type, int nprocs,
			 view_state *client_file_view_state_arr,
			 MPI_Datatype *client_comm_dtype_arr,
			 ADIO_Offset *client_comm_sz_arr,
			 ADIO_Offset *agg_dtype_offset_p,
			 MPI_Datatype *agg_dtype_p)
{
    MPI_Aint **client_disp_arr = NULL, *agg_disp_arr = NULL;
    int **client_blk_arr = NULL, *agg_blk_arr = NULL;
    ADIO_Offset tmp_coll_buf_sz = 0, st_reg = 0, act_reg_sz = 0;
    ADIO_Offset cur_off = -1, cur_reg_max_len = -1;
    ADIO_Offset ds_fr_end = -1;
    ADIO_Offset *fr_st_off_arr = fd->file_realm_st_offs;
    MPI_Datatype *fr_type_arr = fd->file_realm_types;
    int *client_ol_ct_arr = NULL;
    int *client_ol_cur_ct_arr = NULL;
    int agg_ol_ct = 0, agg_ol_cur_ct = 0;
    int cur_off_proc = -1;
    int next_off_idx = -1;
    int i = 0, j = 0, all_done = -1;
    int agg_idx = fd->my_cb_nodes_index;
    heap_t offset_heap;
    ADIO_Offset next_off = -1, next_reg_max_len = -1;

    /* Used for coalescing ol pairs next to each other. */
    ADIO_Offset *client_comm_next_off_arr = NULL;
    ADIO_Offset agg_next_off = -1;
#ifdef AGGREGATION_PROFILE
    MPE_Log_event (5016, 0, NULL);
#endif

    memset(client_comm_sz_arr, 0, nprocs*sizeof(ADIO_Offset));

    if ((client_comm_next_off_arr = (ADIO_Offset *) 
	 ADIOI_Malloc(nprocs*sizeof(ADIO_Offset))) == NULL)
    {
	fprintf(stderr, "ADIOI_Build_agg_reqs: malloc client_next_off_arr "
		"failed\n");
	return -1;
    }
    
    if ((client_ol_ct_arr = (int *) ADIOI_Calloc(nprocs, sizeof(int))) == NULL)
    {
	fprintf(stderr, "ADIOI_Build_agg_reqs: "
		"malloc client_ol_ct_arr failed\n");
	return -1;
    }
    if ((client_ol_cur_ct_arr = 
	 (int *) ADIOI_Calloc(nprocs, sizeof(int))) == NULL)
    {
	fprintf(stderr, "ADIOI_Build_agg_reqs: "
		"malloc client_ol_cur_ct_arr failed\n");
	return -1;
    }

    /* On the first pass see how many offset-length pairs are
     * necessary for each client.  Then allocate the correct amount of
     * offset-length pairs for describing the collective buffer.  All
     * data is processed in order by the aggregator's file realm.  On
     * the second pass, set the offset-length pairs to the correct
     * values. */
    for (i = 0; i < MAX_OFF_TYPE; i++)
    {
	memset(client_comm_next_off_arr, -1, nprocs*sizeof(ADIO_Offset));
	tmp_coll_buf_sz = 0;
	ds_fr_end = -1;

	/* initialize heap */
	ADIOI_Heap_create(&offset_heap, nprocs);
	offset_heap.size = 0;
	
	for (j=0; j<nprocs; j++) {
	    find_next_off(fd, 
			  &client_file_view_state_arr[j],
			  fr_st_off_arr[agg_idx],
			  &(fr_type_arr[agg_idx]),
			  i,
			  &cur_off,
			  &cur_reg_max_len);
	    if ((cur_off != -1) && (cur_reg_max_len > 0)) {
		ADIOI_Heap_insert(&offset_heap, cur_off, j, cur_reg_max_len);
#ifdef DEBUG_HEAP
		printf ("initial: inserting offset %lld with "
			"cur_reg_max_len = %lld for p%d\n",
			cur_off, cur_reg_max_len, j);
#endif
	    }

	}
	if (!offset_heap.size)
	    ADIOI_Heap_insert(&offset_heap, -1, -1, -1);

	while (tmp_coll_buf_sz < fd->hints->cb_buffer_size)
	{
	    /* Find the next process with the next region within the
	     * file realm and the maximum amount that can be added for
	     * this particular file realm as a contiguous region. */
	    ADIOI_Heap_extract_min(&offset_heap, &cur_off, &cur_off_proc,
			     &cur_reg_max_len);
#ifdef DEBUG_HEAP
	    printf ("extracted cur_off %lld from proc %d\n",
		    cur_off, cur_off_proc);
#endif

	    if (cur_off == -1)
		break;
	    
#ifdef DEBUG3
	    fprintf(stderr, "ADIOI_Build_agg_reqs: %s proc %d start/add to"
		    " list (max_reg_fr=%Ld,tmp_coll_buf_sz=%Ld,"
		    "cb_buffer_size=%d)\n", off_type_name[i], cur_off_proc,
		    cur_reg_max_len, tmp_coll_buf_sz, 
		    fd->hints->cb_buffer_size);
#endif
	    
	    /* We process only contiguous file realm regions if we are
	     * using data sieving. Note that we only do this for
	     * writes since reads can be data sieved across each other
	     * without consistency issues. */
	    if ((fd->hints->ds_write == ADIOI_HINT_ENABLE ||
		 fd->hints->ds_write == ADIOI_HINT_AUTO) &&
		rw_type == ADIOI_WRITE && fd->hints->cb_nodes > 1)
	    {
#ifdef DEBUG2
		fprintf(stderr, "ADIOI_Build_agg_reqs: "
			"Warning - Data sieving writes on\n");
#endif
		if (ds_fr_end == -1)
		{
		    ds_fr_end = cur_off + cur_reg_max_len;
#ifdef DEBUG1
		fprintf(stderr, "ADIOI_Build_agg_reqs: "
			"cur_off=%Ld, cur_reg_max_len=%Ld\n"
			"Data sieving file realm end initialized to %Ld\n",
			cur_off,
			cur_reg_max_len,
			ds_fr_end);
#endif
		}
		else
		{
		    /* The next off switched file realms, so we will stop
		     * here. */
		    if (ds_fr_end != cur_off + cur_reg_max_len)
		    {
#ifdef DEBUG1
			fprintf(stderr, "ADIOI_Build_agg_reqs: "
				"Data sieving file realm end changed from "
				"%Ld to %Ld\n", ds_fr_end, 
				cur_off + cur_reg_max_len);
#endif
			break;
		    }
		}
	    }
	    
	    /* Add up to the end of the file realm or the collective
	     * buffer. */
	    if (cur_reg_max_len > (fd->hints->cb_buffer_size - 
				   tmp_coll_buf_sz))
		cur_reg_max_len = fd->hints->cb_buffer_size - tmp_coll_buf_sz;

	    view_state_add_region(
		cur_reg_max_len,
		&(client_file_view_state_arr[cur_off_proc]), 
		&st_reg, &act_reg_sz, i);

	    switch(i)
	    {
		case TEMP_OFF:
		    /* Increment the ol list count for each proc and
		     * the used part of the collective buffer if the
		     * next region is not adjacent to the previous
		     * region. */
		    if (client_comm_next_off_arr[cur_off_proc] != 
			tmp_coll_buf_sz)
		    {
			(client_ol_ct_arr[cur_off_proc])++;
		    }
		    client_comm_next_off_arr[cur_off_proc] = 
			tmp_coll_buf_sz + act_reg_sz;
		    
		    if (agg_next_off != st_reg)
			agg_ol_ct++;
		    agg_next_off = st_reg + act_reg_sz;
		    break;
		case REAL_OFF:
		    /* Add this region to the proper client ol list if
		     * the next region is not adjacent to the previous
		     * region. */
		    next_off_idx = client_ol_cur_ct_arr[cur_off_proc];
		    if (client_comm_next_off_arr[cur_off_proc] != 
			tmp_coll_buf_sz)
		    {
			client_disp_arr[cur_off_proc][next_off_idx] =
			    tmp_coll_buf_sz;
			client_blk_arr[cur_off_proc][next_off_idx] = 
			    act_reg_sz;
			(client_ol_cur_ct_arr[cur_off_proc])++;
		    }
		    else
		    {
			client_blk_arr[cur_off_proc][next_off_idx - 1] 
			    += act_reg_sz;
		    }
		    client_comm_sz_arr[cur_off_proc] += act_reg_sz;
		    client_comm_next_off_arr[cur_off_proc] =
			tmp_coll_buf_sz + act_reg_sz;
		    
		    /* Add to the aggregator filetype if the next
		     * region is not adjacent to the previous
		     * region. */
		    if (agg_next_off != st_reg)
		    {
			/* this will enable initial offsets much further into
			 * the file than an MPI_Aint */
			if (!agg_ol_cur_ct)
			    *agg_dtype_offset_p = st_reg;
			agg_disp_arr[agg_ol_cur_ct] = st_reg -
			    (MPI_Aint) *agg_dtype_offset_p;
			agg_blk_arr[agg_ol_cur_ct] = act_reg_sz;	
			agg_ol_cur_ct++;
		    }
		    else
		    {
			agg_blk_arr[agg_ol_cur_ct - 1] += act_reg_sz;
		    }
		    agg_next_off = st_reg + act_reg_sz;
		    
		    break;
		default:
		    fprintf(stderr, "ADIOI_Build_agg_reqs: Impossible type\n");
	    }
	    tmp_coll_buf_sz += act_reg_sz;

	    find_next_off(fd,
			  &client_file_view_state_arr[cur_off_proc],
			  fr_st_off_arr[agg_idx],
			  &(fr_type_arr[agg_idx]),
			  i,
			  &next_off,
			  &next_reg_max_len);

	    if ((next_off != -1) || (!offset_heap.size)) {
		ADIOI_Heap_insert(&offset_heap, next_off, cur_off_proc,
			    next_reg_max_len);
#ifdef DEBUG_HEAP
		printf ("inserting offset %lld for p%d\n", next_off,
			cur_off_proc);
#endif
	    }
	}
	
	if (i == TEMP_OFF)
	{
	    /* Allocate offset-length pairs for creating hindexed
	     * MPI_Datatypes for both the client and the aggregator. */
	    if ((client_disp_arr = (MPI_Aint **) 
		 ADIOI_Malloc(nprocs*sizeof(MPI_Aint *))) == NULL)
	    {
		fprintf(stderr, "ADIOI_Build_agg_reqs: malloc "
			"client_disp_arr failed\n");
		return -1;
	    }
	    if ((client_blk_arr = (int **) ADIOI_Malloc(
		     nprocs*sizeof(int *))) == NULL)
	    {
		ADIOI_Free(client_disp_arr);
		fprintf(stderr, "ADIOI_Build_agg_reqs: malloc "
			"client_blk_arr failed\n");
		return -1;
	    }    
	    for (j = 0; j < nprocs; j++)
	    {
		if ((client_disp_arr[j] = (MPI_Aint *) ADIOI_Malloc(
			 client_ol_ct_arr[j]*sizeof(MPI_Aint))) == NULL)
		{
		    fprintf(stderr, "ADIOI_Build_agg_reqs: malloc "
			    "client_disp_arr[%d] failed\n", j);
		    return -1;
		}
		if ((client_blk_arr[j] = (int *) 
		     ADIOI_Malloc(client_ol_ct_arr[j]*sizeof(int))) == NULL)
		{
		    ADIOI_Free(client_disp_arr[j]);
		    fprintf(stderr, "ADIOI_Build_agg_reqs: malloc "
			    "client_blk_arr[%d] failed\n", j);
		    return -1;
		}
	    }
	    
	    if (agg_ol_ct > 0) 
	    {
		if ((agg_disp_arr = (MPI_Aint *) ADIOI_Malloc(
			 agg_ol_ct*sizeof(MPI_Aint))) == NULL)
		{
		    fprintf(stderr, 
			    "ADIOI_Build_agg_reqs: malloc disp_arr failed\n");
		    return -1;
		}
		if ((agg_blk_arr = (int *) 
		     ADIOI_Malloc(agg_ol_ct*sizeof(int))) == NULL)
		{
		    ADIOI_Free(agg_disp_arr);
		    fprintf(stderr, 
			    "ADIOI_Build_agg_reqs: malloc blk_arr failed\n");
		    return -1;
		}
	    }
	}
	ADIOI_Heap_free(&offset_heap);
    }
    
    /* Let the clients know if this aggregator is totally finished
     * with all possible client requests. */
    all_done = 1;
    for (i = 0; i < nprocs; i++)
    {
	if ((client_file_view_state_arr[i].cur_state.cur_sz !=
            client_file_view_state_arr[i].sz) ||
            client_comm_sz_arr[i] != 0)
	{
	    all_done = 0;
	    break;
	}
    }
    if (all_done == 1)
    {
	for (i = 0; i < nprocs; i++)
	{
	    client_comm_sz_arr[i] = -1;
	}
    }

    /* Quick check to make sure we found all the ol pairs we thought
     * we did */
    for (i = 0; i < nprocs; i++)
    {
	if (client_ol_cur_ct_arr[i] != client_ol_ct_arr[i])
	{
	    fprintf(stderr, "ADIOI_Build_agg_reqs: ERROR Process %d "
		    "processed only %d out of %d ol pairs\n", i, 
		    client_ol_cur_ct_arr[i],
		    client_ol_ct_arr[i]);
	    return -1;
	}
    }
#ifdef DEBUG1
    fprintf(stderr, "ADIOI_Build_agg_reqs:(client,ol_pairs,size_req)=");
    for (i = 0; i < nprocs; i++)
    {
	fprintf(stderr, "(%d,%d,%Ld)", i, client_ol_ct_arr[i],
		client_comm_sz_arr[i]);
	if (i != nprocs - 1)
	    fprintf(stderr, ",");
    }
    fprintf(stderr, "\n");
#endif
#ifdef DEBUG1
    fprintf(stderr, "ADIOI_Build_agg_reqs: Generated %d of %d "
	    "aggregate offset-length pairs\n", agg_ol_cur_ct, agg_ol_ct);
#endif
#ifdef DEBUG2
    for (i = 0; i < nprocs; i++)
    {
	if (client_ol_ct_arr[i] > 0)
	{
	    fprintf(stderr, "ADIOI_Build_agg_reqs: p %d (off,len) = ", i);
	    for (j = 0; j < client_ol_ct_arr[i]; j++)
	    {
		fprintf(stderr, "[%d](%d,%d) ", j, 
			client_disp_arr[i][j],
			client_blk_arr[i][j]);
	    }
	    fprintf(stderr, "\n");
	}
    }    
    if (agg_ol_ct) {
	fprintf(stderr, "ADIOI_Build_agg_reqs:agg_type(off,len)=");
	for (i = 0; i < agg_ol_ct; i++)
	    {
		fprintf(stderr, "[%d](%d,%d)",
			i, agg_disp_arr[i], agg_blk_arr[i]);
		if (i != agg_ol_ct - 1)
		    fprintf(stderr, ",");
	    }
	fprintf(stderr, "\n");
    }
#endif

    assert(agg_ol_cur_ct == agg_ol_ct);

    /* Create all the client and aggregate MPI_Datatypes */
    for (i = 0; i < nprocs; i++)
    {
	if (client_comm_sz_arr[i] > 0)
	{
	    MPI_Type_hindexed(client_ol_ct_arr[i], client_blk_arr[i],
			      client_disp_arr[i], MPI_BYTE, 
			      &(client_comm_dtype_arr[i]));
	    MPI_Type_commit(&(client_comm_dtype_arr[i]));
	}
	else
	{
	    client_comm_dtype_arr[i] = MPI_BYTE;
	}
	ADIOI_Free(client_blk_arr[i]);
	ADIOI_Free(client_disp_arr[i]);
    }
    ADIOI_Free(client_blk_arr);
    ADIOI_Free(client_disp_arr);

    if (agg_ol_ct > 0) {
	if (agg_ol_ct == 1)
	    MPI_Type_contiguous (agg_blk_arr[0], MPI_BYTE, agg_dtype_p);
	else if (agg_ol_ct > 1)
	    MPI_Type_hindexed(agg_ol_ct, agg_blk_arr, agg_disp_arr, MPI_BYTE,
			      agg_dtype_p);    

	MPI_Type_commit(agg_dtype_p);

	ADIOI_Free(agg_disp_arr);
	ADIOI_Free(agg_blk_arr);
    }
    ADIOI_Free(client_ol_ct_arr);
    ADIOI_Free(client_ol_cur_ct_arr);
    ADIOI_Free(client_comm_next_off_arr);
#ifdef AGGREGATION_PROFILE
    MPE_Log_event (5017, 0, NULL);
#endif
    return 0;
}

/* All sizes from all aggregators are gathered on the clients, which
 * then call this function, which will generate the comm datatypes for
 * each aggregator (agg_comm_dtype_arr) in the upcoming
 * MPI_Alltoallw() */
int ADIOI_Build_client_reqs(ADIO_File fd, 
			    int nprocs,
			    view_state *my_mem_view_state_arr,
			    view_state *agg_file_view_state_arr,
			    ADIO_Offset *agg_comm_sz_arr,
			    MPI_Datatype *agg_comm_dtype_arr)
{
    MPI_Aint **agg_disp_arr = NULL;
    int **agg_blk_arr = NULL;
    view_state *tmp_mem_state_p = NULL, *tmp_file_state_p = NULL;
    ADIO_Offset total_agg_comm_sz = 0, cur_total_agg_comm_sz = 0;
    ADIO_Offset st_reg = 0, act_reg_sz = 0, tmp_reg_sz = 0;
    ADIO_Offset cur_off = -1, cur_reg_max_len = -1;
    ADIO_Offset tmp_cur_off = -1, tmp_cur_reg_max_len = -1;
    ADIO_Offset agg_mem_st_reg = 0, agg_mem_act_reg_sz = 0;
    ADIO_Offset *fr_st_off_arr = fd->file_realm_st_offs;
    ADIO_Offset *agg_comm_cur_sz_arr = NULL;
    MPI_Datatype *fr_type_arr = fd->file_realm_types;
    int *agg_ol_ct_arr = NULL;
    int *agg_ol_cur_ct_arr = NULL;
    int tmp_agg_fr_idx = -1;
    int cur_off_proc = -1;
    int i = 0, j = 0;
    int agg_next_off_idx = -1;
    /* Used for coalescing ol pairs next to each other. */
    ADIO_Offset *agg_mem_next_off_arr = NULL;
#ifdef AGGREGATION_PROFILE
    MPE_Log_event (5018, 0, NULL);
#endif

#ifdef DEBUG
    fprintf(stderr, "ADIOI_Build_client_reqs:(agg,size_req)=");
    for (i = 0; i < nprocs; i++)
    {
	int tmp_agg_idx = ADIOI_Agg_idx(i, fd);
        if (tmp_agg_idx >= 0)
        {
	    fprintf(stderr, "(%d,%Ld)", i, agg_comm_sz_arr[i]);
	    if (i != fd->hints->cb_nodes - 1)
		fprintf(stderr, ",");
	}
	fprintf(stderr, "\n");
    }
#endif
    
    if ((agg_mem_next_off_arr = (ADIO_Offset *) ADIOI_Malloc(
	     nprocs*sizeof(ADIO_Offset))) == NULL)
    {
	fprintf(stderr, "ADIOI_Build_client_reqs: malloc agg_mem_next_off_arr"
		"failed\n");
	return -1;
    }

    if ((agg_comm_cur_sz_arr = (ADIO_Offset *) 
	 ADIOI_Malloc(nprocs*sizeof(ADIO_Offset))) == NULL)
    {
	fprintf(stderr, "ADIOI_Build_client_reqs: malloc agg_comm_cur_sz_arr"
		" failed\n");
	return -1;
    }
    if ((agg_ol_ct_arr = (int *) ADIOI_Calloc(nprocs, sizeof(int)))
	== NULL)
    {
	fprintf(stderr, "ADIOI_Build_client_reqs: "
		"malloc agg_ol_ct_arr failed\n");
	return -1;
    }
    if ((agg_ol_cur_ct_arr = (int *) ADIOI_Calloc(nprocs, sizeof(int)))
	== NULL)
    {
	fprintf(stderr, "ADIOI_Build_client_reqs: "
		"malloc agg_ol_cur_ct_arr failed\n");
	return -1;
    }

    for (i = 0; i < nprocs; i++)
    {
	if (agg_comm_sz_arr[i] > 0)
	    total_agg_comm_sz += agg_comm_sz_arr[i];
    }
    
    /* On the first pass see how many offset-length pairs are
     * necessary for each aggregator.  Then allocate the correct
     * amount of offset-length pairs for handling each aggregator's
     * particular data size.  On the last pass, we actually create the
     * offset-length pairs. */
    for (i = 0; i < MAX_OFF_TYPE; i++)
    {
	cur_total_agg_comm_sz = 0;
	memset(agg_comm_cur_sz_arr, 0, nprocs*sizeof(ADIO_Offset));
	memset(agg_mem_next_off_arr, -1, nprocs*sizeof(ADIO_Offset));
	while (total_agg_comm_sz > cur_total_agg_comm_sz)
	{
	    /* Look for the next aggregator offset among all the
	     * aggregators and their respective file realms. */
	    cur_off = -1;
	    for (j = 0; j < nprocs; j++)
	    {
		tmp_agg_fr_idx = ADIOI_Agg_idx(j, fd);
                assert(tmp_agg_fr_idx < fd->hints->cb_nodes);
		
		/* If this process is not an aggregator or we have
		 * finished all the bytes for this aggregator, move
		 * along. */
		if (tmp_agg_fr_idx < 0 || 
		    agg_comm_cur_sz_arr[j] == agg_comm_sz_arr[j])
		{
		    continue;
		}

		find_next_off(fd,
			      &(agg_file_view_state_arr[j]),
			      fr_st_off_arr[tmp_agg_fr_idx],
			      &(fr_type_arr[tmp_agg_fr_idx]),
			      i,
			      &tmp_cur_off,
			      &tmp_cur_reg_max_len);
		if (tmp_cur_off == -1)
		    continue;	       

		if ((cur_off == -1) || 
		    (cur_off > tmp_cur_off))
		{
		    cur_off_proc = j;
		    cur_off = tmp_cur_off;
		    cur_reg_max_len = tmp_cur_reg_max_len;
		}
	    }

	    assert(cur_off_proc != -1);
	    
	    /* Add up to the end of the file realm or as many bytes
	     * are left for this particular aggregator in the client's
	     * filetype */
	    if (cur_reg_max_len > agg_comm_sz_arr[cur_off_proc] - 
		agg_comm_cur_sz_arr[cur_off_proc])
	    {
		cur_reg_max_len = agg_comm_sz_arr[cur_off_proc] - 
		    agg_comm_cur_sz_arr[cur_off_proc];
	    }
	    assert(cur_reg_max_len > 0);
	    
	    view_state_add_region(
		cur_reg_max_len,
		&(agg_file_view_state_arr[cur_off_proc]),
		&st_reg, &act_reg_sz, i);
	    
#ifdef DEBUG2
	    fprintf(stderr, "ADIOI_Build_client_reqs: %s File region"
		    " (proc=%d,off=%Ld,sz=%Ld)\n",
		    off_type_name[i], cur_off_proc,
		    cur_off, act_reg_sz);
#endif

	    /* Before translating the file regions to memory regions,
	     * we first must advance to the proper point in the
	     * mem_view_state for this aggregator to match the
	     * file_view_state. */
	    tmp_file_state_p = &(agg_file_view_state_arr[cur_off_proc]);
	    tmp_mem_state_p = &(my_mem_view_state_arr[cur_off_proc]);
	    assert(view_state_get_cur_sz(tmp_file_state_p, i) - act_reg_sz >=
		   view_state_get_cur_sz(tmp_mem_state_p, i));
	    while (view_state_get_cur_sz(tmp_file_state_p, i) - act_reg_sz != 
		   view_state_get_cur_sz(tmp_mem_state_p, i))
	    {
		ADIO_Offset fill_st_reg = -1, fill_reg_sz = -1;
		view_state_add_region(
		    view_state_get_cur_sz(tmp_file_state_p, i) - act_reg_sz -
		    view_state_get_cur_sz(tmp_mem_state_p, i),
		    tmp_mem_state_p,
		    &fill_st_reg,
		    &fill_reg_sz, i);
	    }
	    
	    /* Based on how large the act_reg_sz 1. Figure out how
	     * many memory offset-length pairs are necessary. 2. Set
	     * the offset-length pairs. */
	    tmp_reg_sz = 0;
	    while (tmp_reg_sz != act_reg_sz)
	    {
		view_state_add_region(
		    act_reg_sz - tmp_reg_sz,
		    tmp_mem_state_p,
		    &agg_mem_st_reg, &agg_mem_act_reg_sz, 
		    i);
		tmp_reg_sz += agg_mem_act_reg_sz;

#ifdef DEBUG2
		fprintf(stderr, "ADIOI_Build_client_reqs: Mem region %s"
			"(proc=%d,off=%Ld,sz=%Ld)\n",
			off_type_name[i], cur_off_proc,
			agg_mem_st_reg, agg_mem_act_reg_sz);
#endif
		agg_comm_cur_sz_arr[cur_off_proc] += agg_mem_act_reg_sz;
		cur_total_agg_comm_sz += agg_mem_act_reg_sz;	    
		switch(i)
		{
		    case TEMP_OFF:
			/* Increment the ol list count a particular
			 * aggregator if next region is not adjacent
			 * to the previous region. */
			if (agg_mem_next_off_arr[cur_off_proc] != 
			    agg_mem_st_reg)
			{
			    agg_ol_ct_arr[cur_off_proc]++;
			}
			agg_mem_next_off_arr[cur_off_proc] = 
			    agg_mem_st_reg + agg_mem_act_reg_sz;
			break;
		    case REAL_OFF:
			/* Set the ol list for the memtypes that will
			 * map to each aggregator, coaslescing if
			 * possible. */
			agg_next_off_idx = agg_ol_cur_ct_arr[cur_off_proc];
			if (agg_mem_next_off_arr[cur_off_proc] != 
			    agg_mem_st_reg)
			{
			    agg_disp_arr[cur_off_proc][agg_next_off_idx] = 
				agg_mem_st_reg;
			    agg_blk_arr[cur_off_proc][agg_next_off_idx] = 
				agg_mem_act_reg_sz;
			    (agg_ol_cur_ct_arr[cur_off_proc])++;
			}
			else
			{
			    agg_blk_arr[cur_off_proc][agg_next_off_idx - 1]
				+= agg_mem_act_reg_sz;
			}
			agg_mem_next_off_arr[cur_off_proc] = 
			    agg_mem_st_reg + agg_mem_act_reg_sz;
			break;
		    default:
			fprintf(stderr, "ADIOI_Build_client_reqs: "
				"Impossible type\n");
		}
	    }
	}
	
	/* On the first pass, allocate the memory structures for
	 * creating the MPI_hindexed type. */
	if (i == TEMP_OFF)
	{	    
	    /* Allocate offset-length pairs for creating hindexed
	     * MPI_Datatypes for each aggregator */
	    if ((agg_disp_arr = (MPI_Aint **) 
		 ADIOI_Malloc(nprocs*sizeof(MPI_Aint *))) == NULL)
	    {
		fprintf(stderr, 
			"ADIOI_Build_client_reqs: malloc agg_disp_arr failed\n");
		return -1;
	    }
	    if ((agg_blk_arr = (int **) ADIOI_Malloc(nprocs*sizeof(int *))) 
		== NULL)
	    {
		ADIOI_Free(agg_disp_arr);
		fprintf(stderr, 
			"ADIOI_Build_client_reqs: malloc agg_blk_arr failed\n");
		return -1;
	    }    
	    for (j = 0; j < nprocs; j++)
	    {
		if ((agg_disp_arr[j] = (MPI_Aint *) 
		     ADIOI_Malloc(agg_ol_ct_arr[j]*sizeof(MPI_Aint))) == NULL)
		{
		    fprintf(stderr, "ADIOI_Build_client_reqs: malloc "
			    "agg_disp_arr[%d] failed\n", j);
		    return -1;
		}
		if ((agg_blk_arr[j] = (int *) 
		     ADIOI_Malloc(agg_ol_ct_arr[j]*sizeof(int))) == NULL)
		{
		    ADIOI_Free(agg_disp_arr[j]);
		    fprintf(stderr, "ADIOI_Build_client_reqs: malloc "
			    "agg_blk_arr[%d] failed\n", j);
		    return -1;
		}
	    }
	}
    }

#ifdef DEBUG
    fprintf(stderr, "ADIOI_Build_client_reqs:(agg,cur_ol_count=ol_count)=");
    for (i = 0; i < nprocs; i++)
    {
	int tmp_agg_idx = ADIOI_Agg_idx(i, fd);
	if (tmp_agg_idx >= 0)
	{
	    fprintf(stderr, "(%d,%d=%d)", i, agg_ol_cur_ct_arr[i],
		    agg_ol_ct_arr[i]);
	    assert(agg_ol_ct_arr[i] == agg_ol_cur_ct_arr[i]);
	    if (tmp_agg_idx != fd->hints->cb_nodes - 1)
		fprintf(stderr, ",");
	}
    }
    fprintf(stderr, "\n");
#endif

#ifdef DEBUG2
    for (i = 0; i < nprocs; i++)
    {
	if (agg_ol_ct_arr[i] > 0)
	{
	    fprintf(stderr, "ADIOI_Build_client_reqs: p %d (off,len) = ", i);
	    for (j = 0; j < agg_ol_ct_arr[i]; j++)
	    {
		fprintf(stderr, "[%d](%d,%d) ", j,
			agg_disp_arr[i][j],
			agg_blk_arr[i][j]);
	    }
	    fprintf(stderr, "\n");
	}
    }
#endif

    /* Create all the aggregator MPI_Datatypes */
    for (i = 0; i < nprocs; i++)
    {
	if (agg_comm_sz_arr[i] > 0)
	{
	    MPI_Type_hindexed(agg_ol_ct_arr[i], agg_blk_arr[i],
                              agg_disp_arr[i], MPI_BYTE,
                              &(agg_comm_dtype_arr[i]));
            MPI_Type_commit(&(agg_comm_dtype_arr[i]));
	}
	else
	{
	    agg_comm_dtype_arr[i] = MPI_BYTE;
	}
	ADIOI_Free(agg_blk_arr[i]);
	ADIOI_Free(agg_disp_arr[i]);
    }
    ADIOI_Free(agg_blk_arr);
    ADIOI_Free(agg_disp_arr);

    ADIOI_Free(agg_mem_next_off_arr);
    ADIOI_Free(agg_comm_cur_sz_arr);
    ADIOI_Free(agg_ol_ct_arr);
    ADIOI_Free(agg_ol_cur_ct_arr);
#ifdef AGGREGATION_PROFILE
    MPE_Log_event (5019, 0, NULL);
#endif    
    return 0;
}
/* ADIOI_Build_client_pre_req allows a client to calculate the memtype
 * offset-length pairs up (up to a limit - max_pre_req_sz or max
 * ol_ct). It basically allows ADIOI_Build_client_req to do less work.
 * If it called and there already exist some preprocessed memtype
 * offset-length pairs, it will exit immediately if a limit has been
 * reached or if will add on the old limites to reach the new
 * limits. */

int ADIOI_Build_client_pre_req(ADIO_File fd,
			       int agg_rank, int agg_idx,
			       view_state *my_mem_view_state_p,
			       view_state *agg_file_view_state_p,
			       ADIO_Offset max_pre_req_sz,
			       int max_ol_ct)
{
    ADIO_Offset act_reg_sz = 0, tmp_reg_sz = 0;
    ADIO_Offset cur_off = -1, cur_reg_max_len = -1;
    ADIO_Offset agg_mem_st_reg = 0, agg_mem_act_reg_sz = 0;
    int agg_ol_ct = 0, agg_ol_cur_ct = 0;
    int i, agg_next_off_idx = -1;

    ADIO_Offset cur_sz = 0, max_sz = 0, agg_mem_next_off = -1;
    ADIO_Offset fill_st_reg = -1, fill_reg_sz = -1;
    ADIO_Offset *fr_st_off_arr = fd->file_realm_st_offs;
    MPI_Datatype *fr_type_arr = fd->file_realm_types;
    MPI_Aint *tmp_disp_arr = NULL;
    int *tmp_blk_arr = NULL, exit_loop = -1;
    flatten_state *tmp_mem_state_p = NULL, *tmp_file_state_p = NULL;
#ifdef DTYPE_SKIP
    int skip_type_ct;
#endif
    if (agg_idx < 0 || agg_idx >= fd->hints->cb_nodes)
    {
        fprintf(stderr, "ADIOI_Build_client_pre_req: Invalid agg_idx %d\n",
		agg_idx);
        return -1;
    }

    if (agg_file_view_state_p->cur_state.cur_sz == 
	agg_file_view_state_p->sz || max_pre_req_sz <= 0 ||
	max_ol_ct <= 0)
    {
#ifdef DEBUG1
	fprintf(stderr, 
		"ADIOI_Build_client_pre_req: Nothing to preprocess\n");
#endif
	return 0;
    }

    /* The new limits have already been surpassed by what already
     * exists.  Otherwise we will use the next restrictions */
    if ((my_mem_view_state_p->pre_sz >= max_pre_req_sz) ||
	(my_mem_view_state_p->pre_ol_ct >= max_ol_ct))
    {
#ifdef DEBUG1
	fprintf(stderr, 
		"ADIOI_Build_client_pre_req:  Old values surpass new "
		"pre_req values\n");
#endif
	return 0;
    }
    
    /* General idea is to first advance the filetype to the file realm
     * and then the memtype to the filetype.  The memtype is advanced
     * further by peeking at the filetype and then the filetype is
     * advanced. */
    for (i = 0; i < MAX_OFF_TYPE; i++)
    {
	switch(i)
	{
	    case TEMP_OFF:
		tmp_mem_state_p  = &(my_mem_view_state_p->tmp_state);
		tmp_file_state_p = &(agg_file_view_state_p->tmp_state);
		break;
	    case REAL_OFF:
		tmp_mem_state_p  = &(my_mem_view_state_p->cur_state);
		tmp_file_state_p = &(agg_file_view_state_p->cur_state);
		break;
	    default:
		fprintf(stderr, "ADIOI_Build_client_pre_req: "
			"Invalid off type %d\n", i);
	}

	if (i == TEMP_OFF && my_mem_view_state_p->pre_sz > 0)
	{
	    cur_sz = my_mem_view_state_p->pre_sz;
            agg_ol_ct = my_mem_view_state_p->pre_ol_ct;
	    /* Save the old arrays */
	    tmp_disp_arr = my_mem_view_state_p->pre_disp_arr;
	    tmp_blk_arr  = my_mem_view_state_p->pre_blk_arr;
	    my_mem_view_state_p->pre_disp_arr = NULL;
	    my_mem_view_state_p->pre_blk_arr  = NULL;
            agg_mem_next_off =
		tmp_disp_arr[agg_ol_ct - 1] + tmp_blk_arr[agg_ol_ct - 1];
	}
	else if (i == REAL_OFF && my_mem_view_state_p->pre_sz > 0)
	{
	    cur_sz = my_mem_view_state_p->pre_sz;
	    agg_ol_cur_ct = my_mem_view_state_p->pre_ol_ct;
	    
	    /* Copy the old data to the new data, freeing the old
	     * arrays */
	    memcpy(my_mem_view_state_p->pre_disp_arr, tmp_disp_arr, 
		   my_mem_view_state_p->pre_ol_ct * sizeof(MPI_Aint));
	    memcpy(my_mem_view_state_p->pre_blk_arr, tmp_blk_arr, 
		   my_mem_view_state_p->pre_ol_ct * sizeof(int));

	    ADIOI_Free(tmp_disp_arr);
	    ADIOI_Free(tmp_blk_arr);

	    agg_mem_next_off = 
		my_mem_view_state_p->pre_disp_arr[agg_ol_cur_ct - 1] +
		my_mem_view_state_p->pre_blk_arr[agg_ol_cur_ct - 1];
	}
	else
	{
	    cur_sz = 0;
	}
	
	/* Max_pre_req_sz may be larger than the amount of data left
	 * to preprocess */
	if (max_pre_req_sz - cur_sz > 
	    agg_file_view_state_p->sz - tmp_file_state_p->cur_sz)
	{
	    max_sz = cur_sz +
		agg_file_view_state_p->sz - tmp_file_state_p->cur_sz;
	}
	else
	    max_sz = max_pre_req_sz;
	
	assert(cur_sz != max_sz);
#ifdef DEBUG1
	fprintf(stderr, 
		"ADIOI_Build_client_pre_req: (cur_sz=%Ld,agg_ol_ct=%d,"
		"agg_mem_next_off=%Ld,max_sz=%Ld,max_ol_ct=%d)\n", 
		cur_sz, agg_ol_ct, agg_mem_next_off, max_sz, max_ol_ct);
#endif
	while (cur_sz < max_sz)
	{
	    find_next_off(fd, agg_file_view_state_p,
			  fr_st_off_arr[agg_rank],
			  &(fr_type_arr[agg_rank]),
			  i,
			  &cur_off,
			  &cur_reg_max_len);
	    
	    /* find_next_off may show that the file_view_state is done
	     * even if cur_sz != max_sz since find_next_off may
	     * advance the file_view_state to the end here and realize
	     * that it is done. */
	    if (cur_off == -1)
		break;

	    assert(cur_off != -1);
	    
	    /* Before translating the file regions to memory regions,
	     * we first must advance to the proper point in the
	     * mem_view_state for this aggregator to match the
	     * file_view_state. */
	    while (tmp_file_state_p->cur_sz != tmp_mem_state_p->cur_sz)
	    {
#ifdef DTYPE_SKIP
		if (my_mem_view_state_p->flat_type_p->count > 1) {
		    /* let's see if we can skip whole memory datatypes */
		    skip_type_ct =
			(tmp_file_state_p->cur_sz - tmp_mem_state_p->cur_sz) /
			my_mem_view_state_p->type_sz;
		    if (skip_type_ct > 0) {
			tmp_mem_state_p->cur_sz +=
			    skip_type_ct * my_mem_view_state_p->type_sz;
			tmp_mem_state_p->abs_off +=
			    skip_type_ct * my_mem_view_state_p->ext;
			if (tmp_mem_state_p->cur_sz ==
			    tmp_file_state_p->cur_sz)
			    break;
		    }
		}
#endif
		view_state_add_region(
		    tmp_file_state_p->cur_sz - tmp_mem_state_p->cur_sz,
		    my_mem_view_state_p,
		    &fill_st_reg,
		    &fill_reg_sz, i);
	    }

	    /* Now that the filetype and memtype are advanced to the
	     * same position, add memtype ol-pairs while we have not
	     * overstepped the min(end of the current piece in the
	     * file view, end of the file realm, data left in
	     * max_sz) */
	    
	    if (cur_reg_max_len >  
		view_state_get_next_len(agg_file_view_state_p, i))
		cur_reg_max_len =  
		    view_state_get_next_len(agg_file_view_state_p, i);

	    if (cur_reg_max_len > max_sz - cur_sz)
		cur_reg_max_len = max_sz - cur_sz;

	    assert(cur_reg_max_len > 0);

	    /* Add memtype ol pairs while we have not passed
	     * cur_reg_max_len or the max number of ol pairs
	     * allowed */
	    act_reg_sz = 0;
	    exit_loop = 0;
	    while ((act_reg_sz < cur_reg_max_len) && 
		   (exit_loop == 0))
	    {
		view_state_add_region(
		    cur_reg_max_len - act_reg_sz,
		    my_mem_view_state_p,
		    &agg_mem_st_reg, &agg_mem_act_reg_sz, 
		    i);
		act_reg_sz += agg_mem_act_reg_sz;
		
#ifdef DEBUG2
		fprintf(stderr, "ADIOI_Build_client_pre_req: %s Mem region"
			"(proc=%d,off=%Ld,sz=%Ld)\n",
			off_type_name[i], agg_rank, agg_mem_st_reg, 
			agg_mem_act_reg_sz);
#endif
		switch(i)
		{
		    case TEMP_OFF:
			/* Increment the ol list count if the next
			 * region is not adjacent to the previous
			 * region. */
			if (agg_mem_next_off != agg_mem_st_reg)
			{
			    agg_ol_ct++;
			    if (agg_ol_ct == max_ol_ct)
				exit_loop = 1;
			}
			agg_mem_next_off = 
			    agg_mem_st_reg + agg_mem_act_reg_sz;
			break;
		    case REAL_OFF:
			/* Set the ol list for the memtype that
			 * will map to our aggregator, coaslescing
			 * if possible. */
			agg_next_off_idx = agg_ol_cur_ct;
			if (agg_mem_next_off != agg_mem_st_reg)
			{
			    my_mem_view_state_p->
				pre_disp_arr[agg_next_off_idx] = 
				agg_mem_st_reg;
			    my_mem_view_state_p->
				pre_blk_arr[agg_next_off_idx] = 
				agg_mem_act_reg_sz;
			    agg_ol_cur_ct++;
			    if (agg_ol_cur_ct == agg_ol_ct)
				exit_loop = 1;
			}
			else
			{
			    my_mem_view_state_p->
				pre_blk_arr[agg_next_off_idx - 1]
				+= agg_mem_act_reg_sz;
			}
			agg_mem_next_off = 
			    agg_mem_st_reg + agg_mem_act_reg_sz;
			break;
		    default:
			fprintf(stderr, "ADIOI_Build_client_pre_req: "
				"Impossible type\n");
		}
	    }

	    /* Advance the filetype flatten state appropriately to
	     * match the data advanced in the memtype flatten state.
	     * Should only take at most a single view_state_add_region
	     * call since the memtype cannot proceed beyond the end of
	     * a contig piece in the file type. */
	    view_state_add_region(act_reg_sz - tmp_reg_sz,
				  agg_file_view_state_p,
				  &fill_st_reg, &fill_reg_sz, i);
#ifdef DEBUG2
	    fprintf(stderr, "ADIOI_Build_client_pre_req: %s File region"
		    " (proc=%d,off=%Ld,sz=%Ld)\n",
		    off_type_name[i], agg_rank, fill_st_reg, fill_reg_sz);
#endif
	    if (fill_reg_sz != act_reg_sz)
	    {
		fprintf(stderr, "ADIOI_Build_client_pre_req: "
			"view_state_add_region failed to match the memtype\n");
		return -1;
	    }
	    
	    cur_sz += act_reg_sz;
	}
	
	/* On the first pass, allocate the memory structures for
	 * storing the preprocessed information */
	if (i == TEMP_OFF)
	{
	    if ((my_mem_view_state_p->pre_disp_arr = (MPI_Aint *)
		 ADIOI_Malloc(agg_ol_ct * sizeof(MPI_Aint))) == NULL)
	    {
		fprintf(stderr, "ADIOI_Build_client_pre_req: malloc "
                        "pre_disp_arr of size %ld failed\n",
                        (long int)agg_ol_ct * sizeof(MPI_Aint));
                return -1;
	    }
	    if ((my_mem_view_state_p->pre_blk_arr = (int *) 
		 ADIOI_Malloc(agg_ol_ct * sizeof(int))) == NULL)
	    {
		ADIOI_Free(my_mem_view_state_p->pre_disp_arr);
		fprintf(stderr, "ADIOI_Build_client_pre_req: malloc "
			"agg_blk_arr of size %ld failed\n",
			(long int)agg_ol_ct * sizeof(int));
		return -1;
	    }
	}
    }

    my_mem_view_state_p->pre_sz = cur_sz;
    my_mem_view_state_p->pre_ol_ct = agg_ol_ct;

#ifdef DEBUG1
    fprintf(stderr, "ADIOI_Build_client_pre_req:(agg=%d,cur_ol_count=%d"
	    "=ol_count=%d)\n",
	    agg_rank, my_mem_view_state_p->pre_ol_ct, agg_ol_ct);
#endif

#ifdef DEBUG2
    if (agg_ol_ct > 0)
    {
	fprintf(stderr, "ADIOI_Build_client_pre_req: agg=%d,pre_sz=%Ld "
		"(off,len) = \n", agg_rank, my_mem_view_state_p->pre_sz);
	for (i = 0; i < my_mem_view_state_p->pre_ol_ct; i++)
	{
	    fprintf(stderr, "[%d](%d,%d) ", i, 
		    my_mem_view_state_p->pre_disp_arr[i], 
		    my_mem_view_state_p->pre_blk_arr[i]);
	    if (i % 5 == 0 && i != 0)
		fprintf(stderr, "\n");
	}
	fprintf(stderr, "\n");
    }
#endif

    return 0;
}

/* process_pre_req() allows ADIOI_Build_client_req to use the pre_req
 * information. */

static int process_pre_req(ADIO_File fd,
                           int agg_rank,
                           int agg_idx,
                           view_state *my_mem_view_state_p,
                           view_state *agg_file_view_state_p,
                           ADIO_Offset agg_comm_sz,
			   int off_type,
			   MPI_Aint *agg_disp_arr,
			   int *agg_blk_arr,
			   ADIO_Offset *agg_comm_pre_sz_p,
			   ADIO_Offset *agg_comm_cur_sz_p,
			   ADIO_Offset *agg_comm_sz_p,
			   int *agg_ol_cur_ct_p,
			   int *agg_ol_ct_p,
			   ADIO_Offset *agg_mem_next_off_p)
{
    int i, has_partial = 0;
    MPI_Aint partial_disp = 0;
    int partial_len = 0;
    ADIO_Offset tmp_agg_comm_pre_sz = 0;

    assert (my_mem_view_state_p->pre_sz > 0);
    switch(off_type)
    {
	case TEMP_OFF:
	    /* Use only some of the precalculated data */
	    if (my_mem_view_state_p->pre_sz > *agg_comm_sz_p)
	    {
		for (i = 0; i < my_mem_view_state_p->pre_ol_ct; i++)
		{
		    if ((my_mem_view_state_p->pre_blk_arr[i] + 
			 *agg_comm_pre_sz_p) > *agg_comm_sz_p)
		    {
			has_partial = 1;
			partial_len = *agg_comm_sz_p - *agg_comm_pre_sz_p;
			*agg_comm_pre_sz_p = *agg_comm_sz_p;
			i++;
			break;
		    }
		    else if ((my_mem_view_state_p->pre_blk_arr[i] +
			      *agg_comm_pre_sz_p) == *agg_comm_sz_p)
		    {
			*agg_comm_pre_sz_p += 
			    my_mem_view_state_p->pre_blk_arr[i];
			i++;
			break;
		    }
		    else
			*agg_comm_pre_sz_p += 
			    my_mem_view_state_p->pre_blk_arr[i];
		}
		
		if (has_partial == 1)
		{
		    *agg_mem_next_off_p = 
			my_mem_view_state_p->pre_disp_arr[i - 1] + 
			partial_len;
		}
		else
		{
		    *agg_mem_next_off_p = 
			my_mem_view_state_p->pre_disp_arr[i - 1] + 
			my_mem_view_state_p->pre_blk_arr[i - 1];
		}
		
		*agg_comm_cur_sz_p = *agg_comm_pre_sz_p;
		*agg_ol_ct_p = i;
		
	    }
	    else /* Use all the precalculated data */
	    {
		*agg_comm_pre_sz_p = my_mem_view_state_p->pre_sz;
		*agg_comm_cur_sz_p = *agg_comm_pre_sz_p;
		*agg_ol_ct_p = my_mem_view_state_p->pre_ol_ct;
		*agg_mem_next_off_p = 
		    my_mem_view_state_p->pre_disp_arr[
			my_mem_view_state_p->pre_ol_ct - 1] +
		    my_mem_view_state_p->pre_blk_arr[
			my_mem_view_state_p->pre_ol_ct - 1];
	    }
#ifdef DEBUG1
	    fprintf(stderr, "process_pre_req: TEMP_OFF "
		    "agg_comm_pre_sz=%Ld,agg_comm_cur_sz=%Ld,agg_ol_ct=%d\n",
		    *agg_comm_pre_sz_p, *agg_comm_cur_sz_p, *agg_ol_ct_p);
#endif
	    assert(*agg_comm_cur_sz_p <= *agg_comm_sz_p);
	    break;
	case REAL_OFF:
	    /* Set the ol list for the memtype that will map to our
	     * aggregator, coaslescing if possible. */
	    for (i = 0; i < my_mem_view_state_p->pre_ol_ct; i++)
	    {
		agg_disp_arr[i] = my_mem_view_state_p->pre_disp_arr[i];
		agg_blk_arr[i]  = my_mem_view_state_p->pre_blk_arr[i];
		
		if ((my_mem_view_state_p->pre_blk_arr[i] + 
		     tmp_agg_comm_pre_sz) > *agg_comm_pre_sz_p)
		{
		    has_partial = 1;
		    agg_blk_arr[i] = *agg_comm_pre_sz_p - tmp_agg_comm_pre_sz;
		    tmp_agg_comm_pre_sz = *agg_comm_pre_sz_p;
		    partial_disp = my_mem_view_state_p->pre_disp_arr[i] +
			agg_blk_arr[i];
		    partial_len  = my_mem_view_state_p->pre_blk_arr[i] - 
			agg_blk_arr[i];
		    i++;
		    break;
		}
		else if ((my_mem_view_state_p->pre_blk_arr[i] +
			  tmp_agg_comm_pre_sz) == *agg_comm_pre_sz_p)
		{
		    tmp_agg_comm_pre_sz +=  
			my_mem_view_state_p->pre_blk_arr[i];
		    i++;
		    break;
		}
		else
		    tmp_agg_comm_pre_sz +=
			my_mem_view_state_p->pre_blk_arr[i];
	    }
	    *agg_mem_next_off_p = agg_disp_arr[i - 1] + agg_blk_arr[i - 1];
	    *agg_ol_cur_ct_p = i;
	    *agg_comm_cur_sz_p = *agg_comm_pre_sz_p;
	    
	    /* Clean up the ol pairs we used */	    
	    if ((i < my_mem_view_state_p->pre_ol_ct) || (has_partial == 1))
	    {
		int remain_ol_ct = 
		    my_mem_view_state_p->pre_ol_ct - i + has_partial;
		MPI_Aint *new_pre_disp_arr = NULL;
		int *new_pre_blk_arr = NULL;
		
		if ((new_pre_disp_arr = (MPI_Aint *)
		     ADIOI_Malloc(remain_ol_ct * sizeof(MPI_Aint))) == NULL)
		{
		    fprintf(stderr, "process_pre_req: malloc "
			    "new_pre_disp_arr failed\n");
		    return -1;
		}
		if ((new_pre_blk_arr = (int *)
		     ADIOI_Malloc(remain_ol_ct * sizeof(int))) == NULL)
                {
                    fprintf(stderr, "process_pre_req: malloc "
                            "new_pre_blk_arr failed\n");
                    return -1;
                }
		
		memcpy(new_pre_disp_arr, 
		       &(my_mem_view_state_p->pre_disp_arr[i - has_partial]),
		       remain_ol_ct * sizeof(MPI_Aint));
		memcpy(new_pre_blk_arr, 
		       &(my_mem_view_state_p->pre_blk_arr[i - has_partial]),
		       remain_ol_ct * sizeof(int));
		
		/* Set the partial len of the first piece */
		if (has_partial == 1)
		{
		    /* new_pre_disp_arr[remain_ol_ct - 1] = partial_disp;
		       new_pre_blk_arr[remain_ol_ct - 1]  = partial_len; */
		    new_pre_disp_arr[0] = partial_disp;
		    new_pre_blk_arr[0]  = partial_len;
		}
		
		ADIOI_Free(my_mem_view_state_p->pre_disp_arr);
		ADIOI_Free(my_mem_view_state_p->pre_blk_arr);
		
		my_mem_view_state_p->pre_disp_arr = new_pre_disp_arr;
		my_mem_view_state_p->pre_blk_arr  = new_pre_blk_arr;
		my_mem_view_state_p->pre_ol_ct = remain_ol_ct;
		my_mem_view_state_p->pre_sz -= *agg_comm_pre_sz_p;
	    }
	    else /* Used all the precalculated ol pairs */
	    {
		ADIOI_Free(my_mem_view_state_p->pre_disp_arr);
		ADIOI_Free(my_mem_view_state_p->pre_blk_arr);
		
		my_mem_view_state_p->pre_disp_arr = NULL;
		my_mem_view_state_p->pre_blk_arr = NULL;
		my_mem_view_state_p->pre_ol_ct = 0;
		my_mem_view_state_p->pre_sz = 0;
	    }
#ifdef DEBUG1
	    fprintf(stderr, "process_pre_req: REAL_OFF "
		    "agg_comm_pre_sz=%Ld,agg_comm_cur_sz=%Ld,agg_ol_ct=%d,"
		    "agg_ol_cur_ct=%d\n",
		    *agg_comm_pre_sz_p, *agg_comm_cur_sz_p, *agg_ol_ct_p, 
		    *agg_ol_cur_ct_p);
#endif
	    break;
	default:
	    fprintf(stderr, "process_pre_req: Invalid off_type %d\n",
		    off_type);
    }
    return 0;
}

/* ADIOI_Build_client_req() creates a memory datatype to transfer data
 * to/from a particular aggregator. */

int ADIOI_Build_client_req(ADIO_File fd,
			   int agg_rank,
			   int agg_idx,
			   view_state *my_mem_view_state_p,
			   view_state *agg_file_view_state_p,
			   ADIO_Offset agg_comm_sz,
			   MPI_Datatype *agg_comm_dtype_p)
{
    MPI_Aint *agg_disp_arr = NULL;
    int *agg_blk_arr = NULL;
    ADIO_Offset st_reg = 0, act_reg_sz = 0, tmp_reg_sz = 0;
    ADIO_Offset cur_off = -1, cur_reg_max_len = -1;
    ADIO_Offset agg_mem_st_reg = 0, agg_mem_act_reg_sz = 0;
    int agg_ol_ct = 0, agg_ol_cur_ct = 0;
    int i = 0, agg_next_off_idx = -1;
    ADIO_Offset agg_mem_next_off = 0, agg_comm_cur_sz = 0, agg_comm_pre_sz = 0;
    ADIO_Offset *fr_st_off_arr = fd->file_realm_st_offs;
    MPI_Datatype *fr_type_arr = fd->file_realm_types;
    flatten_state *tmp_mem_state_p = NULL, *tmp_file_state_p = NULL;
#ifdef DTYPE_SKIP
    int skip_type_ct;
#endif

    if (agg_idx < 0 || agg_idx >= fd->hints->cb_nodes)
    {
#ifdef DEBUG1
	fprintf(stderr, "ADIOI_Build_client_req: agg_rank %d does not map "
		"to a valid node in cb_node\n", agg_rank);
#endif
	return 0;
    }

#ifdef AGGREGATION_PROFILE
    MPE_Log_event (5018, 0, NULL);
#endif

#ifdef DEBUG1
    fprintf(stderr, "ADIOI_Build_client_req:(agg=%d,size_req=%Ld)\n",
	    agg_idx, agg_comm_sz);
#endif
    
    /* On the first pass see how many offset-length pairs are
     * necessary for each aggregator.  Then allocate the correct
     * amount of offset-length pairs for handling each aggregator's
     * particular data size.  On the last pass, we actually create the
     * offset-length pairs. */
    for (i = 0; i < MAX_OFF_TYPE; i++)
    {
	switch(i)
	{
	    case TEMP_OFF:
		tmp_mem_state_p  = &(my_mem_view_state_p->tmp_state);
		tmp_file_state_p = &(agg_file_view_state_p->tmp_state);
		break;
	    case REAL_OFF:
		tmp_mem_state_p  = &(my_mem_view_state_p->cur_state);
		tmp_file_state_p = &(agg_file_view_state_p->cur_state);
		break;
	    default:
		fprintf(stderr, "ADIOI_Build_client_pre_req: "
			"Invalid off type %d\n", i);
	}

	agg_comm_cur_sz = 0;
	agg_mem_next_off = -1;

	/* First try to preprocess anything we can */
	if (my_mem_view_state_p->pre_sz > 0)
	{
	    process_pre_req(fd,
			    agg_rank,
			    agg_idx,
			    my_mem_view_state_p,
			    agg_file_view_state_p,
			    agg_comm_sz,
			    i,
			    agg_disp_arr,
			    agg_blk_arr,
			    &agg_comm_pre_sz,
			    &agg_comm_cur_sz,
			    &agg_comm_sz,
			    &agg_ol_cur_ct,
			    &agg_ol_ct,
			    &agg_mem_next_off);
	}
	
	while (agg_comm_cur_sz < agg_comm_sz)
	{	
	    find_next_off(fd, agg_file_view_state_p,
			  fr_st_off_arr[agg_idx],
			  &(fr_type_arr[agg_idx]),
			  i,
			  &cur_off,
			  &cur_reg_max_len);
	    
	    assert(cur_off != -1);
	    
	    /* Add up to the end of the file realm or as many bytes
	     * are left for this particular aggregator in the client's
	     * filetype */
	    if (cur_reg_max_len > (agg_comm_sz - agg_comm_cur_sz))
	    {
		cur_reg_max_len = agg_comm_sz - agg_comm_cur_sz;
	    }
	    assert(cur_reg_max_len > 0);
	
	    view_state_add_region(
		cur_reg_max_len,
		agg_file_view_state_p,
		&st_reg, &act_reg_sz, i);
	    
#ifdef DEBUG2
	    fprintf(stderr, "ADIOI_Build_client_req: %s File region"
		    " (proc=%d,off=%Ld,sz=%Ld)\n",
		    off_type_name[i], agg_rank, cur_off, act_reg_sz);
#endif
	    
	    /* Before translating the file regions to memory regions,
	     * we first must advance to the proper point in the
	     * mem_view_state for this aggregator to match the
	     * file_view_state. */
	    
	    assert(tmp_file_state_p->cur_sz - act_reg_sz >= 
		   tmp_mem_state_p->cur_sz);
	    
	    while (tmp_file_state_p->cur_sz - act_reg_sz != 
		   tmp_mem_state_p->cur_sz)
	    {
		ADIO_Offset fill_st_reg = -1, fill_reg_sz = -1;
#ifdef DTYPE_SKIP
		if (my_mem_view_state_p->flat_type_p->count > 1) {
		    /* let's see if we can skip whole memory datatypes */
		    skip_type_ct =
			(tmp_file_state_p->cur_sz - act_reg_sz -
			 tmp_mem_state_p->cur_sz) /
			my_mem_view_state_p->type_sz;
		    if (skip_type_ct > 0) {
			tmp_mem_state_p->cur_sz +=
			    skip_type_ct * my_mem_view_state_p->type_sz;
			tmp_mem_state_p->abs_off +=
			    skip_type_ct * my_mem_view_state_p->ext;
			if ((tmp_mem_state_p->cur_sz - act_reg_sz) ==
			    tmp_file_state_p->cur_sz)
			    break;
		    }
		}
#endif
		view_state_add_region(
		    tmp_file_state_p->cur_sz - 
		    act_reg_sz - tmp_mem_state_p->cur_sz,
		    my_mem_view_state_p,
		    &fill_st_reg,
		    &fill_reg_sz, i);
	    }
	    
	    /* Based on how large the act_reg_sz is, first figure
	     * out how many memory offset-length pairs are
	     * necessary and then set the offset-length pairs. */
	    tmp_reg_sz = 0;
	    while (tmp_reg_sz != act_reg_sz)
	    {
		view_state_add_region(
		    act_reg_sz - tmp_reg_sz,
		    my_mem_view_state_p,
		    &agg_mem_st_reg, &agg_mem_act_reg_sz, 
		    i);
		tmp_reg_sz += agg_mem_act_reg_sz;
		
#ifdef DEBUG2
		fprintf(stderr, "ADIOI_Build_client_req: %s Mem region"
			"(off=%Ld,sz=%Ld)\n",
			off_type_name[i], agg_mem_st_reg, 
			agg_mem_act_reg_sz);
#endif
		agg_comm_cur_sz += agg_mem_act_reg_sz;
		switch(i)
		{
		    case TEMP_OFF:
			/* Increment the ol list count if the next
			 * region is not adjacent to the previous
			 * region. */
			if (agg_mem_next_off != agg_mem_st_reg)
			{
			    agg_ol_ct++;
			}
			agg_mem_next_off = 
			    agg_mem_st_reg + agg_mem_act_reg_sz;
			break;
		    case REAL_OFF:
			/* Set the ol list for the memtype that
			 * will map to our aggregator, coaslescing
			 * if possible. */
			agg_next_off_idx = agg_ol_cur_ct;
			if (agg_mem_next_off != agg_mem_st_reg)
			{
			    agg_disp_arr[agg_next_off_idx] = 
				agg_mem_st_reg;
			    agg_blk_arr[agg_next_off_idx] = 
				agg_mem_act_reg_sz;
			    agg_ol_cur_ct++;
			}
			else
			{
			    agg_blk_arr[agg_next_off_idx - 1]
				+= agg_mem_act_reg_sz;
			}
			agg_mem_next_off = 
			    agg_mem_st_reg + agg_mem_act_reg_sz;
			break;
		    default:
			fprintf(stderr, "ADIOI_Build_client_req: "
				"Impossible type\n");
		}
	    }
	}
	
	/* On the first pass, allocate the memory structures for
	 * creating the MPI_hindexed type. */
	if (i == TEMP_OFF)
	{	    
	    /* Allocate offset-length pairs for creating hindexed
	     * MPI_Datatypes for each aggregator */
	    if ((agg_disp_arr = (MPI_Aint *) 
		 ADIOI_Malloc(agg_ol_ct * sizeof(MPI_Aint))) == NULL)
	    {
		fprintf(stderr, "ADIOI_Build_client_req: malloc "
			"agg_disp_arr of size %ld failed\n",
			(long int)agg_ol_ct * sizeof(MPI_Aint));
		return -1;
	    }
	    if ((agg_blk_arr = (int *) 
		 ADIOI_Malloc(agg_ol_ct * sizeof(int))) == NULL)
	    {
		ADIOI_Free(agg_disp_arr);
		fprintf(stderr, "ADIOI_Build_client_req: malloc "
			"agg_blk_arr of size %ld failed\n",
			(long int)agg_ol_ct * sizeof(int));
		return -1;
	    }
	}
    }

    assert(agg_ol_ct == agg_ol_cur_ct);
#ifdef DEBUG1
    fprintf(stderr, 
	    "ADIOI_Build_client_req:(agg=%d,cur_ol_count=%d=ol_count=%d)\n",
	    agg_rank, agg_ol_cur_ct, agg_ol_ct);
#endif

#ifdef DEBUG2
    if (agg_ol_ct > 0)
    {
	fprintf(stderr, "ADIOI_Build_client_req: p %d (off,len) = ", agg_rank);
	for (i = 0; i < agg_ol_ct; i++)
	{
	    fprintf(stderr, "[%d](%d,%d) ", i, 
		    agg_disp_arr[i], agg_blk_arr[i]);
	    if (i % 5 == 0 && i != 0)
		fprintf(stderr, "\n");
	}
	fprintf(stderr, "\n");
    }
#endif
#ifdef DEBUG1
    fprintf(stderr, 
	    "ADIOI_Build_client_req:(agg=%d,pre_ol_count=%d)\n",
	    agg_idx, my_mem_view_state_p->pre_ol_ct);
#endif

#ifdef DEBUG2
    if (my_mem_view_state_p->pre_sz > 0)
    {
	fprintf(stderr, "ADIOI_Build_client_req: p %d pre(off,len) = ", 
		agg_idx);
	for (i = 0; i < my_mem_view_state_p->pre_ol_ct; i++)
	{
	    fprintf(stderr, "[%d](%d,%d) ", i, 
		    my_mem_view_state_p->pre_disp_arr[i], 
		    my_mem_view_state_p->pre_blk_arr[i]);
	    if (i % 5 == 0 && i != 0)
		fprintf(stderr, "\n");
	}
	fprintf(stderr, "\n");
    }
#endif

    /* Create the aggregator MPI_Datatype */
    if (agg_comm_sz > 0)
    {
	MPI_Type_hindexed(agg_ol_ct, agg_blk_arr, agg_disp_arr, MPI_BYTE,
			  agg_comm_dtype_p);
	MPI_Type_commit(agg_comm_dtype_p);
    }
    else
    {
	*agg_comm_dtype_p = MPI_BYTE;
    }

    ADIOI_Free(agg_blk_arr);
    ADIOI_Free(agg_disp_arr);

#ifdef AGGREGATION_PROFILE
    MPE_Log_event (5019, 0, NULL);
#endif    
    return 0;
}


