/*
 * © Copyright 1996-2017 ECMWF.
 * 
 * This software is licensed under the terms of the Apache Licence Version 2.0
 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 
 * In applying this licence, ECMWF does not waive the privileges and immunities 
 * granted to it by virtue of its status as an intergovernmental organisation nor
 * does it submit to any jurisdiction.
 */

/*

B.Raoult
ECMWF Oct-93

 */

#include "mars.h"
#include <sys/types.h>
#include <sys/stat.h>
#ifndef CRAY
#include <sys/mman.h>
#endif
#include <fcntl.h>
#include <errno.h>
#ifndef SYSV
#include <strings.h>
#endif
#if defined(CRAY)
extern int strcasecmp(const char*,const char*);
#endif
#include <math.h>

#define PAD(l,n)     ( ( ( (l)+(n)-1 ) / (n) * (n) ) - (l) )

#define INIT_SIZE	50 /* Allocate 50 fields at a time */

#define GRIB		0x47524942

#define STACK_SIZE 50

#ifndef SEEK_SET
#define SEEK_SET 0 /* stupid CodeCenter */
#endif


/* Not all versions of ecCodes and GRIB_API support the missingValuesPresent key */
#ifdef ECCODES_VERSION
	#if (ECCODES_VERSION) >= 20401
		#define MISSING_VALUES_PRESENT_KEY_EXISTS 1
    #endif
#else
	#if (GRIB_API_VERSION) >= 12301
		#define MISSING_VALUES_PRESENT_KEY_EXISTS 1
    #endif
#endif



static mempool varmem = {
	10, /* 10 pages = 40k */
	1,  /* clear */
};

#define FASTNEW(type)  	     (type*)fast_new(sizeof(type),&varmem)
#define FASTDEL(x)           fast_delete(x,&varmem)


gribfile* new_gribfile(const char *name)
{
	gribfile* g = FASTNEW(gribfile);
	g->fname = strcache(name?name:marstmp());
	g->temp  = name == NULL;
	g->file  = NULL;
	marslog(LOG_DBUG,"create %s",g->fname);
	return g;
}

FILE *open_gribfile(gribfile *g, const char *mode)
{
	if(g->file)
		return g->file;

	g->file = fopen(g->fname,mode);
	if(g->file == NULL)
		marslog(LOG_WARN|LOG_PERR,"fopen(%s)",g->fname);

	return g->file;
}

void close_gribfile(gribfile *g)
{
	if(g->file)
	{
		if(fclose(g->file))
			marslog(LOG_WARN|LOG_PERR,"fclose(%s)",g->fname);
		g->file = NULL;
	}
}

void free_gribfile(gribfile *g)
{
	if(!g) return;
	g->refcnt--;
	if(g->refcnt <= 0)
	{
		close_gribfile(g);
		if(g->temp)
		{
			marslog(LOG_DBUG,"unlink %s",g->fname);
			unlink(g->fname);
		}

		strfree(g->fname);
		FASTDEL(g);
	}
}

field_request* new_field_request(request *r)
{
	field_request* g = FASTNEW(field_request);
	g->r = clone_one_request(r);
	return g;
}

void free_field_request(field_request* g)
{
	if(!g) return;
	g->refcnt--;
	if(g->refcnt <= 0)
	{
		free_all_requests(g->r);
		FASTDEL(g);
	}
}

static void *alloc_empty(long size)
{
	void *p = reserve_mem(size);
	memset(p,0,size);
	return p;
}



void free_field(field *g)
{
	if(!g) return;
	g->refcnt--;
	if(g->refcnt <= 0)
	{
		free_gribfile(g->file);
		free_field_request(g->r);
		if(g->values)  release_mem(g->values);
		grib_handle_delete(g->handle);
		FASTDEL(g);
	}
}

void free_fieldset(fieldset *v)
{
	int i;
	if(!v) return;
	v->refcnt--;
	if(v->refcnt <= 0)
	{
		marslog(LOG_DBUG,"free_fieldset (%d fields) : ",v->count);

		for(i=0;i<v->count;i++)
			free_field(v->fields[i]);

		release_mem(v->fields);
		FASTDEL(v);
	}
}

field *new_field()
{
	return FASTNEW(field);
}


static void grow_fieldset(fieldset *v,int n)
{
	int i;
	int m = v->count;
	int x = v->max;

	if(n < v->count) return;

	v->count = n;

	while(v->count >= v->max)
		if(v->max< INIT_SIZE)
			v->max = INIT_SIZE;
		else
			v->max += v->max/2 +1;

	if(v->max != x)
	{
		if( v->fields == NULL)
			v->fields = (field**)reserve_mem(sizeof(field*)*v->max);
		else
		{
			field** f = (field**)reserve_mem(sizeof(field*)*v->max);
			for(i=0;i<m;i++)
				f[i] = v->fields[i];
			release_mem(v->fields);
			v->fields = f;
		}

		for(i=m;i<v->max;i++)
			v->fields[i] = NULL;
	}
}

fieldset *new_fieldset(int n)
{
	fieldset *f = FASTNEW(fieldset);
	grow_fieldset(f,n);
	return f;
}

field* read_field(gribfile *file,off_t pos,long length)
{
	field *g = new_field();

	g->file   = file;
	file->refcnt++;
	g->offset = pos;

	if(length == 0)
	{
		FILE *f = open_gribfile(file,"r");
		if(f)
		{
			fseek(f,pos,SEEK_SET);
			_readgrib(f,NULL,&length);
			close_gribfile(file);
		}
	}
	g->length = length;
	g->shape  = packed_file;

	return g;
}

err add_field(fieldset *v,field *g)
{
	int m = v->count;
	grow_fieldset(v, v->count + 1);
	v->fields[m] = g;
	g->refcnt++;
	return NOERR;
}

err set_field(fieldset *v,field *g,int pos)
{
	field *h;
	if(pos >= 0)
	{
		grow_fieldset(v,pos+1);
		h = v->fields[pos];
		v->fields[pos] = g;
		g->refcnt++;
		if(h) free_field(h);
	}
	return NOERR;
}

err check_fieldset(fieldset *v, int expect)
{
	int i;
	int hole = 0;
	for(i=0;i<v->count;i++)
		if(v->fields[i] == 0)
			hole++;

	if((v->count - hole) != expect)
	{
		marslog(LOG_EROR,"Inconsistency in field ordering, expected %d, recognised %d",expect,v->count - hole);
		return -2;
	}

	return NOERR;
}

field *copy_field(field *gx,boolean copydata)
{
	field *gv   = new_field();

	gv->value_count  = gx->value_count;
	gv->shape        = gx->shape;
	gv->missing      = gx->missing;
	gv->bitmap       = gx->bitmap;
	gv->missing_vals = gx->missing_vals;

	if(gx->r) {
		gv->r = gx->r;
		gv->r->refcnt++;
	}

	if(gx->handle)
		gv->handle = grib_handle_clone(gx->handle);

	if(gx->shape == expand_mem)
	{
		gv->values   = (double*)reserve_mem(sizeof(double)*gx->value_count);
		if(copydata)
			memcpy((void*)gv->values,(void*)gx->values,
					gv->value_count*sizeof(double));
	}


	return gv;
}

void copy_missing_vals(field *gc, field *ga, field *gb)
{
	boolean doit = ( ga && ga->missing_vals) || (gb && gb->missing_vals);

	if(doit)
	{
		field *f = (ga)?ga:gb;
		field *g = (gb)?gb:ga;
		int j = 0;
		if(f == g)
		{
			for(j=0;j<f->value_count;j++)
				if(MISSING_VALUE(f->values[j]))
					gc->values[j] = mars.grib_missing_value;
		}
		else
		{
			for(j=0;j<f->value_count;j++)
				if(MISSING_VALUE(f->values[j]) ||  MISSING_VALUE(g->values[j]))
					gc->values[j] = mars.grib_missing_value;
		}

		set_bitmap(gc);
	}
}

void remove_bitmap(field *gc)
{
	int err;
#ifdef COMEBACK
	gc->bitmap = false;
	gc->ksec1[4] &= ~0x40; /* ~0x40 == 0xAF */
#endif
	gc->bitmap = false;
	gc->missing_vals = false;
	err = grib_set_long(gc->handle,"bitmapPresent",0);
	if(err) {
		marslog(LOG_EXIT,"grib_set_long(bitmapPresent) failed: %s", grib_get_error_message(err));
	}
}


void set_bitmap(field *gc)
{
	/* only flag a bitmap to be created if field currently has no concept of missing values
	   (if bitmap=true, then no need to add one, and in this case missing_vals will be true
	   too; if bitmap=false and missing_vals=true then we have one of these packing types
	   that allows missing values to be encoded directly in the data, and we don't want
	   to add a bitmap to that) */

	if (!gc->missing_vals)
	{
		gc->bitmap = true;
		gc->missing_vals = true;
	}
}


fieldset *copy_fieldset(fieldset *x,int count,boolean copydata)
{
	fieldset *v = new_fieldset(count);

	if( count != 0 )
	{
		int i;
		for(i=0;i<count;i++)
		{
			field *gx = get_field(x,i,expand_mem);
			v->fields[i] = copy_field(gx,copydata);
			v->fields[i]->refcnt++;
			release_field(gx);
		}
	}
	return v;

}

fieldset *merge_fieldsets(fieldset *x,fieldset *y)
{
	int i;
	int xcnt    = x ? x->count : 0;
	int ycnt    = y ? y->count : 0;
	fieldset *v = new_fieldset(xcnt + ycnt);

	if( (v->count = xcnt + ycnt) != 0 )
	{
		for(i=0;i<xcnt;i++)
		{
			field *g = x->fields[i];
			v->fields[i] = g;
			g->refcnt++;
		}

		for(i=0;i<ycnt;i++)
		{
			field *g = y->fields[i];
			v->fields[i + xcnt] = g;
			g->refcnt++;
		}
	}

	return v;

}

static void check(int *v,int a,int b)
{
	if(*v < a)
	{
		marslog(LOG_WARN,"Bad index : %d < %d",*v,a);
		marslog(LOG_WARN,"The value is changed to %d",a);
		*v = a;
	}

	if(*v > b)
	{
		marslog(LOG_WARN,"Bad index : %d > %d",*v,b);
		marslog(LOG_WARN,"The value is changed to %d",b);
		*v = b;
	}

}

fieldset *sub_fieldset(fieldset *v,int from,int to,int step)
{
	fieldset *w;
	int count,i,n;

	if(from == 0 && to == 0 && step == 0)
		return v;

	if(to   == 0) to   = from;
	if(step == 0) step = 1;

	check(&from,1,v->count);
	check(&to  ,1,v->count);

	count = (to - from) / step + 1;

	if ( count <= 0)
	{
		marslog(LOG_EROR,"Cannot range from %d to %d by %d",from,to,step);
		return NULL;
	}

	w = new_fieldset(count);

	for(i=0,n=from-1;i<count;i++,n+=step)
	{
		field *g = v->fields[n];
		w->fields[i] = g;
		g->refcnt++;
	}

	return w;

}

int best_packing(fortint current)
{
	if(mars.accuracy > 0)
	{
		marslog(LOG_DBUG,"best_packing: Using %d bits for packing",mars.accuracy);
		return mars.accuracy;
	}

	if(mars.accuracy == -1)
		return current;

	return 24;
}

static err to_packed_mem(field *g)
{

	err e = NOERR;
	int i;

	if(g->shape == packed_mem) return NOERR;

	if(g->shape == expand_mem)
	{
		/* Accuracy */
		long accuracy = -1;
		if((e = grib_get_long(g->handle,"bitsPerValue",&accuracy)))
		{
			marslog(LOG_EROR,"%s: cannot get accuracy %s",grib_get_package_name(),grib_get_error_message(e));
			return e;
		}

		accuracy = best_packing(accuracy);
		if( (e = grib_set_long(g->handle,"bitsPerValue",accuracy)) )
		{
			marslog(LOG_EROR,"%s: cannot set accuracy to %ld (%s)",grib_get_package_name(),accuracy,grib_get_error_message(e));
			return e;

		}

		/* Bitmap. Note that missingValuesPresent is read-only, so we do not need to set it */
		if((e = grib_set_double(g->handle,"missingValue",mars.grib_missing_value)))
		{
			marslog(LOG_EROR,"%s: cannot set missingValue %s",grib_get_package_name(),grib_get_error_message(e));
			return e;
		}
		if(g->bitmap)
		{
			if((e = grib_set_long(g->handle,"bitmapPresent",1)))
			{
				marslog(LOG_EROR,"%s: cannot set bitmapPresent %s",grib_get_package_name(),grib_get_error_message(e));
				return e;
			}
		}


		/* Compute flag */
		if(mars.computeflg)
		{
			if( (e = grib_set_long(g->handle,"generatingProcessIdentifier",mars.computeflg)) )
			{
				marslog(LOG_EROR,"%s: cannot set generating process to %ld (%s)",grib_get_package_name(),mars.computeflg,grib_get_error_message(e));
				return e;
			}
		}

		/* Avoid computing Laplacian Operator, so results are bit comparable with MARS/gribex */
		{
			char grid_type[80];
			size_t size;
			size = sizeof(grid_type);
			if( (e = grib_get_string(g->handle,"typeOfGrid",grid_type,&size)) )
			{
				marslog(LOG_EROR,"%s: cannot get typeOfGrid %s",grib_get_package_name(),grib_get_error_message(e));
				return e;
			}

			if(strcmp(grid_type,"sh") == 0)
				grib_set_long(g->handle,"computeLaplacianOperator",0);

		}


		if((e = grib_set_double_array(g->handle,"values",g->values,g->value_count)))
		{
			marslog(LOG_EROR,"%s: cannot encode values %s",grib_get_package_name(),grib_get_error_message(e));
			return e;
		}
		release_mem(g->values);
		g->values      = NULL;
		g->value_count = 0;
		g->shape  = packed_mem;
		return NOERR;
	}

	if(g->shape == packed_file)
	{
		const void* dummy = NULL;

		FILE *f = open_gribfile(g->file,"r");
		if(!f)
		{
			marslog(LOG_EROR|LOG_PERR,"%s",g->file->fname);
			return -1;
		}
		marslog(LOG_DBUG,"loading file %s",g->file->fname);

		if( g->offset )
		   fseek(f,g->offset,SEEK_SET);

		g->handle = grib_handle_new_from_file(0,f,&e);

		if( g->handle )
			grib_get_message(g->handle,&dummy,&g->length);

		close_gribfile(g->file);
		g->shape = packed_mem;
		if(e) return e;
	}
	return NOERR;
}




static err to_expand_mem(field *g)
{
	err e;
	int i;

	if(g->shape == expand_mem)  return NOERR;

	if(g->shape == packed_file)
	{
		const void* dummy = NULL;

		FILE *f = open_gribfile(g->file,"r");
		if(!f)
		{
			marslog(LOG_EROR|LOG_PERR,"%s",g->file->fname);
			return -1;
		}
		marslog(LOG_DBUG,"loading file %s",g->file->fname);

		if( g->offset )
		   fseek(f,g->offset,SEEK_SET);

		g->handle = grib_handle_new_from_file(0,f,&e);

		if( g->handle )
			grib_get_message(g->handle,&dummy,&g->length);

		close_gribfile(g->file);
		if(!g->handle)
			return -1;

		if(g->values) release_mem(g->values);
		g->values = NULL;
	}

	if(g->values == NULL)
	{
		size_t count = 0;
		long bitmap = 0;
		long missing_vals = 0;

		if((e = grib_get_size(g->handle,"values",&g->value_count)))
		{
			marslog(LOG_EROR,"%s: cannot get number of values %s",grib_get_package_name(),grib_get_error_message(e));
			return e;
		}

		count = g->value_count;

		if((e = grib_set_double(g->handle,"missingValue",mars.grib_missing_value)))
		{
			marslog(LOG_EROR,"%s: cannot set missingValue %s",grib_get_package_name(),grib_get_error_message(e));
			return e;
		}

		g->values = (double*)reserve_mem(sizeof(double)*g->value_count);
		if((e = grib_get_double_array(g->handle,"values",g->values,&count)))
		{
			marslog(LOG_EROR,"%s: cannot get decode values %s",grib_get_package_name(),grib_get_error_message(e));
			return e;
		}

		if(count != g->value_count)
			marslog(LOG_EXIT,"%s: value count mismatch %d %d",grib_get_package_name(),count,g->value_count);

		if((e = grib_get_long(g->handle,"bitmapPresent",&bitmap)))
		{
			marslog(LOG_EROR,"%s: cannot get bitmapPresent %s",grib_get_package_name(),grib_get_error_message(e));
			return e;
		}

#ifdef MISSING_VALUES_PRESENT_KEY_EXISTS
		if((e = grib_get_long(g->handle,"missingValuesPresent",&missing_vals)))
		{
			marslog(LOG_EROR,"%s: cannot get missingValuesPresent %s",grib_get_package_name(),grib_get_error_message(e));
			return e;
		}
#else
		/* if this key is not supported by the GRIB-decoding package, we revert to the 'old'
		   behaviour, which is to equate a bitmap with the existence of missing values */
		missing_vals = bitmap;
#endif

		g->bitmap       = (bitmap != 0);
		g->missing_vals = (missing_vals != 0);

#ifdef COMEBACK
		set g->missing
#endif
	}


	g->shape =  expand_mem;

	return e;
}

void set_field_state(field *g,field_state shape)
{
	switch(shape)
	{
		case expand_mem:
			to_expand_mem(g);
			break;

		case packed_mem:
			to_packed_mem(g);
			break;

		case packed_file:
			release_field(g);
			break;

		default:
			marslog(LOG_EXIT,"Internal error %s %d",__FILE__,__LINE__);
			break;

	}
}

field *get_field(fieldset *v,int n,field_state shape)
{
	field *g = v->fields[n];
	set_field_state(g,shape);
	return g;
}

field *get_nonmissing_field(fieldset *v,field_state shape)
{
	int n = 0;
	field *g = v->fields[n];

	/* Try to get a non missing field */
	while(g->missing && ++n<v->count)
	{
		g = v->fields[n];
	}

	if(n==v->count) /* If all missing, take the first */
		g = v->fields[0];

	set_field_state(g,shape);
	return g;
}

void inform_missing_fieldset(const char *name)
{
	if(name)
		marslog(LOG_WARN,"All fields in '%s' are missing",name);
	else
		marslog(LOG_WARN,"All fields in fieldset are missing");
}

void release_field(field *g)
{
	if(g->file) {
		if(g->values)  release_mem(g->values);
		g->values  = NULL;
		g->shape  = packed_file;
		grib_handle_delete(g->handle);
		g->handle = NULL;
	}
}

const void *field_message(field *g,long* s)
{
	const void *p;
	size_t size;
	err e;

	if( (e=grib_get_message(g->handle,&p,&size)) )
	{
		marslog(LOG_EROR,"%s: cannot set message %s",grib_get_package_name(),grib_get_error_message(e));
		return NULL;
	}

	*s = size;
	return p;
}

err write_field(FILE *f,field *g)
{
	int pad;
	err e = 0;
	static int first = true;
	static char c[10240];      /* 10Kb of padding should be enough */
	long size;
	const void *p = field_message(g,&size);

	if(!p) return -1;


	if(fwrite(p,1,size,f) != size)
	{
		marslog(LOG_EROR|LOG_PERR,"Error while writing to disk");
		e = -2;
	}

	pad = PAD(size,120);

	if(first)
	{
		memset(c,0,sizeof(c));
		first=false;
	}

	if(fwrite(c,1,pad,f) != pad)
	{
		marslog(LOG_EROR|LOG_PERR,"Error while writing to disk");
		return -2;
	}

	return e;
}

err save_fieldset(fieldset *v)
{
	int i;
	err e = NOERR;
	gribfile *file = NULL;
	gribfile *last = NULL;
	FILE *f = 0;

	for(i=0;i<v->count;i++)
	{
		field *g = v->fields[i];

		/* find last file used */
		if(g != NULL && g->file != NULL && g->file->temp)
			last = g->file;

		if(g != NULL && g->file == NULL)
		{
			if(file == NULL)
			{
				file = last?last:new_gribfile(NULL);

				f = fopen(file->fname,"a");
				if(!f)
				{
					marslog(LOG_EROR|LOG_PERR,"Cannot open %s",file->fname);
					return -2;
				}
			}
			set_field_state(g,packed_mem);
			g->file = file;
			file->refcnt++;
			g->offset = ftell(f);
			e = write_field(f,g);
			release_field(g);
		}
	}

	if(f)
	{
		if(fclose(f))
		{
			marslog(LOG_EROR|LOG_PERR,"Error while writing to disk");
			e = -2;
		}
	}

	return e;
}

fieldset *read_fieldset(const char *fname,request *filter)
{
	fieldset *v;
	FILE *f;
	long i;
	off_t offset;
	err e = 0;
	long  buflen = 0;
	long length = 0;
	char *buffer = NULL;
	static char *read_disk_buffer = NULL;
	request *r = NULL;
	gribfile *file;
	hypercube *h = NULL;
	timer *t;
	char buf[1024];

#ifndef METVIEW
	sprintf(buf,"Reading file %s",fname);
	t=get_timer(buf,NULL,true);
#endif

	if(filter)
		h = new_hypercube_from_mars_request(filter);


	f = fopen(fname,"r");
	if(f == NULL)
	{
		marslog(LOG_EROR|LOG_PERR,"Cannot open %s",fname);
		return NULL;
	}

	v = new_fieldset(0);

	if(filter) r = empty_request(NULL);


	/* count fields */

	i = offset = length = 0;
	buflen = mars.readany_buffer_size;
	length = buflen;
	buffer = reserve_mem(length);
	file = new_gribfile(fname);
#ifndef METVIEW
	timer_start(t);
#endif

	if(mars.readdisk_buffer > 0)
	{
        if(read_disk_buffer == NULL)
        {
            read_disk_buffer = reserve_mem(mars.readdisk_buffer);
			marslog(LOG_DBUG,"Setting I/O read buffer to %d bytes",mars.readdisk_buffer);
			if(setvbuf(f,read_disk_buffer,_IOFBF,mars.readdisk_buffer))
				marslog(LOG_WARN|LOG_PERR,"setvbuf failed");
		}
	}

	while((e = _readany(f,buffer,&length)) == NOERR || (e == BUF_TO_SMALL))
	{
		boolean ok = true;

		if(filter)
		{
			grib_to_request(r,buffer,length);
			ok = cube_order(h,r) != -1;
		}

		if(ok) set_field(v,read_field(file,offset,length),i++);

		if(mars.progress)
		{
			mars.done += length;
			mars.progress();
		}

		offset = ftell(f);
		length = buflen;
	}

	fclose(f);
#ifndef METVIEW
	timer_stop(t,0);
#endif

	if(buffer) release_mem(buffer);
	free_all_requests(r);
	if(h)
		free_hypercube(h);


	if(e != EOF)
	{
		marslog(LOG_EROR,"Error %d while reading %s",e,fname);
		return NULL;
	}

	return v;
}

static fieldset *_request_to_fieldset(request *r)
{
	const char *path = get_value(r,"PATH",0);
	fieldset *v = NULL;
	int n = count_values(r,"OFFSET");
	int m = count_values(r,"LENGTH");
	int p = count_values(r,"INDEX");
	/* 	char *temp = get_value(r,"TEMPORARY",0); */
	/*  int tmp = temp?atoi(temp):0; */

	int i;

	if(n == 0 && m == 0)
	{
		v = read_fieldset(path,NULL);
		if(!v) return NULL;
		/* 		if(v->count) */
		/* 			v->fields[0]->file->temp = tmp; */
	}
	else if(n != m)          /*-- [vk] fails here when LENGTH has not been set! --*/
		marslog(LOG_EROR,
				"Bad data request: offsets != lengths (%d and %d)",n,m);
	else
	{
		gribfile *file = new_gribfile(path);

		/* file->temp = tmp; */

		v = new_fieldset(n);

		for(i=0;i<n;i++)
		{
			long length = atol(get_value(r,"LENGTH",i));
			off_t offset;
#ifdef LARGE_FILES_SUPPORT
			offset = atoll(get_value(r,"OFFSET",i));
#else
			offset = atol(get_value(r,"OFFSET",i));
#endif 
			set_field(v,read_field(file,offset,length),i);
			if(mars.progress) {
				mars.done += length;
				mars.progress();
			}
		}
	}
	return v;
}


fieldset *request_to_fieldset(request *r)
{
	fieldset *v = NULL;

	if(mars.progress) {
		/* Count size */
		request *u = r;
		mars.todo = 0;
		mars.done = 0;
		while(r && EQ(r->name,"GRIB"))
		{
			int m = count_values(r,"LENGTH");
			if(m)  {
				int i;
				for(i=0;i<m;i++)
					mars.todo += atol(get_value(r,"LENGTH",i));
			}
			else {
				const char* p = get_value(r,"PATH",0);
				struct stat s;
				if(p && stat(p,&s) == 0)
					mars.todo += s.st_size;
			}
			r = r->next;
		}
		mars.progress();
		r = u;
	}

	if(r && (r->next == NULL || !EQ(r->next->name,"GRIB")) )
		return _request_to_fieldset(r);
	while(r)
	{
		fieldset *w = _request_to_fieldset(r);
		fieldset *x = merge_fieldsets(v,w);
		free_fieldset(w);
		free_fieldset(v);
		v = x;
		r = r->next;
	}
	return v;
}

static request *_make_one_request(fieldset *v)
{
	char *tmp = marstmp();
	FILE *f = fopen(tmp,"w");
	err e = 0;
	int i;
	request *r;

	if(!f)
	{
		marslog(LOG_EROR|LOG_PERR,"%s",tmp);
		return NULL;
	}

	for(i=0;i<v->count;i++)
	{
		field *g = v->fields[i];
		set_field_state(g,packed_mem);
		e = e?e:write_field(f,g);
		release_field(g);
	}

	if(fclose(f))
	{
		marslog(LOG_EROR|LOG_PERR,"Error while writing to disk");
		return NULL;
	}
	if(e) return NULL;

	r = empty_request("GRIB");
	set_value(r,"PATH","%s",tmp);
	set_value(r,"TEMPORARY","1");
	return r;
}

request *fieldset_to_request(fieldset *v)
{
	request *r = NULL;
	request *first = NULL;
	request *last = NULL;
	char *path = NULL;
	int i;
	int n = 0;

	if(save_fieldset(v))
		return NULL;

	/* no multi  grib yet... */
#if 1
	for(i=0;i<v->count;i++)
	{
		field *g = v->fields[i];
		if(g->file->fname != path)
		{
			path = g->file->fname;
			n++;
		}
	}
	path = NULL;

	if(n > 1)
		return _make_one_request(v);

#endif

	for(i=0;i<v->count;i++)
	{
		field *g = v->fields[i];
		if(g->file->fname != path)
		{
			path = g->file->fname;
			r = empty_request("GRIB");
			set_value(r,"PATH","%s",path);
			set_value(r,"TEMPORARY","%d",g->file->temp);

			/* If a requests is made, some other modules
			   needs the data, so make it non-temporary*/

			g->file->temp = false;

			if(first == NULL) first = r;
			if(last) last->next = r;
			last = r;
		}
#ifdef LARGE_FILES_SUPPORT
		add_value(r,"OFFSET","%lld",g->offset);
#else
		add_value(r,"OFFSET","%d",g->offset);
#endif

		add_value(r,"LENGTH","%d",g->length);
	}


	return first;

}

request *fieldset_to_mars_request(fieldset *fs)
{
	int i;
	request* r = empty_request("GRIB");

	if(!fs)
		return 0;

	for(i = 0; i < fs->count; i++)
	{
		request *s = field_to_request(fs->fields[i]);
		reqmerge(r,s);
	}

	return r;
}

request* field_to_request(field* f)
{
	if(f->r == 0)
	{
		field_state state = f->shape;
		request *r = empty_request(
#ifdef COMEBACK
				((f->ksec1 == NULL) || (f->ksec1[2] != mars.computeflg)) ? "GRIB" : "COMPUTED");
#else
		"GRIB");
#endif

		set_field_state(f,packed_mem);
		handle_to_request(r,f->handle,NULL);
		set_field_state(f,state);

		f->r = new_field_request(r);
		free_all_requests(r);
	}
	return f->r->r;
}
