/*
 * © Copyright 1996-2012 ECMWF.
 *
 * This software is licensed under the terms of the Apache Licence Version 2.0
 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
 * In applying this licence, ECMWF does not waive the privileges and immunities
 * granted to it by virtue of its status as an intergovernmental organisation nor
 * does it submit to any jurisdiction.
 */

#include "mars.h"
#include "mars_client_config.h"

#ifdef ODB_SUPPORT
#include "odb.h"
#endif
#include <setjmp.h>
#include <signal.h>
#include <errno.h>



#ifdef _AIX
#include <netdb.h>
#endif

static jmp_buf env;

static void catch_alarm(int sig)
{
	longjmp(env,1);
}

static void dhs_init(void);

static err  dhs_open(void *data,request*,request*,int);
static err  dhs_close(void *data);
static err  dhs_read(void *data,request *r,void *buffer,long *length);
static err  dhs_write(void *data,request *r,void *buffer,long *length);
static err  dhs_cntl(void *data,int code,void *param,int size);
static err  dhs_archive(void *data,request *r);
static boolean dhs_check(void *data,request *r);
static err  dhs_validate(void *data,request*,request*,int);

typedef struct dhsdata {
	int               port;
	char              *host;
	char              *address;
	int               local_port;

	int               callback_port; /* For Kubernetis */
	char              *callback_host;

	int               csoc; /* Current sockect */
	int               asoc; /* Accept socket */
	ulong64           id;
	mstream           s;
	request           *req;
	boolean  		  finished;
	boolean  		  obs;
	boolean  		  finish_on_error;
	boolean			  is_bufrfile;
	boolean			  odb;
	boolean           netcdf;
	FILE			  *f;
	int               count;
	int               retry;
	int               timeout;
	boolean           retry_on_timeout;
	wind*             u_v;
	long64            total_read;
	long64            total_toread;
	long64            total_tosend;
	long64            data_size;
	long			  save_expansion_flags;
	boolean           know_obs;
	boolean           makeuv;

	boolean		      fetch;
	request          *metadata;
	request          *current;
	int              index;
	long64           *reclen;

	boolean          quiet;
	boolean          open;

	char             *check;
	boolean          retry_on_client;
	boolean          retry_forever_on_client;

	boolean          server_send_error;
	boolean          server_send_data;

	char             *read_disk_buffer;

	boolean          print_callback;
	boolean          skip_odb_check;

	mars_field_index* head_idx;
	mars_field_index* tail_idx;

	boolean          mars2mars;
	request*         othermars;
	long             expect;

	char*            random_hosts;

} dhsdata;

static option opts[] = {

	{"port","MARS_DHS_PORT","-dhsport","9000",t_int,sizeof(int),
		OFFSET(dhsdata,port)},

	{"host","MARS_DHS_HOST","-dhshost","porthos",t_str,sizeof(char*),
		OFFSET(dhsdata,host)},

	{"retry",NULL,NULL,"0",t_int,sizeof(int),
		OFFSET(dhsdata,retry)},

	{"timeout","MARS_DHS_TIMEOUT",NULL,"20",t_int,sizeof(int),
		OFFSET(dhsdata,timeout)},

	{"retry_on_timeout","MARS_DHS_RETRY_ON_TIMEOUT",NULL,"0",t_boolean,sizeof(boolean),
		OFFSET(dhsdata,retry_on_timeout)},

	{NULL,"MARS_DHS_LOCALHOST",NULL,NULL,t_str,sizeof(char*),
		OFFSET(dhsdata,address)},

	{"random_hosts","MARS_DHS_RANDOM_HOSTS",NULL,NULL,t_str,sizeof(char*),
		OFFSET(dhsdata,random_hosts)},

	{"local_port","MARS_DHS_LOCALPORT",NULL,"0",t_int,sizeof(int),
		OFFSET(dhsdata,local_port)},

	{"callback_host","MARS_DHS_CALLBACK_HOST",NULL,NULL,t_str,sizeof(char*),
		OFFSET(dhsdata,callback_host)},

	{"callback_port","MARS_DHS_CALLBACK_PORT",NULL,"0",t_int,sizeof(int),
		OFFSET(dhsdata,callback_port)},

	{"obs","MARS_KNOW_OBS",NULL,"0",t_boolean,sizeof(boolean),
		OFFSET(dhsdata,know_obs)},

	{"makeuv",NULL,NULL,"1",t_boolean,sizeof(boolean),
		OFFSET(dhsdata,makeuv)},

	{"finish_on_error","MARS_FINISH_ON_ERROR",NULL,"0",t_boolean,sizeof(boolean),
		OFFSET(dhsdata,finish_on_error)},

	{"check","MARS_DHSBASE_CHECK",NULL,"chk/dhs.chk",t_str,sizeof(char*),
		OFFSET(dhsdata,check)},

	{"print_callback","MARS_DHS_PRINT_CALLBACK",NULL,"0",t_boolean,sizeof(boolean),
		OFFSET(dhsdata,print_callback)},

	{"skip_odb_check","MARS_DHS_SKIP_ODB_CHECK",NULL,"0",t_boolean,sizeof(boolean),
		OFFSET(dhsdata,skip_odb_check)},

};

base_class _dhsbase = {

	NULL,                      /* parent class */
	"dhsbase",                 /* name         */

	false,                     /* inited       */

	sizeof(dhsdata),           /* private size */
	NUMBER(opts),              /* option count */
	opts,                      /* options      */

	dhs_init,                  /* init         */

	dhs_open,                  /* open         */
	dhs_close,                 /* close        */

	dhs_read,                  /* read         */
	dhs_write,                 /* write        */

	dhs_cntl,                  /* control      */

	dhs_check,                 /* check        */
	NULL,                      /* query        */

	dhs_archive,               /* archive      */
	NULL,                      /* admin        */

	dhs_validate,              /* validate */
};


base_class *dhsbase = &_dhsbase;

static err  dhs_wait(dhsdata*);

static void dhs_init(void)
{
}


/* Checks that all single values contained in request 'a'
   are in request 'b' */
static err fix_check(const request *grib,const request *req, int cnt)
{
	/* Should have everything which is not used for field ordering */

	static struct {
		const char *name;     /* Name of the MARS parameter to check */
		boolean     always;   /* If consistency grib<->request must always exist */
	} checks [] = {

		{ "CLASS",    true, },
		{ "TYPE",     true, },
		{ "STREAM",   true, },
		{ "LEVTYPE",  true, },
		{ "_GRID_EW", false, },
		{ "_GRID_NS", false, },
		{ "ORIGIN",   true, },
		{ "MODEL",   true, },

		{ "PRODUCT",   true, },
		{ "SECTION",   true, },
		{ "METHOD",    true, },
		{ "SYSTEM",    true, },

		/* Already in field_order */
		/* "EXPVER", */
		/* "DATE", */
		/* "TIME", */
		/* "REFERENCE", */
		/* "STEP", */
		/* "FCMONTH", */
		/* "FCPERIOD", */
		/* "DOMAIN", */
		/* "DIAGNOSTIC", */
		/* "ITERATION", */
		/* "NUMBER", */
		/* "LEVELIST", */
		/* "LATITUDE", */
		/* "LONGITUDE", */
		/* "RANGE", */
		/* "PARAM", */
		/* "FREQUENCY", */
		/* "DIRECTION", */

	};

	int j = 0;
	if(mars.pseudogrib)
		return NOERR;

	if(grib && req)
	{
		while(j < NUMBER(checks))
		{
			int i = 0;
			const char *p = checks[j].name;
			const char *s = get_value(grib,p,i);
			const char *r = get_value(req,p,i);

#if 1
			/* This extra check needs
			   exhaustive testing */
			/* If value is provided in the request
			   but not in the GRIB header, fail */
			if(r && !s)
			{
				marslog(LOG_EROR,"Cannot determine '%s' from GRIB header for grib %d",p,cnt);
				marslog(LOG_EROR,"Grib description follows:");
				print_all_requests(grib);
				return -6;
			}
#endif

			while(s)
			{
				if( !value_exist(req,p,s) )
				{
#if 0
					const char *str  = get_value(grib,"STREAM",0);
					const char *clss = get_value(grib,"CLASS",0);
					boolean     grid = ( strcmp(p,"_GRID_EW")==0 || strcmp(p,"_GRID_NS")==0);
					/* In ERA 40 they do not care about the grid */
					boolean     supd = ( strcmp(str,"SUPD") == 0 && strcmp(str,"E4") == 0);

					if( supd || (!supd && !grid))
					{
#endif
						marslog(LOG_EROR,"Grib %d contains %s = %s. Grib description follows:",cnt,p,s);
						print_all_requests(grib);
						return -6;
#if 0
					}
#endif
				}
				s = get_value(grib,p,++i);
			}
			j++;
		}
	}
	return NOERR;
}

static const char* fast_value(const request *r,const char* name)
{
	if(r && name)
	{
		parameter *p = r->params;
		while(p)
		{
			if(p->name == name) return p->values?p->values->name:NULL;
			p = p->next;
		}
	}
	return NULL;
}

static int cmp_no_hidden(const request *a,const request *b)
{
	int n;

	static const char* repres   = 0;

	if(repres == 0)
		repres   = strcache("REPRES");

	if(a && b)
	{
		parameter *p = a->params;

		while(p)
		{
			const char *s = fast_value(a,p->name);
			const char *t = fast_value(b,p->name);

			if(s && t && (*p->name!='_') && p->name != repres )
			{

				n = (s != t);
				if(n != 0 && is_number(s)) n = atof(s) - atof(t);
				if(n) {
					if(mars.debug)
						marslog(LOG_DBUG,"Compare failed: %s -> %s <> %s",p->name,s,t);
					return n;
				}
			}

			p = p->next;
		}
	}
	return 0;
}

static int find_order(request **v,request *g)
{
	request *w = 0;
	request *u = *v;

	while(u)
	{
		if(cmp_no_hidden(u,g) == 0)
		{
			int order = u->order;
			if(w != 0)
				w->next = u->next;
			else
				*v = u->next;

#if 1
			/* try to reorganise the list so next time we scan the minumum */

			if(w != 0)
			{
				request *z = w->next;
				request *p = 0;

				while(z) { p = z; z = z->next; }


				if(p != 0)
				{
					p->next = *v;
					*v      = w->next;
					w->next = 0;
				}

			}
#endif

			u->next = NULL;
			free_one_request(u);

			return order;
		}
		w = u;
		u = u->next;
	}

	return -1;
}

int find_long(long* p,int count,long search)
{
	int i;
	for (i = 0; i < count; i++)
		if (p[i] == search)
			return TRUE;
	return FALSE;
}

static err init_subtype_tables(request* r,
		char* subtype_group,
		char* known_subtypes)
{
	memset((void*) subtype_group,0,256);
	memset((void*) known_subtypes,0,256);

	if (count_values(r,"OBSGROUP") == 1)
	{
		request*	dummy_request;
		request*	expanded_request;
		int			count;
		int			i;

		/*----------------------------------------------------------------*/
		/* We expand a dummy request with the obstype value to get all    */
		/* the subtypes in the obstypes. We put 2 into every entry in the */
		/* table                                                          */
		/*----------------------------------------------------------------*/
		count = count_values(r,"OBSTYPE");
		for (i = 0; i < count; i++)
		{
			long st = atol(get_value(r,"OBSTYPE",i));
			if (st < 0 || st > 255)
			{
				marslog(LOG_EROR,"Subtype %d is greater than 255. "\
						"If this is a valid subtype, please contact the MARS group",
						st);
				return -6;
			}
			subtype_group[st] = 2;
		}

		/*----------------------------------------------------------------*/
		/* We expand a dummy request with the obsgroup value to get all   */
		/* the subtypes in the obsgroup. Whenever an entry contains 2     */
		/* it means that it was in the OBSTYPE, we put 1 in it meaning    */
		/* it is also in the obsgroup.                                    */
		/*----------------------------------------------------------------*/
		dummy_request = empty_request("ARCHIVE");
		set_value(dummy_request,"TYPE","OB");
		set_value(dummy_request,"OBSTYPE",get_value(r,"OBSGROUP",0));
		expanded_request = expand_mars_request(dummy_request);
		count = count_values(expanded_request,"OBSTYPE");

		for (i = 0; i < count; i++)
		{
			long st = atol(get_value(expanded_request,"OBSTYPE",i));
			if (st < 0 || st > 255)
			{
				marslog(LOG_EROR,"Subtype %d is greater than 255. "\
						"If this is a valid subtype, please contact the MARS group",
						st);
				return -6;
			}
			if (subtype_group[st] == 2)
				subtype_group[st] = 1;
		}
		free_one_request(expanded_request);

		/*----------------------------------------------------------------*/
		/* After this, if there are any 2s left in the table, then        */
		/* OBSTYPE is not a subset of OBSGROUP.                           */
		/*----------------------------------------------------------------*/
		for (i = 0; i < 256; i++)
		{
			if (subtype_group[i] == 2)
			{
				marslog(LOG_EROR,"OBSTYPE %d is not a subset of OBSGROUP",i);
				marslog(LOG_EROR,"Add it to OBSGROUP %s in mars.def",get_value(r,"OBSGROUP",0));
				return -6;
			}
		}

		/*----------------------------------------------------------------*/
		/* We expand a dummy request with the all the known subtypes      */
		/* (KNOWN).														  */
		/*----------------------------------------------------------------*/
		set_value(dummy_request,"OBSTYPE","KNOWN");
		expanded_request = expand_mars_request(dummy_request);
		count = count_values(expanded_request,"OBSTYPE");
		for (i = 0; i < count; i++)
		{
			long st = atol(get_value(expanded_request,"OBSTYPE",i));
			if (st < 0 || st > 255)
			{
				marslog(LOG_EROR,"Subtype %d is greater than 255. "\
						"If you think this is a valid subtype, please contact the MARS team",
						st);
				return -6;
			}
			known_subtypes[st] = TRUE;
		}

		free_one_request(dummy_request);
		free_one_request(expanded_request);
	}
	else
	{
		marslog(LOG_EROR,"Archiving requires only one OBSGROUP value");
		return -6;
	}

	return 0;
}

/* assumes only one date, one time, one range */
static void get_archive_period(request* r,time_interval* t)
{
	long time = atol(get_value(r,"TIME",0));
	datetime begin = date_time_2_datetime(
			mars_date_to_julian(atol(get_value(r,"DATE",0))),
			time / 100,
			time % 100,
			0);
	long range = atol(get_value(r,"RANGE",0));

	/* range is in minutes */
	init_time_interval(t,begin,range * 60 + 59);
}

static err dhs_fb_check(dhsdata* dhs,request* original_req)
{
	marslog(LOG_INFO,"No checking is carried out on extraction/feedback data");

	return 0;
}

static err dhs_bias_check(dhsdata* dhs,request* original_req)
{
	marslog(LOG_INFO,"No checking is carried out on bias data");

	return 0;
}

static err dhs_track_check(dhsdata* dhs,request* original_req)
{
	marslog(LOG_INFO,"No client check carried out on track data");
	return 0;
}

static long64 file_size(int fd)
{
	long64 size = 0;

#ifdef __linux__
	/* On Linux -m32, lseek64 doesn't work. Use fstat64 instead */
	struct stat st;
	if(fstat(fd,&st) == -1)
		size = -1;
	else
		size = st.st_size;

#else
	if((size = lseek(fd,0,SEEK_END)) < 0)
		size = -1;
#endif
	return size;
}

static err dhs_obs_check(dhsdata* dhs,request* original_req)
{
	long 			length = 1310720;
	char 			buffer[1310720];
	int  			cnt = 0;
	err 			e    = 0;
	FILE 			*f;
	char 			buf[80];
	int				q = 0;
	const 			char* s;
	time_interval	archive_period;
	long			err_count = 0;

	long64			file_length = 0;

	/*----------------------------------------------------------------*/
	/* We want to make sure that data included in the file to be	  */
	/* archived complies with the description given in the request.	  */
	/* 1. the subtypes found in the BUFR files have to be known in    */
	/*    mars language. Otherwise failure.                           */
	/* 2. the subtypes in the OBSTYPE parameter have to be the same   */
	/*    as in the OBSGROUP or a subset of it. Otherwise failure.	  */
	/*	  This ensures that what is archived is what people who are   */
	/*    archiving think is being archived.                          */
	/* 3. There sould only be one date in the request.				  */
	/* 4. There sould only be one time in the request.				  */
	/* 5. There sould only be one range in the request.				  */
	/* 6. Messages which are not in the [date.time,date.time+range]   */
	/*    are not accepted, failure.                                  */
	/* 7. I should have started with this: any problem in the BUFR    */
	/*    file (key missing, wrong length etc..) is a failure.        */
	/*                                                                */
	/* Any other idea?                                                */
	/*----------------------------------------------------------------*/

	/*----------------------------------------------------------------*/
	/* We check subtypes assuming a subtype is less than 256 which is */
	/* currently the case. This speeds up dramatically data processing*/
	/* since we use a table.										  */
	/* If subtypes were codes using more than a byte in the future,   */
	/* this code should be changed (dream on).                        */
	/*                                                                */
	/* Ths subtype_group array contains all the subtypes in the       */
	/* OBSGROUP less the subtypes which are not in the OBSTYPE.       */
	/* Any discrepancy between those cause init_subtype_tables to     */
	/* return an error.                                               */
	/*----------------------------------------------------------------*/
	char		subtype_group[256];
	char		known_subtypes[256];

	e = init_subtype_tables(dhs->req,subtype_group,known_subtypes);
	if (e != 0)
		return e;

	if (count_values(dhs->req,"DATE") > 1)
	{
		marslog(LOG_EROR,"Only one date must be specified for archiving data");
		return -6;
	}
	if (count_values(dhs->req,"TIME") > 1)
	{
		marslog(LOG_EROR,"Only one time must be specified for archiving data");
		return -6;
	}

	get_archive_period(dhs->req,&archive_period);

	dhs->total_tosend = 0;

	start_timer();

	while(e == 0 && (s = get_value(dhs->req,"SOURCE",q++)) != NULL)
	{
		s = no_quotes(s);

		marslog(LOG_INFO,"Scanning and analysing BUFR file %s",s);

		f = fopen(s,"r");
		if(f == NULL)
		{
			marslog(LOG_EROR|LOG_PERR,"ARCHIVE : '%s'",s);
			e = -6;
			break;
		}

		if((file_length = file_size(fileno(f))) < 0)
		{
			marslog(LOG_EROR|LOG_PERR,"Cannot compute size on '%s'",s);
			e = -2;
		}
		else
		{
			if (dhs->total_tosend == 0)
				set_value_long64(original_req,"_length",file_length);
			else
				add_value_long64(original_req,"_length",file_length);
			dhs->total_tosend += file_length;
		}

		if(fseek(f,(off_t)0,SEEK_SET) != 0)
		{
			marslog(LOG_EROR|LOG_PERR,"fseek(%s)",s);
			e = -2;
		}

		length = sizeof(buffer);
		while( !e && (e = _readany(f,buffer,&length)) == NOERR )
			/* || (e == BUF_TO_SMALL)) */
		{
			packed_key k;
			packed_key* keyptr=&k;
			int keyOk;

			keyOk = get_packed_key(buffer,&k);
			if (keyOk)
				keyOk = verify_bufr_key(buffer,length,&k);
			if (keyOk)
			{
				datetime obs_date;
				long subtype;
				const char *obsgroup = get_value(dhs->req,"OBSGROUP",0);
				/*------------------------------------------------------*/
				/* if conventional data, IDENT is checked.				*/
				/* since subtypes 82 to 88 are in the conventional group*/
				/* and these are satellite data, we skip them.			*/
				/*------------------------------------------------------*/
#if 0
				if (EQ(obsgroup,"CONVENTIONAL"))
				{
					if (k.header.subtype < 82 || k.header.subtype > 88)
						if (KEY_IDENT(&k)[0] == ' ')
						{
							marslog(LOG_EROR,
									"IDENT of report %d, subtype %d is left justified.",
									cnt+1,k.header.subtype);
							err_count++;
						}
				}
#endif
				/*------------------------------------------------------*/
				/* check subtypes to be archived						*/
				/*------------------------------------------------------*/
				subtype = k.header.subtype;
				if (subtype < 0 || subtype > 255)
				{
					marslog(LOG_EROR,"Subtype %d is greater than 255. "\
							"If this is a valid subtype, please contact the MARS team",
							subtype);
					err_count++;
				}
				else if (!known_subtypes[subtype])
				{
					marslog(LOG_EROR,"Found report %d with subtype %d which "\
							"is unknown",cnt+1,subtype);
					err_count++;
					marslog(LOG_EROR,"If valid, add %d to the list of KNOWN subtypes in mars.def",subtype);
				}
				else if (!subtype_group[subtype])
				{
					marslog(LOG_EROR,"Found report %d with subtype %d which "\
							"does not match the specified OBSTYPE/OBSGROUP",cnt+1,
							subtype);
					err_count++;
				}

				/*------------------------------------------------------*/
				/* check the message is in archive period				*/
				/*------------------------------------------------------*/
				obs_date = key_2_datetime(keyptr);
				if ( ((obs_date < archive_period.begin) ||
				      (obs_date > archive_period.end) ) && (e == 0) )
				{
					char date[20];
					char time[20];
					print_key_date(keyptr,date);
					print_key_time(keyptr,time);
					marslog(LOG_EROR,"Bufr message n.%d %s %s is out of the archiving period specified in the request",
							cnt+1,date,time);
					err_count++;
				}

				cnt++;
			}
			else
			{
				marslog(LOG_EROR,
						"Key of report %d is not valid",cnt+1);
				err_count++;
			}
			length = sizeof(buffer);

			if (err_count > 50)
				e = -6;
		}

		if(e == EOF) e = 0;
		if(f) fclose(f);

	} /* Loop on sources */

	if (err_count > 0)
		e = -6;

	if(e == NOERR)
	{
		marslog(LOG_INFO,"Archiving %d report(s) from %d file(s)",cnt,q-1);
		dhs->count = cnt;
	}

	set_value(original_req,"_SIZE","%d",cnt);

	stop_timer(buf);
	if(*buf) marslog(LOG_INFO,"Analyse time: %s",buf);

	return e;
}

typedef struct multidatabase {
	boolean done;
	long64  total_tosend;
	int     databases;
} multidatabase;

static err sanity_check(request* r)
{
	const char *suite = getenv("SUITE");
	const char *t = get_value(r,"STREAM",0);

	if(suite && EQ(suite,"bc") && t)
	{
		if((!EQ(t,"SCWV") && !EQ(t,"SCDA")))
		{
			int exp = atol(get_value(r,"EXPVER",0));
			FILE *f = mail_open(mars.dhsmail,"sanity check failed stream: %s",t);
			mail_msg(f,"Archiving from suite bc stream %s",t);
			mail_request(f,"MARS Request:",r);
			mail_close(f);

			if(exp == 1)
			{
				marslog(LOG_EROR,"Archiving from suite %s stream %s. Exiting...",suite,t);
				return -10;
			}

			marslog(LOG_WARN,"Archiving from suite %s stream %s",suite,t);
		}
	}
	return NOERR;

}

static long compute_total(request *r)
{
	long n      = 1;
	parameter *p = r->params;
	while(p)
	{
		n *= count_values(r,p->name);
		p = p->next;
	}
	return n;
}

static err previous_dhs_grib_check(dhsdata *dhs,request* original_req)
{
	long length = 1024*1024*64;
	long field_length;
#ifdef fujitsu
	char *buffer = valloc(length);
#else
	char *buffer = reserve_mem(length);
#endif
	int  cnt = 0;
	err e    = 0;
	FILE *f;
	char buf[80];
	int q = 0;
	const char* s,*t, *cl;
	int count,total;
	char *found;
	boolean isfg = false;
	static multidatabase once = { false, 0, 0};
	request *u = unwind_one_request(dhs->req);
	int dbcount = count_values(dhs->req,"DATABASE");
	long64			file_length = 0;

	timer *read_for_analyse = get_timer("Read GRIBs for analyse",NULL,true);

	s  = get_value(dhs->req,"TYPE",0);
	t  = get_value(dhs->req,"STREAM",0);
	cl = get_value(dhs->req,"CLASS",0);

	isfg = (s && t && EQ(s,"FG") && !EQ(t,"WAVE") && !EQ(t,"SCWV") && !EQ(cl,"MS"));

	if(isfg)
	{
		request *k = un_first_guess_all(u);
		free_all_requests(u);
		u = k;
		marslog(LOG_INFO,"Data is first-guess");
		if(mars.debug)
			print_one_request(u);
#ifndef FG_IS_REAL_FG
		set_value(dhs->req,"TYPE","FC");
#endif
	}


	/* If language accept multi TYPE/CLASS/..., update count_fields
	   and fixed_check */

	count = count_fields(dhs->req);
	total = compute_total(dhs->req);

	/* Avoid performing the check for the same database */
	if(once.done)
	{
		marslog(LOG_INFO,"Source data already analysed");


		dhs->total_tosend = once.total_tosend;
		once.databases++;

		/* Reset it for further requests */
		if(once.databases == dbcount)
		{
			once.done = false;
			once.databases = 0;
			once.total_tosend = 0;
		}

		return NOERR;
	}

	found = NEW_ARRAY_CLEAR(char,total);

	dhs->total_tosend = 0;

	if(mars.debug)
		print_all_requests(u);

	start_timer();

	while(e == 0 && (s = get_value(dhs->req,"SOURCE",q++)) != NULL)
	{

		s = no_quotes(s);

		marslog(LOG_INFO,"Scanning and analysing %s",s);

		f = fopen(s,"r");
		if(f == NULL)
		{
			marslog(LOG_EROR|LOG_PERR,"ARCHIVE : '%s'",s);
			e = -6;
			break;
		}

		if((file_length = file_size(fileno(f))) < 0)
		{
			marslog(LOG_EROR|LOG_PERR,"Cannot compute size on '%s'",s);
			e = -2;
		}
		else
			dhs->total_tosend += file_length;

		if(fseek(f,(off_t)0,SEEK_SET) != 0)
		{
			marslog(LOG_EROR|LOG_PERR,"fseek(%s)",s);
			e = -2;
		}

		if(mars.readdisk_buffer > 0)
		{
			if(setvbuf(f,dhs->read_disk_buffer,_IOFBF,mars.readdisk_buffer))
				marslog(LOG_WARN|LOG_PERR,"setvbuf failed");
		}

		field_length = length;
		while( !e && (e = timed_readany(f,buffer,&field_length,read_for_analyse)) == NOERR )
		{
			int n = -1;
			request *g = empty_request("GRIB");

			if((e = grib_to_request(g,buffer,field_length)) != NOERR)
			{
				marslog(LOG_EROR,"Error while building MARS request from GRIB %d",cnt+1);
				marslog(LOG_EROR,"MARS description for Grib %d is:",cnt+1);
				print_all_requests(g);
			}

			if(mars.debug)
			{
				marslog(LOG_DBUG,"Field %d:",cnt);
				print_all_requests(g);
			}

			if((e == NOERR) && (e = fix_check(g,dhs->req,cnt+1)) == NOERR)
			{
				n = find_order(&u,g);

				if(n < 0 )
				{
					if(field_order(dhs->req,g) >= 0)
						marslog(LOG_EROR,"Grib %d in '%s' is duplicated",cnt+1,s);
					else
						marslog(LOG_EROR,"Grib %d is not in ARCHIVE request",cnt+1);
					marslog(LOG_EROR,"MARS description for Grib %d is:",cnt+1);
					print_all_requests(g);
					e = -6;
				}
				else if(n >= total)
				{
					marslog(LOG_EROR,"MARS internal error analysing grib %d",cnt+1);
					marslog(LOG_EROR,"MARS description for Grib %d is:",cnt+1);
					print_all_requests(g);
					e = -6;
				}
				else if(found[n])
				{
					marslog(LOG_EROR,"Grib %d in '%s' is duplicated",cnt+1,s);
					marslog(LOG_EROR,"MARS description for Grib %d is:",cnt+1);
					print_all_requests(g);
					e = -6;
				}
			}

			/* if((e == NOERR) || (e == BUF_TO_SMALL)) */
			if(e == NOERR)
			{
				found[n] = true;
				cnt++;
			}

			free_all_requests(g);
			field_length = length;
		}

		if(e == EOF) e = 0;
		if(f) fclose(f);

	} /* Loop on sources */

	free_all_requests(u);
	FREE(found);

#ifdef fujitsu
	free(buffer);
#else
	release_mem(buffer);
#endif

	if(!e && cnt != count)
	{
		marslog(LOG_EROR,"Wrong number of fields :");
		marslog(LOG_EROR,"Request describes %d field(s)",count);
		marslog(LOG_EROR,"Source contains %d field(s)",cnt);
		e = -6;
	}
	else if(e == NOERR)
	{
		if(dbcount > 1)
		{
			once.done = true;
			once.total_tosend = dhs->total_tosend;
			once.databases++;
		}
		marslog(LOG_INFO,"Archiving %d field(s) from %d file(s)",cnt,q-1);
		dhs->count = cnt;
	}

	set_value(original_req,"_SIZE","%d",cnt);


	stop_timer(buf);
	if(*buf) marslog(LOG_INFO,"Analyse time: %s",buf);

	if(isfg) set_value(dhs->req,"TYPE","FG"); /* Reset type */

	return e;
}

static err dhs_odb_check(dhsdata *dhs, request* original_req)
{
	err e = NOERR;

#ifdef ODB_SUPPORT
	if(dhs->skip_odb_check)
	{
		marslog(LOG_WARN,"Skip analysing ODB file");
		return NOERR;
	}

	timer *read_for_analyse = get_timer("Read ODB contents for analysis", NULL, true);
	const char *s;
	int q = 0;

	while(e == 0 && (s = get_value(dhs->req, "SOURCE", q++)) != NULL)
	{
		const char *fileName = no_quotes(s);
		marslog(LOG_INFO, "Scanning and analysing ODB file '%s'", fileName);
		request *r = empty_request(0);

        if ((e = odb_to_request_from_file(r, fileName)) != 0)
		{
			marslog(LOG_EROR, "odb_to_request_from_file: %d", e);
			return e;
		}
		else
		{
			if (mars.debug)
			{
				marslog(LOG_INFO, "file's metadata:");
				print_all_requests(r);
				marslog(LOG_INFO, "user's request:");
				print_all_requests(original_req);
			}

			if ((e = odb_compare_attributes_of_first_request(r, original_req)) != 0)
			{
				marslog(LOG_INFO, "file's metadata:");
				print_all_requests(r);
				marslog(LOG_EROR, "odb_compare_attributes_of_first_request: %d", e);
				return e;
			}

		}
	}
#else
	marslog(LOG_EROR,"This MARS client doesn't support ODB");
	marslog(LOG_EROR,"Please, contact the MARS team");
	e = -1;
#endif

	return e;
}

#if mars_client_HAVE_NETCDF
static err dhs_netcdf_check(dhsdata *dhs,request* original_req)
{
    int i = 0;
    const char* path;
    request *netcdf = NULL;
    off_t offset = 0;
    struct stat st;

     /* we are at the ed of the message */


    while((path = get_value(original_req, "SOURCE", i++)) != NULL)
    {
    	err e = 0;
    	request* r;
        path = no_quotes(path);

    	if(stat(path, &st))
		{
			marslog(LOG_EROR|LOG_PERR,"Cannot stat %s", path);
			free_all_requests(netcdf);
			return -2;
		}


    	mars_field_index* idx = mars_field_index_new( offset, st.st_size);
    	offset += st.st_size;

        r = netcdf_to_request(path, 1, idx, &e);


        if(e != NOERR) {
            marslog(LOG_EROR, "NetCDF validation failed for %s", path);
            free_all_requests(netcdf);
            return -1;
        }

        if(netcdf) {
        	reqmerge(netcdf, r);
        	free_one_request(r);
        }
        else {
        	netcdf = r;
        }

        if( dhs->head_idx == NULL )
		{
			dhs->head_idx = dhs->tail_idx = idx;
		}
		else
		{
			dhs->tail_idx->next = idx;
			dhs->tail_idx = idx;
		}
    }

    print_all_requests(netcdf);
    // mars_field_index_print(dhs->head_idx);

    return 0;
}
#endif

static err dhs_grib_check(dhsdata *dhs,request* original_req)
{
#if 0
	long length  = preferred_IO_blocksize(no_quotes(get_value(dhs->req,"SOURCE",0)),1024*1024*20);
#endif
	long length = 0;
	char *buffer;
	int  cnt = 0;
	err e    = 0;
	FILE *f;
	char buf[80];
	int q = 0;
	const char* s,*t, *cl;
	int count;
	boolean isfg = false;
	boolean isclim = false;
	static multidatabase once = { false, 0, 0};
	int dbcount = count_values(dhs->req,"DATABASE");
	long64			file_length = 0;
	hypercube  *h = 0;
	long field_length = length;
	long64   total_prev_files = 0;

	if(mars.autoarch) {
		marslog(LOG_INFO,"Source data already analysed (MARS_AUTO_ARCH=%ld)", mars.autoarch);
		return 0;
	}

	length = mars.readany_buffer_size;
	field_length = length;

	timer *read_for_analyse = get_timer("Read GRIBs for analyse",NULL,true);

	t  = get_value(dhs->req,"TYPE",0);
	s  = get_value(dhs->req,"STREAM",0);
	cl = get_value(dhs->req,"CLASS",0);

	isfg   = (s && t && EQ(t,"FG") && !EQ(s,"WAVE") && !EQ(s,"SCWV") && !EQ(cl,"MS"));
	isclim = (s && t && EQ(t,"CL") && EQ(s,"MNTH"));


	if(isfg || isclim)
	{
		marslog(LOG_INFO,"Special check on first-guess or climatology");
		return previous_dhs_grib_check(dhs,original_req);
	}

#ifdef fujitsu
	buffer = valloc(length);
#else
	buffer = reserve_mem(length);
#endif

	/* If language accept multi TYPE/CLASS/..., update count_fields
	   and fixed_check */

	count = count_fields(dhs->req);

	/* Avoid performing the check for the same database */
	if(once.done)
	{
		marslog(LOG_INFO,"Source data already analysed");


		dhs->total_tosend = once.total_tosend;
		once.databases++;

		/* Reset it for further requests */
		if(once.databases == dbcount)
		{
			once.done = false;
			once.databases = 0;
			once.total_tosend = 0;
		}

		return NOERR;
	}

	dhs->total_tosend = 0;

	start_timer();

	h = new_hypercube_from_mars_request(dhs->req);
	while(e == 0 && (s = get_value(dhs->req,"SOURCE",q++)) != NULL)
	{
		int field = 0;

		s = no_quotes(s);

		marslog(LOG_INFO,"Scanning and analysing GRIB file %s",s);

		f = fopen(s,"r");
		if(f == NULL)
		{
			marslog(LOG_EROR|LOG_PERR,"ARCHIVE : '%s'",s);
			e = -6;
			break;
		}

		if((file_length = file_size(fileno(f))) < 0)
		{
			marslog(LOG_EROR|LOG_PERR,"Cannot compute size on '%s'",s);
			e = -2;
		}
		else
			dhs->total_tosend += file_length;

		if(fseek(f,(off_t)0,SEEK_SET) != 0)
		{
			marslog(LOG_EROR|LOG_PERR,"fseek(%s)",s);
			e = -2;
		}


		field_length = length;
		while( !e && (e = timed_readany(f,buffer,&field_length,read_for_analyse)) == NOERR )
			/* || (e == BUF_TO_SMALL)) */
		{
			mars_field_index* idx = NULL;

			request *g = empty_request(0);

			/* if(e == BUF_TO_SMALL) e = NOERR; */

			if( mars.build_grib_index ) {

				off_t here = ftell(f) + total_prev_files;
				idx = mars_field_index_new( here - field_length, field_length); /* we are at the ed of the message */

				if( dhs->head_idx == NULL )
				{
					dhs->head_idx = dhs->tail_idx = idx;
				}
				else
				{
					dhs->tail_idx->next = idx;
					dhs->tail_idx = idx;
				}
			}

			if((e = grib_to_request_index(g,buffer,field_length,idx)) != NOERR)
			{
				marslog(LOG_EROR,"Error while building MARS request from GRIB %d",cnt+1);
				marslog(LOG_EROR,"MARS description for Grib %d is:",cnt+1);
				print_all_requests(g);
			}

			/* If BUDG or TIDE, create request from GRIB with
			   original request as template */
			if(mars.pseudogrib)
			{
				request *copy = get_cubelet(h,0);

				free_all_requests(g);
				g = clone_one_request(copy);

				if((e = grib_to_request_index(g,buffer,field_length,idx)) != NOERR)
				{
					marslog(LOG_EROR,"Error while building MARS request from pseudo-GRIB %d",cnt+1);
					marslog(LOG_EROR,"MARS description for pseudo-GRIB %d is:",cnt+1);
					print_all_requests(g);
				}

			}

			if(mars.debug)
			{
				marslog(LOG_DBUG,"Field %d:",cnt);
				print_all_requests(g);
			}

			if(e == NOERR)
				e = remove_field_from_hypercube(h,g,cnt+1);

			if(e == NOERR)
			{
				cnt++;
				field++;
			}

			field_length = length;
			free_all_requests(g);
		}

		if(e == EOF) e = 0;
		if(e != 0)
			marslog(LOG_EROR,"Error occured for field %d",field+1);

		if(f) fclose(f);

		total_prev_files += file_length;
	} /* Loop on sources */

	free_hypercube(h);

#ifdef fujitsu
	free(buffer);
#else
	release_mem(buffer);
#endif

	if(!e && cnt != count)
	{
		marslog(LOG_EROR,"Wrong number of fields :");
		marslog(LOG_EROR,"Request describes %d field(s)",count);
		marslog(LOG_EROR,"Source contains %d field(s)",cnt);
		e = -6;
	}
	else if(e == NOERR)
	{
		if(dbcount > 1)
		{
			once.done = true;
			once.total_tosend = dhs->total_tosend;
			once.databases++;
		}
		marslog(LOG_INFO,"Archiving %d field(s) from %d file(s)",cnt,q-1);
		dhs->count = cnt;
	}

	set_value(original_req,"_SIZE","%d",cnt);


	stop_timer(buf);
	if(*buf) marslog(LOG_INFO,"Analyse time: %s",buf);

	if(isfg) set_value(dhs->req,"TYPE","FG"); /* Reset type */

	return e;
}

static err dhs_image_check(dhsdata *dhs)
{
	err e    = 0;
	FILE *f;
	int q = 0;
	const char* s;
	char buf[80];
	long64 file_length = 0;


	dhs->total_tosend = 0;
	start_timer();

	while(e == 0 && (s = get_value(dhs->req,"SOURCE",q++)) != NULL)
	{
		s = no_quotes(s);
		marslog(LOG_INFO,"Scanning %s",s);

		f = fopen(s,"r");
		if(f == NULL)
		{
			marslog(LOG_EROR|LOG_PERR,"ARCHIVE : '%s'",s);
			e = -6;
			break;
		}

		if((file_length = file_size(fileno(f))) < 0)
		{
			marslog(LOG_EROR|LOG_PERR,"Cannot compute size on '%s'",s);
			e = -2;
		}
		else
			dhs->total_tosend += file_length;

		fclose(f);

	} /* Loop on sources */


	stop_timer(buf);
	if(*buf) marslog(LOG_INFO,"Scanning time: %s",buf);

	return e;
}

static err dhs_store_check(dhsdata *dhs)
{
	long length = 32768;
	char buffer[32768];
	err e    = 0;
	FILE *f;
	char buf[80];
	int q = 0;
	const char* s;
	long64 file_length = 0;

	dhs->total_tosend = 0;

	start_timer();

	while(e == 0 && (s = get_value(dhs->req,"SOURCE",q++)) != NULL)
	{

		s = no_quotes(s);

		marslog(LOG_INFO,"Scanning and analysing %s",s);

		f = fopen(s,"r");
		if(f == NULL)
		{
			marslog(LOG_EROR|LOG_PERR,"STORE : '%s'",s);
			e = -6;
			break;
		}

		if((file_length = file_size(fileno(f))) < 0)
		{
			marslog(LOG_EROR|LOG_PERR,"Cannot compute size on '%s'",s);
			e = -2;
		}
		else
			dhs->total_tosend += file_length;

		rewind(f);

		/*---------------------------*/

		/* Check for GRIB or BUFR */

		if(_readany(f,buffer,&length) != -1)
		{
			marslog(LOG_EROR,"STORE: %s contains GRIB or BUFR",s);
			e = -7;
		}

		rewind(f);

		/* check for something else....*/

		/*---------------------------*/

		fclose(f);

	} /* Loop on sources */

	stop_timer(buf);
	if(*buf) marslog(LOG_INFO,"Analyse time: %s",buf);

	return e;
}

/* End of Aux functions */

static err send_request(mstream* s,request *r)
{
	int n = 0;
	parameter *p = r->params;

	/* stream_write_start(s,"Request"); */
	stream_write_string(s,r->name);

	/* count */

	while(p) { n++; p = p->next; }
	stream_write_int(s,n);

	p = r->params;
	while(p)
	{
		value *v = p->values;

		if( p->values == NULL)
			marslog(LOG_EXIT,"Internal error: missing value for %s",p->name);

		stream_write_string(s,p->name);

		n = 0;
		while(v) { n++; v = v->next; }
		stream_write_int(s,n);

		v = p->values;
		while(v)
		{
			stream_write_string(s,v->name);
			v = v->next;
		}

		p = p->next;
	}

	/* stream_write_end(s); */

	return s->error;
}

static err compute_size(dhsdata *dhs)
{
	int q = 0;
	err e = 0;
	const char* s = NULL;
	struct stat st;

	dhs->total_tosend = 0;

	while(e == 0 && (s = get_value(dhs->req,"SOURCE",q++)) != NULL)
	{
		s = no_quotes(s);
		if(stat(s,&st))
		{
			marslog(LOG_EROR|LOG_PERR,"Cannot stat %s",s);
			return -2;
		}
		dhs->total_tosend += st.st_size;
	}

	return e;
}

static err send_source(dhsdata *dhs)
{
	long length  = preferred_IO_blocksize(no_quotes(get_value(dhs->req,"SOURCE",0)),1024*1024);
#ifdef fujitsu
	char *buffer = valloc(length);
#else
	char *buffer = (char *) reserve_mem(length);
#endif
	long readbufferlen = 1024*1024*20;
	char *readbuffer = NULL;
	err e = 0;
	const char *s;
	int q = 0;
	long64 total = 0;
	long n;
	FILE *f;
	unsigned long crc = 0xffffffffL;

	timer *disk_time = get_timer("Read from disk",NULL,true);
	timer *net_time  = get_timer("Write to network","transfertime",true);
	timer *send_time = get_timer("Transfer to server",NULL,true);

	double start_send_time = timer_value(send_time);
	double start_disk_time = timer_value(disk_time);
	double start_net_time = timer_value(net_time);

	if(dhs->total_tosend == 0)
	{
		e = compute_size(dhs);
		if(e) return e;
	}

	timer_start(send_time);

	stream_write_longlong(&dhs->s,dhs->total_tosend);

	while(e == 0 && (s = get_value(dhs->req,"SOURCE",q++)) != NULL)
	{
		s = no_quotes(s);
		marslog(LOG_INFO,"Sending %s",s);

		f = fopen(s,"r");

		if(!f)
		{
			marslog(LOG_EROR|LOG_PERR,"fopen(%s)",s);
			e =  -2;
			break;
		}

		if(mars.readdisk_buffer > 0)
		{
			if(setvbuf(f,dhs->read_disk_buffer,_IOFBF,mars.readdisk_buffer))
				marslog(LOG_WARN|LOG_PERR,"setvbuf failed");
		}

		while( (n = timed_fread(buffer,1,length,f,disk_time))  > 0 )
		{
			if(timed_writetcp(&dhs->csoc,buffer,n,net_time) != n)
			{

				marslog(LOG_EROR|LOG_PERR,"writetcp(%s)",s);

				if(errno > 0)
				{
					char h[80];
					FILE *f = 0;

					gethostname(h,sizeof(h));
					f = mail_open(mars.dhsmail,"(%s) from %s to %s", no_quotes(strerror(errno)), h, dhs->host);
					mail_msg(f,"Archive job failed");
					mail_request(f,"MARS Request:",dhs->req);
					mail_close(f);
				}

				e = RETRY_FOREVER_ERR;
				break;
			}
			total += n;
		}

		if(dhs->obs || dhs->is_bufrfile)
			log_statistics("reports","%d",dhs->count);
		else
			log_statistics("fields","%d",dhs->count);
		log_statistics("database","%s",database_name(dhs));
		log_statistics_unique("bytes","%lld",total);

		if(ferror(f))
		{
			marslog(LOG_EROR|LOG_PERR,"fread(%s)",s);
			e = -2;
		}

		fclose(f);

	}

	if(total != dhs->total_tosend)
	{
		marslog(LOG_EROR|LOG_PERR,"Only %lld bytes out of %lld sent.",
				total,dhs->total_tosend);
		e = e ? e : -2;
	}

	/* Send CRC */

	{
		crc ^= 0xffffffff;
		stream_write_ulong(&dhs->s,2);      /* crc version */
		stream_write_ulonglong(&dhs->s,crc);
		if(mars.crc) marslog(LOG_INFO,"CRC %u\n",crc);
	}

	timer_stop(send_time,total);

	timer_partial_rate(send_time,start_send_time,total);
	timer_partial_rate(disk_time,start_disk_time,total);
	timer_partial_rate(net_time,start_net_time,total);


#ifdef fujitsu
	free(buffer);
#else
	release_mem(buffer);
#endif

	return e;

}

static err receive_data(dhsdata *dhs)
{
	long64 total = 0;
	long len;
	char buffer[1024*100];
	unsigned int n = sizeof(buffer);
	long64 size;

	size = stream_read_longlong(&dhs->s);

	while( (len = readtcp(&dhs->csoc,buffer,n)) >= 0)
		total += len;

	marslog(LOG_INFO,"%lld bytes received",total);

	if(size != total)
	{
		marslog(LOG_EROR,"Got only %lld bytes out of %lld", total, size);
		return -2;
	}

	return 0;

}

typedef struct retrymsg_ {
	char    *msg;
	boolean forever;
} retrymsg;

static void check_msg(dhsdata *dhs, const char* msg)
{
	static retrymsg retry[] ={
		{ "Retry: ",             true},   /* Server asks for a retry */
		{ "ANS1314E (RC14)",     false},  /* File data currently unavailable on server */
		{ "ANS1017E (RC-50)",    false},  /* Session rejected: TCP/IP connection failure */
		{ "ANS1301E (RC1)",      false},  /* Server detected system error */
		{ "ANS1351E (RC51)",     false},  /* Session rejected: all server sessions are currently in use */
		{ "ANS0221E (RC2014)",   false},  /* There was an error in the TSM API internals */
		{ "Bad Tag",             false},  /* MARS server error */
		{ "Bad tag",             false},  /* MARS server error */
		{ "Write error on pipe", false},  /* MARS server error */
		{ "Read error on pipe", false},   /* MARS server error */
		{ "Failed HPSS call",    true},   /* Calls to HPSS unrecoverable on server for time being */
		{ "Double buffer error: Failed HPSS call",    true},   /* Calls to HPSS unrecoverable on server for time being */
	};

	if(*msg)
	{
		int i = 0;
		for(i=0; i<NUMBER(retry); ++i)
		{
			int msglen = strlen(msg);
			int len = strlen(retry[i].msg);
			int min = len<msglen?len:msglen;

			if(strncmp(retry[i].msg,msg,min) == 0)
			{
				if(retry[i].forever)
				{
					dhs->retry_forever_on_client = true;
					marslog(LOG_WARN,"Retry failure forever");
				}
				else
					dhs->retry_on_client = true;

			}
		}
	}
}

static void msg(dhsdata *dhs,int flg)
{
	const char* p = stream_read_string(&dhs->s);
	if(*p) marslog(flg,"%s [%s]",p,database_name(dhs));
	check_msg(dhs,p);
}


static void notification(dhsdata *dhs)
{
	const char* p = stream_read_string(&dhs->s);
	if(*p) {
		if(strcmp(p,"wind conversion requested") == 0) {
			mars.wind_requested_by_server = true;
			marslog(LOG_INFO,"Wind conversion requested by server");
		}
		// else if(strcmp(p,"data is netcdf") == 0) {
		// 	dhs->netcdf = true;
		// 	marslog(LOG_INFO,"Data retrieved is in NetCDF format");
		// }
		else {
			marslog(LOG_WARN,"Unknown notification recieved: %s [%s]",p,database_name(dhs));
		}
	}
	/* Send acknowlegment */
	stream_write_int(&dhs->s,0);
}

static void notificationStart(dhsdata *dhs)
{


	/* Send acknowlegment */
	stream_write_int(&dhs->s,0);
}

static void statistics(dhsdata *dhs)
{
	int n = stream_read_int(&dhs->s);
	char *log_server_statistics = getenv("MARS_LOG_SERVER_STATISTICS");

	marslog(LOG_DBUG,"Got %d statistics from server",n);
	if(!log_server_statistics)
		marslog(LOG_DBUG,"Server statistics disabled. Set MARS_LOG_SERVER_STATISTICS to enable");

	while(n-->0)
	{
		char* a = strcache(stream_read_string(&dhs->s));
		char* b = strcache(stream_read_string(&dhs->s));
		if(a && b && log_server_statistics)
		{
			marslog(LOG_DBUG,"Server statistics %s = %s",a,b);
			log_statistics(a,"%s",b);
		}
		strfree(a);
		strfree(b);
	}
}

static err list(dhsdata* dhs)
{
	char buffer[1024];
	int length = sizeof(buffer);
	int n;
	FILE *f = stdout;
	err e = 0;
	const char *target = no_quotes(mars.webmars_target ? mars.webmars_target : get_value(dhs->req,"TARGET",0));
	static int first = true;
	long long size = 0;

	int dbcount = count_values(dhs->req,"DATABASE");

	if(target)
	{
		if(first)
		{
			f  = fopen(target,target_open_mode(target));
		}
		else
			f = fopen(target,"a+");

		if(!f)
		{
			marslog(LOG_EROR|LOG_PERR,"fopen(%s)",target);
			return -2;
		}
	}

	if(dbcount>1)
	{
		if(!first)
			fprintf(f,"\n\n==================================================================\n\n");
		fprintf(f,"datab   = %s\n",database_name(dhs));
	}

	if(first)
		first = false;

	while( (n = fread(buffer,1,length,dhs->f))  > 0 )
	{
		fwrite(buffer,1,n,f);
		length = sizeof(buffer);
		size += n;
	}

	if(ferror(dhs->f))
	{
		marslog(LOG_EROR|LOG_PERR,"Error during list");
		e = -2;
	}

	socket_file_close(dhs->f);
	dhs->f = NULL;

	if(dhs->total_toread != 0 && dhs->total_toread != size)
	{
		marslog(LOG_EROR,"LIST: read only %lld out of %lld",size,dhs->total_toread);
		e = -2;
	}

	if(target && f)
	{
		if(fclose(f) != 0)
		{
			marslog(LOG_EROR|LOG_PERR,"fclose(%s)",target);
			return -2;
		}
	}

	return e;
}

static err get(dhsdata* dhs)
{
	char buffer[1024*1024];
	int length = sizeof(buffer);
	int n;
	err e = 0;
	const char *target = no_quotes(mars.webmars_target ? mars.webmars_target : get_value(dhs->req,"TARGET",0));
	FILE *f = 0;

	if(target)
	{

		if(*target == '|')
			f = popen(target+1,"w");
		else
			f = fopen(target,"w");
		if( !f )
		{
			marslog(LOG_EROR|LOG_PERR,"fopen(%s)",target);
			return -2;
		}
	}
	else
	{
		marslog(LOG_EROR,"Missing target");
		return -2;
	}

	while( (n = fread(buffer,1,length,dhs->f))  > 0 )
	{
		if(fwrite(buffer,1,n,f) != n)
		{
			marslog(LOG_EROR|LOG_PERR,"fwrite(%s)",target);
			e = -2;
			break;
		}
		length = sizeof(buffer);
	}

	if(ferror(dhs->f))
	{
		marslog(LOG_EROR|LOG_PERR,"Error during list");
		e = -2;
	}

	socket_file_close(dhs->f);
	dhs->f = NULL;

	n = (*target == '|') ? pclose(f) : fclose(f);
	if(n)
	{
		marslog(LOG_EROR|LOG_PERR,"fclose(%s)",target);
		return -2;
	}

	return e;
}

static err read_metadata(dhsdata *dhs)
{
	const char *name;
	int version = 0;
	long64 length = 0;
	mstream s;

	if(dhs->total_read == dhs->total_toread)
		return EOF;

	make_file_stream(&s,dhs->f);

	free_all_requests(dhs->metadata);
	dhs->metadata = NULL;

	FREE(dhs->reclen);
	dhs->reclen = NULL;

	name    = stream_read_start(&s);
	version = stream_read_long(&s);
	length  = stream_read_longlong(&s);

	marslog(LOG_DBUG,"read_metadata(%s)",name);
	marslog(LOG_DBUG,"read_metadata(%d)",version);
	marslog(LOG_DBUG,"read_metadata(%lld)",length);

	stream_read_start(&s);
	dhs->metadata = read_request(&s);
	stream_read_end(&s);
	stream_read_end(&s);

	if(mars.debug)
		print_all_requests(dhs->metadata);

	dhs->total_read += s.in;
	dhs->data_size   = length;

	return s.error?RETRY_ERR:0;
}

static err timed_dhs_wait(dhsdata *dhs)
{
	err e = 0;
	char buf[1024];
	timer *wait_time = 0;

	sprintf(buf,"Processing in %s",database_name(dhs));
	wait_time=get_timer(buf,NULL,true);

	timer_start(wait_time);
	e = dhs_wait(dhs);
	timer_stop(wait_time,0);
	return e;
}

static err dhs_wait(dhsdata *dhs)
{
	struct sockaddr_in from;
	marssocklen_t fromlen = sizeof(struct sockaddr_in);
	ulong64 id;
	char code;
	err e = 0;
	long t = 0;
	static int protocol_mismatch = 0;

	if(dhs->finished)
		return EOF;

	if(dhs->csoc < 0)
	{
		if(setjmp(env) != 0) {
			marslog(LOG_WARN,"Timeout waiting for %s",dhs->host);
			return dhs->retry_on_timeout ? RETRY_ERR : -2;
		}

		if(dhs->timeout) {
			signal(SIGALRM,catch_alarm);
			alarm(dhs->timeout*60);
		}

		socket_buffers(dhs->asoc);

		/* marslog(LOG_INFO,"Opening socket"); */
		dhs->csoc =  accept(dhs->asoc, (struct sockaddr*)&from, &fromlen);

		alarm(0);

		if(dhs->csoc < 0)
		{
			marslog(LOG_EROR|LOG_PERR,"accept");
			return -2;
		}

		if(!dhs->quiet)
		{
			print_address("Mars client is on",addr_of(dhs->csoc));
			traceroute(&from);
			print_address("Mars server is on",&from);
			dhs->quiet = true;
		}


		if(dhs->print_callback)
		{
			print_address("Got connection from", &from);
		}
	}
	else
	{
		/* marslog(LOG_INFO,"Socket already open"); */
	}

	id = stream_read_ulonglong(&dhs->s);

	if(dhs->s.error)
	{
		marslog(LOG_WARN,"dhs_wait: stream_read_ulonglong: got error %d",dhs->s.error);
		return dhs->s.error;
	}

	if(id != dhs->id)
	{
		marslog(LOG_EROR,"Failed to check callback id");
		return -1;
	}

	code = stream_read_char(&dhs->s);
	if(dhs->s.error)
	{
		marslog(LOG_WARN,"dhs_wait: stream_read_char: got error %d",dhs->s.error);
		return dhs->s.error;
	}

	marslog(LOG_DBUG,"Code = %c",code);

	switch(code)
	{
		/* OK */
		case 'o':
			dhs->finished = true;
			if(EQ(dhs->req->name,"RETRIEVE") && (!dhs->server_send_error && !dhs->server_send_data))
			{
				marslog(LOG_EROR,"Receive end of request from server without data. Retrying %d",++protocol_mismatch);
				if(protocol_mismatch < 3)
					e = RETRY_ERR;
				else
					marslog(LOG_EROR,"Too many errors in protocol. Give up");
			}
			break;

			/* Aknowlegde */
		case 'a':
			break;

			/* read source */
		case 'r':
			e = send_source(dhs);
			if(e == RETRY_ERR || e == RETRY_FOREVER_ERR)
			{
				dhs->retry_on_client         = (e == RETRY_ERR);
				dhs->retry_forever_on_client = (e == RETRY_FOREVER_ERR);
				marslog(LOG_WARN,"Transfer failed, retrying...");
				e = 0; /* Reset error, we will be called again */
			}
			break;

		case 'h':
			dhs->f = fdopen(dhs->csoc,"r");
			dhs->total_toread = 0;
			dhs->csoc = -1;
			return get(dhs);
			/* break; */

			/* read source */
		case 'w':

			dhs->server_send_data = true;
			dhs->f = fdopen(dhs->csoc,"r");
			dhs->total_toread = stream_read_longlong(&dhs->s);

			if(dhs->total_toread)
				marslog(LOG_INFO,"Transfering %lld bytes",dhs->total_toread);

			dhs->csoc = -1;
			if(dhs->s.error)  return RETRY_ERR;

			/* it's a LIST */
			if(EQ(dhs->req->name,"LIST"))
				return list(dhs);

			if(EQ(dhs->req->name,"BROWSE"))
				return list(dhs);

			/* it's a GET */
			if(EQ(dhs->req->name,"GET"))
				return get(dhs);

			return 0;
			/* break; */

		case 'm':
			e =  read_metadata(dhs);
			break;

		case 'X':
			if(mars.private_key)
			{
				/* Authentication challenge */
				char cmd[1024];
				long size = 0;
				char *tmp = marstmp();
				FILE *f;
				const char *message = stream_read_blob(&dhs->s, &size);
				sprintf(cmd,"openssl dgst -sha1 -sign %s -out %s", mars.private_key, tmp);
				f = popen(cmd,"w");
				if(!f) {  e = -2;  }
				else {
					char digest[4096];
					if(fwrite(message,1,size,f) != size) { e = -2; }
					if(pclose(f)) { e = -2 ; }
					f = fopen(tmp,"r"); unlink(tmp);
					if(!f) {e = -2; }
					else {
						char digest[4096];
						size = fread(digest,1,sizeof(digest),f);
						if(size <=0 || size >= sizeof(digest))
						{
							e = -2;
						}
						fclose(f);
						stream_write_blob(&dhs->s,digest,size);

					}

				}

				if(e) {
					marslog(LOG_EROR,"Server requested authentication, error during signing");
				}
			}
			else
			{
				marslog(LOG_EROR,"Server requested authentication, but MARS_PRIVATE_KEY not defined");
				e = -2;
			}
			break;

		case 'e':

			marslog(LOG_EROR,"Mars server task finished in error");

			dhs->server_send_error = true;
			if(dhs->finish_on_error)
				dhs->finished=true;

			msg(dhs,LOG_EROR);
			if(dhs->retry_on_client)
				e = RETRY_ERR;
			else if(dhs->retry_forever_on_client)
				e = RETRY_FOREVER_ERR;
			else
				e = -2;

			marslog(LOG_EROR,"Error code is %d", e);

			break;

		case 'y':    /* retry */
			dhs->finished=true;
			msg(dhs,LOG_WARN);
			e = RETRY_ERR;
			break;

		case 'I': /* info */
			msg(dhs,LOG_INFO);
			break;

		case 'W': /* warning */
			msg(dhs,LOG_WARN);
			break;

		case 'D': /* debug */
			msg(dhs,LOG_DBUG);
			break;

		case 'E': /* error */
			msg(dhs,LOG_EROR);
			break;

        case 'G': /* send the index scan, used to be only GRIB, but we now support NETCDF too */
            marslog(LOG_INFO,"Sending metadata index");
			if( dhs->head_idx )
			{
                timer *send_time = get_timer("Transfer metadata index to server",NULL,true);
				timer_start(send_time);
				e = mars_field_index_send(dhs->head_idx,&dhs->s);
				timer_stop(send_time,0);
			}else
			{
                marslog(LOG_EROR,"Could not find metadata index");
				e =  -2;
			}
			break;

		case 'N': /* notification */

			notification(dhs);
			break;

		case 'p': /* ping */
			stream_write_char(&dhs->s,'p');
			break;

		case 's': /* statistics */
			statistics(dhs);
			break;

		case 'S': /* notification start */
			notificationStart(dhs);
			break;

		case 't': /* new timeout */
			t = stream_read_long(&dhs->s);
			if(t != 0) /* New timeout requested */
			{
				if(t < 0) /* Infinite */
				{
					marslog(LOG_INFO,"New MARS server: timeout set to infinite");
					dhs->timeout = 0;
				}
				else {
					marslog(LOG_INFO,"New MARS server: timeout set to %d minute(s)",t);
					dhs->timeout = t;
				}
			}
			break;

		default:
			marslog(LOG_EROR,"Unknown code %c (%d)",code,code);
			e =  -2;
			break;
	}

	socket_close(dhs->csoc);
	dhs->csoc = -1;

	if(e == NOERR && dhs->s.error) e =  dhs->s.error;

	if(e)
		dhs->finished = true;

	if(dhs->retry_on_client) e = RETRY_ERR;
	if(dhs->retry_forever_on_client) e = RETRY_FOREVER_ERR;

	return e;

}

static err dhs_send(dhsdata *dhs,request *r,request *e)
{
	char address[80];
	int  port;
	char *addr;
	timer *connect_time = 0;
	char buf[1024];
#if AIX
	int  retry = 0;
#endif

	sprintf(buf,"Connecting to %s",database_name(dhs));
	connect_time=get_timer(buf,NULL,true);

	/* Open the callback port */

	if(dhs->local_port)
	{
		port = dhs->local_port;
		dhs->asoc = tcp_server(port);
		gethostname(address,sizeof(address));
	}
	else
	{
		dhs->asoc = server_mode(&port,address);
		/* don't trust socket address as it sometimes resolve as a localhost */
		if(strncmp(address, "127.", 4) == 0) {
			marslog(LOG_WARN, "Local socket address resolve to local IP: [%s]", address);
			gethostname(address,sizeof(address));
			marslog(LOG_WARN,"Using hostname for callback address instead: [%s]", address);
		}
	}

	if(dhs->asoc<0)
		return -1;

#if AIX
	if(!dhs->local_port)
		while(++retry<10)
		{
			int tmp = dhs->asoc;
			struct servent *tcpserv = getservbyport(port,"tcp");
			if(tcpserv == NULL)
				break;
			marslog(LOG_INFO,"port %d already defined for service '%s'",port,tcpserv->s_name);
			dhs->asoc = server_mode(&port,address);
			if(dhs->asoc<0)
				return -1;
			close(tmp);
		}
#endif

	marslog(LOG_INFO,"Calling mars on '%s', local port is %d",dhs->host,port);

	timer_start(connect_time);
	dhs->csoc = call_server(dhs->host,dhs->port,dhs->retry);
	timer_stop(connect_time,0);
	if(dhs->csoc<0)
		return -1;


	/* Set up the stream to send the request */

	make_socket_stream(&dhs->s,&dhs->csoc);

	/* This should be unique */

	dhs->id     =  (((ulong64)getpid()) << 32) | (ulong64)time(0);

	/* Send info */

	stream_write_start(&dhs->s,"MarsTask");

	/* send id */
	stream_write_ulonglong(&dhs->s,0);

	if(dhs->mars2mars) {
		set_value(r, "_expect", "%d", dhs->expect);
	}

	/* Send requests */

	send_request(&dhs->s,r);
	send_request(&dhs->s,e);

	/* Send cb info */

	addr = dhs->address?dhs->address:address;

	if(dhs->callback_host) {
		addr = dhs->callback_host;
	}

	if(dhs->callback_port) {
		port = dhs->callback_port;
	}

	if(mars.show_hosts) {
		marslog(LOG_INFO,"Callback at address %s, port %d",addr,port);
	}

	stream_write_string(&dhs->s,addr);
	stream_write_int(&dhs->s,port);
	stream_write_ulonglong(&dhs->s,dhs->id);

	/* Send datahandle */

	if(dhs->mars2mars) {
		request* req = empty_request("RETRIEVE");
		stream_write_start(&dhs->s,"Mars2MarsHandle");


		reqcpy_no_underscores(req, r);
		send_request(&dhs->s, req);
		free_all_requests(req);

		send_request(&dhs->s, dhs->othermars);

		stream_write_string(&dhs->s, no_quotes(get_value(r, "SOURCE", 0)) + 7);

		stream_write_end(&dhs->s);

	}
	else {

		stream_write_start(&dhs->s,"MarsHandle");

		stream_write_string(&dhs->s,addr);
		stream_write_int(&dhs->s,port);
		stream_write_ulonglong(&dhs->s,dhs->id);

		stream_write_int(&dhs->s,mars.crc);

		stream_write_end(&dhs->s);

	}


	stream_write_end(&dhs->s);

	if(dhs->s.error)
		return dhs->s.error;


	dhs->u_v   = wind_new(r,&dhs->total_read, dhs->makeuv);

	return timed_dhs_wait(dhs);
}

static err check_file_sizes(const request *r)
{
	err e = NOERR;
	int q = 0;
	const char *s = 0;

	while(e == 0 && (s = get_value(r,"SOURCE",q++)) != NULL)
	{
		long64 file_length = 0;
		FILE *f = NULL;
		s = no_quotes(s);

		f = fopen(s,"r");
		if(f == NULL)
		{
			marslog(LOG_EROR|LOG_PERR,"ARCHIVE : '%s'",s);
			e = -6;
			break;
		}

		if((file_length = file_size(fileno(f))) < 0)
		{
			marslog(LOG_EROR|LOG_PERR,"Cannot compute size on '%s'",s);
			e = -1;
		}
		else
		{
			marslog(LOG_DBUG,"File '%s': %sbyte(s)",s,bytename(file_length));

			if(mars.max_filesize && file_length > mars.max_filesize)
			{
				marslog(LOG_EROR,"File '%s': %sbyte(s)",s,bytename(file_length));
				marslog(LOG_EROR,"MARS can archive files up to %sbyte(s)",bytename(mars.max_filesize));
				marslog(LOG_EROR,"Please, split source file and archive request");
				e = -1;
			}
		}

		if(f) fclose(f);

	}
	return e;
}


static err  dhs_open(void *data,request *r,request *e,int mode)
{
	dhsdata *dhs = (dhsdata*)data;
	err error = 0;
	int maxretry = 10;
	int retry = 0;

	mars.wind_requested_by_server = false;

	set_value(r,"_client_can_index_grib", "%d", mars.build_grib_index );
	set_value(r,"_client_can_index_netcdf", "%d", 1);
	set_value(r,"_accept_client_notifications", "1");
	if(mars.private_key)
	{
		if(access(mars.private_key, R_OK) == 0)
			set_value(r,"_accept_authentication_requests", "1" );
		else
			marslog(LOG_WARN|LOG_PERR,"%s",mars.private_key);
	}

	dhs->req             = clone_one_request(r);
	dhs->is_bufrfile     = feedback(r) || bias(r) || track(r);
	dhs->obs             = observation(r);
	dhs->odb             = is_odb(r);
	dhs->open            = false;
	dhs->retry_on_client         = false;
	dhs->retry_forever_on_client = false;

	if(dhs->random_hosts) {

		const char *p = dhs->random_hosts;
		char host[10240] = {0,};
		int port = dhs->port;
		int mode = 0;
		int i;

		int n = 1;
		while(*p) {
			if(*p == ' ') {
				n += 1;
			}
			p++;
		}

		if(n < 2) {
			marslog(LOG_EROR,"dhsbase random_hosts, only %d values", n);
			return -5;
		}

		int choice = rand() % n;

		p = dhs->random_hosts;
		n = 0;
		i = 0;

		while(*p) {

			char c = *p++;

			if(c == ' ') {
				n += 1;
				mode = 0;
				continue;
			}

			if(c == ':') {
				mode = 1;
				continue;
			}

			if(n == choice) {
				if(i >= sizeof(host) - 1) {
					marslog(LOG_EROR,"dhsbase random_hosts, host name too large");
					return -5;
				}
				if(mode) {
					port *= 10;
					port += c - '0';
				}
				else {
					host[i++] = c;
				}
			}

		}

		marslog(LOG_INFO,"dhsbase selecting random server %s:%d", host, port);

		dhs->port = port;
		strfree(dhs->host);
		dhs->host = strcache(host);

	}

	if(mars.readdisk_buffer > 0)
		dhs->read_disk_buffer = reserve_mem(mars.readdisk_buffer);
	else
		dhs->read_disk_buffer = NULL;

	if( fetch(r) )
		dhs->fetch = true;
	else if( EQ(r->name,"STORE") )
	{
		if((error = dhs_store_check(dhs)))
			return error;
	}
	else if( EQ(r->name,"ARCHIVE") )
	{
		const char *s = strcache(no_quotes(get_value(r,"SOURCE",0)));

		if(s == NULL)
		{
			marslog(LOG_EROR,"SOURCE parameter in missing");
			return -5;
		}

		if(strncmp(s, "mars://", 7) == 0) {

			request *setup = mars.setup;

			dhs->expect = count_fields(r);
			marslog(LOG_INFO,"Using MARS to MARS third party transfer, archiving %ld field(s)", dhs->expect);
			if(count_values(r, "SOURCE") != 1) {
				marslog(LOG_EROR,"Multi-source not supported for third party transfer");
				return -5;
			}

            while(setup && !dhs->othermars) {
                int i;
                for(i=0; i<count_values(setup,"name"); i++) {
                    if(EQ(get_value(setup, "name", i), s+7)) {
                        dhs->othermars = setup;
                    }
                }
				setup = setup->next;
			}

			if(!dhs->othermars || !EQ(get_value(dhs->othermars,"class", 0), "dhsbase")) {
				marslog(LOG_EROR, "Invalid database for third party transfer: %s", s+7);
				marslog(LOG_EROR, "Possible values are:");
				setup = mars.setup;
				while(setup) {
					if(EQ(get_value(setup, "class", 0), "dhsbase")) {
                        int i;
                        for(i=0; i<count_values(setup,"name"); i++) {
                            marslog(LOG_EROR, "  %s", get_value(setup, "name", i));
                        }
					}
					setup = setup->next;
				}
				return -5;
			}


			dhs->mars2mars = true;

		}
		else {

			if((error = sanity_check(dhs->req)) != NOERR)
				return error;

			if((error = check_file_sizes(dhs->req) != NOERR))
				return error;

			if(observation(r))
			{
				if((error = dhs_obs_check(dhs,r)))
					return error;
			}
			/* Images should be checked as other GRIB, from grib_api 1.10.0 */
			else if (image(r))
			{
				if( (error = dhs_image_check(dhs)) )
					return error;
			}
			else if (feedback(r))
			{
				if( (error = dhs_fb_check(dhs,r)) )
					return error;
			}
			else if (bias(r))
			{
				if( (error = dhs_bias_check(dhs,r)) )
					return error;
			}
			else if (track(r))
			{
				if( (error = dhs_track_check(dhs,r)) )
					return error;
			}
			else if (is_odb(r))
			{
				if( (error = dhs_odb_check(dhs,r)) )
					return error;
			}
            else
            {
#if mars_client_HAVE_NETCDF
                if(source_is_netcdf(r)) {
                    dhs->netcdf = true;
                    if((error = dhs_netcdf_check(dhs,r)) != NOERR)
                        return error;
                }
                else
                {
                     if((error = dhs_grib_check(dhs,r)) != NOERR)
                        return error;
                }
#else
                if((error = dhs_grib_check(dhs,r)) != NOERR)
                    return error;
#endif
            }


			if(getenv("MARS_NO_ARCH"))
			{
				marslog(LOG_WARN,"'MARS_NO_ARCH' selected. Skipping archive");
				dhs->finished = true;
				return 0;
			}

			if(!mars.fields_are_ok)
			{
				FILE *f = mail_open(mars.dhsmail,"Mars ARCHIVE failure (dhsbase)");
				mail_request(f,"MARS Request:",r);
				mail_request(f,"MARS Environment:",e);
				mail_close(f);

				marslog(LOG_EROR,"Some of the fields are rejected by the new mars server");
				return -1;
			}

		}

	}

    while((error = dhs_send(dhs,r,e)) != 0 && (retry++ < maxretry))
	{
		marslog(LOG_WARN,"Error %d received while calling server on '%s', retrying.",error,dhs->host);
		dhs_close(dhs);
		dhs->req         = clone_one_request(r);
		sleep(10);
	}

	if(!error)
		dhs->open = true;

	return error;
}

static err  dhs_close(void *data)
{
	dhsdata *dhs = (dhsdata*)data;
	err e = 0;
	int retrieve = EQ(dhs->req->name,"RETRIEVE") && (dhs->total_toread != 0);

#if 1
	int retry = 0;
	int maxretry = 5;
	int save_time = dhs->timeout;

	dhs->timeout = 1;

	if(dhs->f)       socket_file_close(dhs->f);
	if(dhs->csoc>=0) socket_close(dhs->csoc);


	if(dhs->open)
		while(!(dhs->finished) && (retry++ < maxretry))
		{
			if(timed_dhs_wait(dhs) != NOERR)
				dhs->finished = true;
		}
	if(dhs->asoc>=0) socket_close(dhs->asoc);

	dhs->timeout = save_time;

#else
	if(dhs->csoc == -1 && dhs->asoc == -1 && dhs->f == NULL)
		timed_dhs_wait(dhs);

	if(dhs->f)
	{
		socket_file_close(dhs->f);
	}
	if(dhs->open)
	{
		if(dhs->csoc>=0) socket_close(dhs->csoc);
		if(dhs->asoc>=0) socket_close(dhs->asoc);
	}
#endif
	wind_free(dhs->u_v);
	free_all_requests(dhs->req);
	free_all_requests(dhs->metadata);
	FREE(dhs->reclen);

	marslog(LOG_DBUG,"%d messages received from '%s'",dhs->count,dhs->host);

	if(dhs->total_read != dhs->total_toread)
		marslog(LOG_WARN,"Got %lld out of %lld bytes",
				dhs->total_read,dhs->total_toread);

	/* mars_field_index_print( dhs->head_idx ); */
	mars_field_index_free( dhs->head_idx );

	dhs->csoc   = dhs->asoc = -1;
	dhs->f      = NULL;
	dhs->u_v    = NULL;
	dhs->total_read = dhs->total_toread = 0;
	dhs->req    = NULL;
	dhs->total_tosend = 0;

	if(dhs->read_disk_buffer)
		release_mem(dhs->read_disk_buffer);

	if(mars.wind_requested_by_server && retrieve) {
		marslog(LOG_EXIT,"Convertion from VO/D to U/V was requested by server, but not done on client");
	}

	return e;
}

static err  dhs_read(void *data,request *r,void *buffer,long *length)
{
	dhsdata *dhs = (dhsdata*)data;
	err e = 0;

	/* Timers */
	timer *net_time  = get_timer("Read from network","transfertime",true);

	for(;;)
	{
		/* Server send OK, but not all data has been received */
		if(dhs->finished)
		{
			if(dhs->total_read < dhs->total_toread)
				return -2;

			return EOF;
		}

		if(dhs->f) {

			if(dhs->fetch)
			{
				long len;

				while(dhs->reclen[dhs->index] == 0)
				{
					dhs->current = dhs->current->next;
					if(dhs->current == NULL)
						return EOF;
					dhs->index++;
					dhs->count++;
				}

				len = MIN(*length,dhs->reclen[dhs->index]);

				if(timed_fread((char *) buffer, 1, len, dhs->f, net_time) != len)
				{
					marslog(LOG_EROR|LOG_PERR,"fetch");
					return RETRY_ERR;
				}

				*length                  = len;
				dhs->reclen[dhs->index] -= len;
				dhs->total_read         += len;

				if(r) reqcpy(r,dhs->current);
			}
			else if(dhs->obs)
			{
				e = timed_readany(dhs->f, (char *)buffer, length, net_time);
				if(e == 0)
				{
					dhs->count++;
					dhs->total_read += *length;
				}
			}
			else if (dhs->is_bufrfile)
			{
				e = 0;
				if (dhs->data_size ==0)
				{
					e = read_metadata(dhs);
					if(r) reqcpy(r,dhs->metadata);
				}
				if(e == 0)
				{
					e = timed_readany(dhs->f, (char *) buffer, length, net_time);
					if(e == 0)
					{
						dhs->count++;
						dhs->total_read += *length;
						dhs->data_size  -= *length;
					}
				}
			}
			else if (dhs->netcdf) {
				printf("NetCDF size %ld\n", *length);
				e = timed_readany(dhs->f, (char *) buffer, length, net_time);
				printf("NetCDF size %ld err=%d\n", *length, e);
				if(e == 0)
				{
					dhs->count++;
					dhs->total_read += *length;
					dhs->data_size  -= *length;
				}
			}
			else if (dhs->odb)
			{
#ifdef ODB_SUPPORT
				{
					const char *target = no_quotes(mars.webmars_target ? mars.webmars_target : get_value(dhs->req, "TARGET", 0));
					const char *filter = get_value(dhs->req, "FILTER", 0);
					FILE *out = NULL;
					if(target)
					{
						out = fopen(target, target_open_mode(target));
						if(! out)
						{
							marslog(LOG_EROR|LOG_PERR, "fopen(%s)", target);
							return -2;
						}

						dhs->total_read = odb_filter(filter, dhs->f, out, dhs->total_toread);
                    	e = fclose(out);
                    	if(e == 0)
                        	e = EOF;
						/* If odb_filter fails, do not retry */
						if(dhs->total_read == -1) {
							e = ODB_ERROR;
						}
						else {
							mars.retrieve_size += dhs->total_read;
							if(mars.max_retrieve_size && mars.retrieve_size > mars.max_retrieve_size) {
								marslog(LOG_EROR, "Maximum retrieve size %s reached. Please split your request.", bytename(mars.max_retrieve_size));
								e = -2;
							}
						}
					}
					else {
						marslog(LOG_EROR, "No target");
						e = -2;
					}

				}
#else
				marslog(LOG_EROR,"This MARS client doesn't support ODB");
				marslog(LOG_EROR,"Please, contact the MARS team");
				e = -1;
#endif
			}
			else
			{
				e =  timed_wind_next(dhs->u_v, dhs->f, (char *) buffer, length, net_time);
				if(e == 0)
				{
					dhs->count++;
					if(r)
					{
						grib_to_request(r, (char *)buffer, *length);
						set_value(r,"_ORIGINAL_FIELD","1");
					}
				}

			}

			if(e == POSTPROC_ERROR)
				return e;

			if(getenv("MARS_STOP_TRANSFER_RANDOM") != 0)
			{
				static int count = 0;
				marslog(LOG_INFO,"read number %d",count);
				if(count > 11)
				{
					marslog(LOG_WARN,"Forcing failure, count %d",count);
					e = RETRY_ERR;
					count = 0;
				}
				count++;
			}
			if(e && (dhs->total_read < dhs->total_toread))
			{
				marslog(LOG_DBUG,"ftell: %d",ftell(dhs->f));
				marslog(LOG_WARN,"Transfer interrupted, got %lld out of %lld bytes err=%d", dhs->total_read,dhs->total_toread,e);
				if(e == NOT_FOUND_7777)
				{
					marslog(LOG_WARN,"Group 7777 not found at the end of GRIB message");
				}
				if(e == ODB_ERROR)
				{
					marslog(LOG_WARN,"Error decoding ODB data");
				}
				else
				{
					e = RETRY_ERR;
				}
				socket_file_close(dhs->f);
				dhs->f = NULL;
				dhs->total_read = 0;
			}

			return e;
		}
		e =  timed_dhs_wait(dhs);
		if(e) break;
	}

	return e;
}

static err  dhs_write(void *data,request *r,void *buffer,long *length)
{
	return -1;
}

static err  dhs_archive(void *data,request *r)
{
	dhsdata *dhs = (dhsdata*)data;
	err e = 0;
	while(!dhs->finished)
	{
		e = timed_dhs_wait(dhs);
		if(e) return e;
	}
	return 0;
}

static err  dhs_cntl(void *data,int code,void *param,int size)
{
	dhsdata *dhs = (dhsdata*)data;
	err ret = -120;

	marslog(LOG_DBUG,"Control in dhsbase, code %d",code);

	switch(code)
	{
		case CNTL_FLUSH:
		case CNTL_LIST:
		case CNTL_STAGE:
		case CNTL_REGISTER:
		case CNTL_ATTACH:
		case CNTL_REMOVE:
		case CNTL_STORE:
		case CNTL_FETCH:
			while(!dhs->finished)
			{
				err e = timed_dhs_wait(dhs);
				if(e) return e;
			}
			ret = 0;
			marslog(LOG_DBUG,"Remove finished %d, return of %d", dhs->finished, ret);
			break;

		default:
			marslog(LOG_WARN,"Unknown control %d on dhsbase",code);
			return -1;
			/*NOTREACHED*/
			break;

	}
	marslog(LOG_DBUG,"Control finished in dhsbase with %d",ret);
	return ret;
}

static boolean dhs_check(void *data,request *r)
{
	return true;
}

static err  dhs_validate(void *data,request* r,request* env,int mode)
{
	return 0;
}
