/*
 * © Copyright 1996-2012 ECMWF.
 *
 * This software is licensed under the terms of the Apache Licence Version 2.0
 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
 * In applying this licence, ECMWF does not waive the privileges and immunities
 * granted to it by virtue of its status as an intergovernmental organisation nor
 * does it submit to any jurisdiction.
 */

#include "mars.h"

extern base_class *targetbase;

static int visited = 0;

boolean enough(request *r, int count)
{
	static int  need   = 0;
	static boolean init   = false;
	static boolean fields = false;
	static request *p = NULL;

	if(p != r || !init)
	{
		p = r;
		need = count_fields(r);
		if(is_bufr(r))
			fields = false;
		else if(fetch(r))
			fields = false;
		else if(is_odb(r))
			fields = false;
		else
			fields = true;
		init = true;

	}

	if(!fields && count>=need && count > 0)
		return true;

    if( need != 0 && count == need)
        return true;

	/* In case of ALL */
	if( need == 0 && count != 0)
		return true;

	return false;
}

request *findbase(const char *name)
{
	request *t = 0;
	request *s = mars.setup;

	if(!name) return 0;

	while(s && !t)
	{
		if(EQ(s->name,"database"))
		{
			const char *p ; int i = 0;

			while( (p = get_value(s,"name",i++)) && !t )
					if(strcasecmp(p,name) == 0)
						t = s;
		}
		s = s->next;
	}

	if(t == 0)
	{
		request *s = mars.setup;

		marslog(LOG_EROR,"The database '%s' is not known",name);
		marslog(LOG_EROR,"The valid values are: ");

		while(s)
		{
			if(EQ(s->name,"database"))
			{
				const char *p ; int i = 0;
				while( (p = get_value(s,"name",i++)) )
					marslog(LOG_INFO,"%s",p);
			}
			s = s->next;
		}
	}

	return t;
}

database *openbase(request *s,request *r,const char **name,request **cache,int  mode)
{
	request *env = get_environ();
	const char    *c   = get_value(s,"class",0);
	const char    *n   = get_value(s,"name" ,0);
	const char    *x   = get_value(s,"cache",0);
	const char    *a   = get_value(s,"active",0);
	database *base = 0;

	if(a && EQ(a,"false")) {
		marslog(LOG_EROR,"Database %s is not active",n);
		return NULL;
	}

	*name  = n;
    *cache = NULL;

	qmonitor("open %s for %s",n,mode == READ_MODE?"read":"write");
	base  = database_open(base_class_by_name(c),n,r,env,mode);


	if(base && !database_check(base,r))
	{
		database_close(base);
		qmonitor("close %s",n);
		return NULL;
	}

	*cache = findbase(x);

	return base;
}


static err retrieve(request *r, request* setup, database *target, int *cnt, postproc pproc)
{
	request      *w;
	int		     ret    = NOERR;
	long         length;
	boolean      done = false;
	const char   *name;
	database     *base  = NULL;
	database     *cache = NULL;
	request      *cachesetup = 0;
	const char   *cachename = 0;
	boolean      fields;
	long64       total        = 0;
	long64       total_target = 0;

	*cnt = 0;

	/* Structure to return 2 or more fields from post-processing.
	   Most times (ie, scalar post-processing), only 1 field is returned */
	static ppbuffer_t *pp       = NULL;
	static int        nbuffers  = 2;
	static long       buflen    = 0;
	       int        i         = 0;
	static boolean    inform    = true;

	buflen = mars.readany_buffer_size;

	if(ppestimate() > buflen)
	{
		buflen = ppestimate() + 1024*1024;
		if(inform)
		{
			marslog(LOG_WARN,"Using large buffer for post-processing (%sbytes)",bytename(buflen));
			inform = false;
		}
	}

	if(pp == NULL)
	{
		pp = reserve_mem(sizeof(ppbuffer_t)*nbuffers);
		for(i = 0; i<nbuffers; ++i)
		{
			pp[i].buffer=NULL;
			pp[i].buflen=buflen;
			pp[i].outlen=0;
		}
	}

	for(i=0; i<nbuffers; ++i)
	{
		if(pp[i].buflen < buflen)
		{
			if(pp[i].buffer) release_mem(pp[i].buffer);
			pp[i].buflen = buflen;
			pp[i].buffer = NULL;
		}

		if(pp[i].buffer == NULL)
		{
			marslog(LOG_DBUG,"Work buffer is %d",pp[i].buflen);
			pp[i].buffer = reserve_mem(pp[i].buflen);
		}
	}

	base = openbase(setup,r,&name,&cachesetup,READ_MODE);

    if(!base) {
        return TOO_SHORT_ERR;
    }

	/* rewind target */
	database_control(target,CNTL_REWIND,NULL,0);

	visited++;

	ret  = NOERR;

	if(is_bufr(r))
	{
		w      = NULL;
		cache  = NULL;
		fields = false;

		/* For AI or AF we receive metadata request from DHS */
		if(feedback(r))
			w = clone_one_request(r);
	}
	else if(fetch(r))
	{
		fields = false;
		w      = clone_one_request(r);
		cache  = NULL;
	}
	else if(is_odb(r))
	{
		fields = false;
		w      = clone_one_request(r);
		cache  = NULL;
	}
	else
	{
		fields = true;
		w      = clone_one_request(r);
		cache  = NULL;
	}

	while(!done)
	{
		long   len = pp[0].buflen;
		char   *buffer = pp[0].buffer;
		long   n       = 0;

		length = pp[0].buflen;

		ret = database_read(base,w,buffer,&length);

		switch(ret)
		{
		case NOERR:

			total += length;

			/* open the cache if needed */
			if(cachesetup && !cache)
			{

				if(!is_bufr(r))
				{
					request *dummy;
					cache = openbase(cachesetup,r,&cachename, &dummy,WRITE_MODE);
				}

				/* don't try to open more than once */

				cachesetup = NULL;
			}

			/* write to cache before pproc */

			if(cache) {
				long len = length;
				if(database_write(cache,w,buffer,&len) != 0)
				{
					database_close(cache);
					cache = NULL;
				}
			}

			pp[0].inlen  = length;
			pp[0].outlen = pp[0].buflen;
			n = 0;
			ret = pproc(pp,&n);
			length = pp[0].outlen;
			buffer = pp[0].buffer;

			if(ret == 0)
			{
				int i = 0;
				for(i = 0; i<n; ++i)
				{
					long written = pp[i].outlen;
					if(fields)
						grib_to_request(w,pp[i].buffer,written);
					ret = database_write(target,w,pp[i].buffer,&written);
					if(ret == NOERR)
						(*cnt)++;
					else
						done = true;
					total_target += written;

					mars.retrieve_size += written;
					if(mars.max_retrieve_size && mars.retrieve_size > mars.max_retrieve_size) {
						marslog(LOG_EROR, "Maximum retrieve size %s reached. Please split your request.", bytename(mars.max_retrieve_size));
						ret = QUOTA_ERROR;
						done = true;
					}
				}
			}
			else
			{
				if(ret == BUF_TO_SMALL)
				{
					marslog(LOG_WARN,
						"Buffer is too small (%d bytes, %d needed. Retrying)",
						buflen,length);
					release_mem(buffer);
					buffer = NULL;
					buflen = length;
				}

				done = true;
			}
			break;

		case EOF:
			ret  = 0;
			done = true;
			break;

		case BUF_TO_SMALL:
			marslog(LOG_WARN,
			    "Buffer is too small (%d bytes, %d needed. Retrying)",
			    buflen,length);
			release_mem(buffer);
			buffer = NULL;
			buflen = length;
			done = true;
			break;


		case RETRY_ERR:
			marslog(LOG_WARN,"Retrying..");
			(*cnt) = 0;
			database_control(target,CNTL_REWIND,NULL,0);
			done = true;
			break;

		case NOT_FOUND_7777:
			marslog(LOG_EROR,"Group 7777 not found at the end of GRIB message");
			database_control(target,CNTL_ERROR,&ret,sizeof(ret));
			done = true;
			break;

		case POSTPROC_ERROR:
			marslog(LOG_EROR,"Error in interpolation. Exit");
			database_control(target,CNTL_ERROR,&ret,sizeof(ret));
			done = true;
			break;

		default:
			marslog(LOG_DBUG,"Get error %d while reading data",ret);
			done = true;
			break;
		}
	}

	/* ppdone(); */

	for(i=0; i<nbuffers; ++i)
	{
		if(pp[i].buffer) release_mem(pp[i].buffer);
		pp[i].buffer = NULL;
	}

	free_all_requests(w);

	if(ret == NOERR)
	{

		if( enough(r,*cnt) )
		{
			if(fetch(r))
			{
				/* some message here */
			}
			if(observation(r) || track(r))
			{
				int in,out;
				ppcount(&in,&out);
				marslog(LOG_INFO,"%d reports retrieved from '%s'",in,name);
				marslog(LOG_INFO,"%d reports left after filtering",out);
				log_statistics("reports","%d",out);
			}
			else if(feedback(r) || bias(r))
			{
				marslog(LOG_INFO,"%d reports retrieved from '%s'",*cnt,name);
				log_statistics("reports","%d",*cnt);
			}
			else if(is_odb(r))
			{
				marslog(LOG_INFO,"%d ODB message%s retrieved from '%s'",*cnt,*cnt==1?"":"s",name);
				log_statistics("odbs","%d",*cnt);
			}
			else
			{
				marslog(LOG_INFO,"%d field%s retrieved from '%s'",*cnt,*cnt==1?"":"s",name);
				log_statistics("fields","%d",*cnt);
			}

			log_statistics("database","%s",name);
			log_statistics("bytes","%lld",total);
			log_statistics("written","%lld",total_target);
		}
		else
			ret = TOO_SHORT_ERR; /* we don't have enough data */

	}

	if(base)  {
		database_close(base);
		qmonitor("close %s",name);
	}
	if(cache) {
		database_close(cache);
		qmonitor("close %s",cachename);
	}

	return ret;
}

void mars_task(void)
{
}


static err visit_database(request *r,request *s, database *target,int need,int* cnt, postproc pproc)
{
	int count = 0;
	err e = 0;
	char buf[1024];
	timer *t;
	int retry = 0;
	int slp = 2;
	const char *dbname = get_value(s,"name",0);

	sprintf(buf,"Visiting %s",dbname);
	t=get_timer(buf,NULL,true);

	timer_start(t);
	do {
		marslog(LOG_DBUG,"Visiting database '%s'",dbname);
		e  = retrieve(r,s,target, &count, pproc);
		if(e == RETRY_ERR)
		{
			if(retry<mars.maxretries)
			{
				retry++;
				marslog(LOG_WARN,"Sleeping %d minutes",slp);
				sleep(slp*60);
			}
			else
			{
				e = -2;
				marslog(LOG_WARN,"Giving up after %d retries",retry);
			}
		}

		if(e == RETRY_FOREVER_ERR)
		{
			if(slp<20)
				slp += 2;
			marslog(LOG_WARN,"Sleeping %d minutes",slp);
			sleep(slp*60);
		}

	} while(e == BUF_TO_SMALL || e == RETRY_ERR || e == RETRY_FOREVER_ERR);
	timer_stop(t,0);

	*cnt = *cnt + count;
	if(need != *cnt && count > 0) {
		marslog(LOG_WARN,"Visiting database %s : expected %d, got %d", dbname, need, count);
	}

	return e;
}

err handle_retrieve(request *r,void *data)
{
	int need = count_fields(r);
	int cnt  = 0;
	database *target;
	err e,f;
	request *u   = r->next;
	request *env = get_environ();
	request *s   = mars.setup;
	int bases = count_values(r,"DATABASE");
	postproc  pproc;
	boolean isbufr = is_bufr(r);
	boolean fields = !isbufr && !fetch(r) && !image(r) && !is_odb(r);

	const char *p      = get_value(r,"EXPECT",0);
	boolean    expect     = p?atol(p):0;
	boolean  expect_any   = (p && (atol(p) == 0));

	visited = 0;

	if(expect_any)
	{
		marslog(LOG_INFO,"Requesting any number of %s (request describes %d)",fields?"fields":"observations",count_fields_in_request(r));
	}
	else if(fields)
	{
		if(all_is_used(r))
		{
			marslog(LOG_WARN,"Cannot compute number of fields from request.");
			marslog(LOG_WARN,"Try to avoid the use of the value 'ALL'");
		}
		else
		{
			if(need == 0)
			{
				marslog(LOG_WARN,"The request does not represent any field");
				marslog(LOG_WARN,"Inform the MARS group if you think it is unusual");
			}
			else
			{
				marslog(LOG_INFO,"Requesting %d field%s",need,need==1?"":"s");
			}
		}
	}

	e = -1;
	target = database_open(targetbase,NULL,r,env,WRITE_MODE);
	if(target == NULL)
		return -1;


	if(isbufr && mars.bufr_empty_target)
	{
		/* For obs, make sure we have at least an empty file */
		long length = 0;
		database_write(target,0,0,&length);
	}

	r->next = NULL; /* avoid sending to much on the net */

	ppinit(r,&pproc);

	if(bases != 0)
	{
		int i;
		for(i=0;i<bases;i++)
		{
			const char *b = get_value(r,"DATABASE",i);
			request *t    = findbase(b);
			if(t)
			{
				int last = i==(bases-1);
				boolean save = mars.infomissing;

				if(last) mars.infomissing=true;
				e = visit_database(r,t,target,need,&cnt,pproc);
				mars.infomissing=save;
				if(e == NOERR) break;
				if(e == QUOTA_ERROR) break;
			}
		}
	}
	else
	{
		while(s)
		{
			if(EQ(s->name,"database"))
			{
				const char *p = get_value(s,"visit",0);
				if(p == 0 || !EQ(p,"false"))
				{
					e = visit_database(r,s,target,need,&cnt,pproc);
					if(e == NOERR) break;
					if(e == QUOTA_ERROR) break;
				}
			}

			s = s->next;
		}
	}

	r->next = u;

	ppdone();

	if((e == EOF) || (e == TOO_SHORT_ERR) )
	{

		if(!enough(r,cnt) && (is_bufr(r) || is_odb(r)) && visited > 0)
		{
			e = NOERR;
		}
		else if(!enough(r,cnt))
		{
			const char* expect = get_value(r,"EXPECT",0);
			if(expect == NULL || atol(expect) != 0)
			{
				if(all_is_used(r))
					marslog(LOG_EROR,"MARS considers %d fields retrieved are not enough",cnt);
				else
					marslog(LOG_EROR,"Expected %d, got %d.", need,cnt);
				e = -1;
			}
			else {
				if(e == TOO_SHORT_ERR)
				{
					long length = 0;
					marslog(LOG_WARN,"No data, but EXPECT was provided");
					database_write(target,0,0,&length);
					e = 0;
				}
			}
		}
	}

	if(e)
		database_control(target,CNTL_ERROR,&e,sizeof(e));

	f = database_close(target);
	if(f && !e) e = f;

	if(visited == 0)
	{
		marslog(LOG_EROR,"Could not find a MARS database to perform request");
		e = -2;
	}

	if(e) marslog(LOG_EROR,"Request failed");

	return e;

}
