/*
 * © Copyright 1996-2012 ECMWF.
 * 
 * This software is licensed under the terms of the Apache Licence Version 2.0
 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 
 * In applying this licence, ECMWF does not waive the privileges and immunities 
 * granted to it by virtue of its status as an intergovernmental organisation nor
 * does it submit to any jurisdiction.
 */

#include "mars.h"

/* Checks that all single values contained in request 'a' 
   are in request 'b' */
static err fix_check(const request *grib,const request *req, int cnt)
{
	/* Should have everything which is not used for field ordering */

	static struct {
		const char *name;     /* Name of the MARS parameter to check */
		boolean     always;   /* If consistency grib<->request must always exist */
	} checks [] = {

		{ "CLASS",    true, },
		{ "TYPE",     true, },
		{ "STREAM",   true, },
		{ "LEVTYPE",  true, },
		{ "_GRID_EW", false, },
		{ "_GRID_NS", false, },
		{ "ORIGIN",   true, },

		{ "PRODUCT",   true, },
		{ "SECTION",   true, },
		{ "METHOD",    true, },
		{ "SYSTEM",    true, },

		/* Already in field_order */
		/* "EXPVER", */
		/* "DATE", */
		/* "TIME", */
		/* "REFERENCE", */
		/* "STEP", */
		/* "DOMAIN", */
		/* "DIAGNOSTIC", */
		/* "ITERATION", */
		/* "NUMBER", */
		/* "LEVELIST", */
		/* "PARAM", */
		/* "FREQUENCY", */
		/* "DIRECTION", */

	};

	int j = 0;
	if(mars.pseudogrib)
		return NOERR;

	if(grib && req)
	{
		while(j < NUMBER(checks))
		{
			int i = 0;
			const char *p = checks[j].name;
			const char *s = get_value(grib,p,i);
			while(s)
			{
				if( !value_exist(req,p,s) )
				{
					const char *str  = get_value(grib,"STREAM",0);
					const char *clss = get_value(grib,"CLASS",0);
					boolean     grid = ( strcmp(p,"_GRID_EW")==0 || strcmp(p,"_GRID_NS")==0);
				    /* In ERA 40 they do not care about the grid */
					boolean     supd = ( strcmp(str,"SD") == 0 && strcmp(str,"E4") == 0);

					if( supd || (!supd && !grid))
					{
						marslog(LOG_EROR,"Grib %d contains %s = %s. Grib description follows:",cnt,p,s);
						print_all_requests(grib);
						return -6;
					}
				}
				s = get_value(grib,p,++i);
			}
			j++;
		}
	}
	return NOERR;
}

static long compute_total(request *r)
{
    long n      = 1;
    parameter *p = r->params;
    while(p)
    {
        n *= count_values(r,p->name);
        p = p->next;
    }
    return n;
}

static const char* fast_value(const request *r,const char* name)
{
	if(r && name)
	{
		parameter *p = r->params;
		while(p)
		{
			if(p->name == name) return p->values?p->values->name:NULL;
			p = p->next;
		}
	}
	return NULL;
}

static int cmp_no_hidden(const request *a,const request *b)
{
	int n;

	static const char* repres   = 0;

	if(repres == 0)
		repres   = strcache("REPRES");

	if(a && b)
	{
		parameter *p = a->params;

		while(p)
		{
			const char *s = fast_value(a,p->name);
			const char *t = fast_value(b,p->name);

			if(s && t && (*p->name!='_') && p->name != repres )
			{
				
				n = (s != t);
				if(n != 0 && is_number(s)) n = atof(s) - atof(t);
				if(n) {
					if(mars.debug)
					marslog(LOG_DBUG,"Compare failed: %s -> %s <> %s",p->name,s,t);
					return n;
				}
			}

			p = p->next;
		}
	}
	return 0;
}

static int find_order(request **v,request *g)
{
	request *w = 0;
	request *u = *v;

	while(u)
	{
		if(cmp_no_hidden(u,g) == 0)
		{
			int order = u->order;
			if(w != 0)
				w->next = u->next;
			else
				*v = u->next;

#if 1
			/* try to reorganise the list so next time we scan the minumum */

			if(w != 0)
			{
				request *z = w->next;
				request *p = 0;

				while(z) { p = z; z = z->next; }


				if(p != 0)
				{
					p->next = *v;
					*v      = w->next;
					w->next = 0;
				}

			}
#endif

			u->next = NULL;
			free_one_request(u);

			return order;
		}
		w = u;
		u = u->next;
	}

	return -1;
}

int find_long(long* p,int count,long search)
{
	int i;
	for (i = 0; i < count; i++)
		if (p[i] == search)
			return TRUE;
	return FALSE;
}

static err init_subtype_tables(request* r,
							char* subtype_group,
							char* known_subtypes)
{
	memset((void*) subtype_group,0,256);
	memset((void*) known_subtypes,0,256);

	if (count_values(r,"OBSGROUP") == 1)
	{
		request*	dummy_request;
		request*	expanded_request;
		int			count;
		int			i;

		/*----------------------------------------------------------------*/
		/* We expand a dummy request with the obstype value to get all    */
		/* the subtypes in the obstypes. We put 2 into every entry in the */
		/* table                                                          */
		/*----------------------------------------------------------------*/
		count = count_values(r,"OBSTYPE");
		for (i = 0; i < count; i++)
		{
			long st = atol(get_value(r,"OBSTYPE",i));
			if (st < 0 || st > 255)
			{
				marslog(LOG_EROR,"Subtype %d is greater than 255. "\
				"If this is a valid subtype, please contact the MARS group",
						st);
				return -6;
			}
			subtype_group[st] = 2;
		}

		/*----------------------------------------------------------------*/
		/* We expand a dummy request with the obsgroup value to get all   */
		/* the subtypes in the obsgroup. Whenever an entry contains 2     */
		/* it means that it was in the OBSTYPE, we put 1 in it meaning    */
		/* it is also in the obsgroup.                                    */
		/*----------------------------------------------------------------*/
		dummy_request = empty_request("ARCHIVE");
		set_value(dummy_request,"TYPE","OB");
		set_value(dummy_request,"OBSTYPE",get_value(r,"OBSGROUP",0));
		expanded_request = expand_mars_request(dummy_request);
		count = count_values(expanded_request,"OBSTYPE");

		for (i = 0; i < count; i++)
		{
			long st = atol(get_value(expanded_request,"OBSTYPE",i));
			if (st < 0 || st > 255)
			{
				marslog(LOG_EROR,"Subtype %d is greater than 255. "\
				"If this is a valid subtype, please contact the MARS group",
						st);
				return -6;
			}
			if (subtype_group[st] == 2)
				subtype_group[st] = 1;
		}
		free_one_request(expanded_request);

		/*----------------------------------------------------------------*/
		/* After this, if there are any 2s left in the table, then        */
		/* OBSTYPE is not a subset of OBSGROUP.                           */
		/*----------------------------------------------------------------*/
		for (i = 0; i < 256; i++)
		{
			if (subtype_group[i] == 2)
			{
				marslog(LOG_EROR,"OBSTYPE %d is not a subset of OBSGROUP",i);
				marslog(LOG_EROR,"Add it to OBSGROUP %s in mars.def",get_value(r,"OBSGROUP",0));
				return -6;
			}
		}

		/*----------------------------------------------------------------*/
		/* We expand a dummy request with the all the known subtypes      */
		/* (KNOWN).														  */
		/*----------------------------------------------------------------*/
		set_value(dummy_request,"OBSTYPE","KNOWN");
		expanded_request = expand_mars_request(dummy_request);
		count = count_values(expanded_request,"OBSTYPE");
		for (i = 0; i < count; i++)
		{
			long st = atol(get_value(expanded_request,"OBSTYPE",i));
			if (st < 0 || st > 255)
			{
				marslog(LOG_EROR,"Subtype %d is greater than 255. "\
				"If you think this is a valid subtype, please contact the MARS team",
						st);
				return -6;
			}
			known_subtypes[st] = TRUE;
		}

		free_one_request(dummy_request);
		free_one_request(expanded_request);
	}
	else
	{
		marslog(LOG_EROR,"Archiving requires only one OBSGROUP value");	
		return -6;	
	}

	return 0;
}

/* assumes only one date, one time, one range */
static void get_archive_period(request* r,time_interval* t)
{
	long time = atol(get_value(r,"TIME",0));
	datetime begin = date_time_2_datetime(
					mars_date_to_julian(atol(get_value(r,"DATE",0))),
					time / 100,
					time % 100,
					0);
	long range = atol(get_value(r,"RANGE",0));

	/* range is in minutes */
	init_time_interval(t,begin,range * 60 + 59);
}

static err dhs_fb_check(dhsdata* dhs,request* original_req)
{
	marslog(LOG_INFO,"No checking is carried out on extraction/feedback data");

	return 0;
}

static err dhs_obs_check(dhsdata* dhs,request* original_req)
{
	long 			length = 65536;
	char 			buffer[65536];
	int  			cnt = 0;
	err 			e    = 0;
	FILE 			*f;
	char 			buf[80];
	int				q = 0;
	const 			char* s;
	time_interval	archive_period;
	long			err_count = 0;

	long64			file_length = 0;

	/*----------------------------------------------------------------*/
	/* We want to make sure that data included in the file to be	  */
	/* archived complies with the description given in the request.	  */
	/* 1. the subtypes found in the BUFR files have to be known in    */
	/*    mars language. Otherwise failure.                           */
	/* 2. the subtypes in the OBSTYPE parameter have to be the same   */
	/*    as in the OBSGROUP or a subset of it. Otherwise failure.	  */
	/*	  This ensures that what is archived is what people who are   */
	/*    archiving think is being archived.                          */
	/* 3. There sould only be one date in the request.				  */
	/* 4. There sould only be one time in the request.				  */
	/* 5. There sould only be one range in the request.				  */
	/* 6. Messages which are not in the [date.time,date.time+range]   */
	/*    are not accepted, failure.                                  */
	/* 7. I should have started with this: any problem in the BUFR    */
	/*    file (key missing, wrong length etc..) is a failure.        */
	/*                                                                */
	/* Any other idea?                                                */
	/*----------------------------------------------------------------*/

	/*----------------------------------------------------------------*/
	/* We check subtypes assuming a subtype is less than 256 which is */
	/* currently the case. This speeds up dramatically data processing*/
	/* since we use a table.										  */
	/* If subtypes were codes using more than a byte in the future,   */
	/* this code should be changed (dream on).                        */
	/*                                                                */
	/* Ths subtype_group array contains all the subtypes in the       */
	/* OBSGROUP less the subtypes which are not in the OBSTYPE.       */
	/* Any discrepancy between those cause init_subtype_tables to     */
	/* return an error.                                               */
	/*----------------------------------------------------------------*/
	char		subtype_group[256];
	char		known_subtypes[256];

	e = init_subtype_tables(dhs->req,subtype_group,known_subtypes);
	if (e != 0)
		return e;

	if (count_values(dhs->req,"DATE") > 1)
	{
		marslog(LOG_EROR,"Only one date must be specified for archiving data");
		return -6;
	}
	if (count_values(dhs->req,"TIME") > 1)
	{
		marslog(LOG_EROR,"Only one time must be specified for archiving data");
		return -6;
	}

	get_archive_period(dhs->req,&archive_period);

	dhs->total_tosend = 0;

	start_timer();

	while(e == 0 && (s = get_value(dhs->req,"SOURCE",q++)) != NULL)
	{
		s = no_quotes(s);

		marslog(LOG_INFO,"Scanning and analysing %s",s);

		f = fopen(s,"r");
		if(f == NULL)
		{
			marslog(LOG_EROR|LOG_PERR,"ARCHIVE : '%s'",s);
			e = -6;
			break;
		}

		if(fseek(f,0,SEEK_END) != 0)
		{
			marslog(LOG_EROR|LOG_PERR,"fseek(%s)",s);
			e = -2;
		}

		file_length = ftell(f);
		if (dhs->total_tosend == 0)
			set_value_long64(original_req,"_length",file_length);
		else
			add_value_long64(original_req,"_length",file_length);
		dhs->total_tosend += file_length;

		if(fseek(f,0,SEEK_SET) != 0)
		{
			marslog(LOG_EROR|LOG_PERR,"fseek(%s)",s);
			e = -2;
		}

		length = sizeof(buffer);
		while( !e && (e = _readany(f,buffer,&length)) == NOERR || 
			(e == BUF_TO_SMALL))
		{
			packed_key k;
			packed_key* keyptr=&k;
			int keyOk;

			keyOk = get_packed_key(buffer,&k);
			if (keyOk)
				keyOk = verify_bufr_key(buffer,length,&k);
			if (keyOk)
			{	
				datetime obs_date;
				long subtype;
				const char *obsgroup = get_value(dhs->req,"OBSGROUP",0);
				/*------------------------------------------------------*/
				/* if conventional data, IDENT is checked.				*/
				/* since subtypes 82 to 88 are in the conventional group*/
				/* and these are satellite data, we skip them.			*/
				/*------------------------------------------------------*/
#if 0
				if (EQ(obsgroup,"CONVENTIONAL"))
				{
					if (k.header.subtype < 82 || k.header.subtype > 88)
						if (KEY_IDENT(&k)[0] == ' ')
						{
							 marslog(LOG_EROR,
								"IDENT of report %d, subtype %d is left justified.",
								cnt+1,k.header.subtype);
							 err_count++;
						}
				}
#endif
				/*------------------------------------------------------*/
				/* check subtypes to be archived						*/
				/*------------------------------------------------------*/
				subtype = k.header.subtype;
				if (subtype < 0 || subtype > 255)
				{
					marslog(LOG_EROR,"Subtype %d is greater than 255. "\
					"If this is a valid subtype, please contact the MARS team",
							subtype);
					err_count++;
				} 
				else if (!known_subtypes[subtype])
				{
					marslog(LOG_EROR,"Found report %d with subtype %d which "\
					"is unknown",cnt+1,subtype);
					err_count++;
					marslog(LOG_EROR,"If valid, add %d to the list of KNOWN subtypes in mars.def",subtype);
				}
				else if (!subtype_group[subtype])
				{
					marslog(LOG_EROR,"Found report %d with subtype %d which "\
					"does not match the specified OBSTYPE/OBSGROUP",cnt+1,
						subtype);
					err_count++;
				}

				/*------------------------------------------------------*/
				/* check the message is in archive period				*/
				/*------------------------------------------------------*/
				obs_date = key_2_datetime(keyptr);
				if (obs_date < archive_period.begin || 
					obs_date > archive_period.end && e == 0)
				{
					char date[20];
					char time[20];
					print_key_date(keyptr,date);
					print_key_time(keyptr,time);
					marslog(LOG_EROR,"Bufr message n.%d %s %s is out of the archiving period specified in the request",
					cnt+1,date,time);
					err_count++;
				}
						
				cnt++;
			}
			else
			{
				 marslog(LOG_EROR,
						"Key of report %d is not valid",cnt+1);
				 err_count++;
			}
			length = sizeof(buffer);

			if (err_count > 50)
				e = -6;
		}

		if(e == EOF) e = 0;
		if(f) fclose(f);

	} /* Loop on sources */

	if (err_count > 0)
		e = -6;

	if(e == NOERR)
	{
		marslog(LOG_INFO,"Archiving %d report(s) from %d file(s)",cnt,q-1);
		dhs->count = cnt;
	}

	set_value(original_req,"_SIZE","%d",cnt);

	stop_timer(buf);
	if(*buf) marslog(LOG_INFO,"Analyse time: %s",buf);

	return e;
}

typedef struct multidatabase {
	boolean done;
	long64  total_tosend;
	int     databases;
} multidatabase;

static err sanity_check(request* r)
{
	const char *suite = getenv("SUITE");
	const char *t = get_value(r,"STREAM",0);

	if(suite && EQ(suite,"bc") && t)
	{
		if((!EQ(t,"SCWV") && !EQ(t,"SCDA")))
		{
			int exp = atol(get_value(r,"EXPVER",0));
			FILE *f = mail_open(mars.dhsmail,"sanity check failed stream: %s",t);
			mail_msg(f,"Archiving from suite bc stream %s",t);
			mail_request(f,"MARS Request:",r);
			mail_close(f);

			if(exp == 1)
			{
				marslog(LOG_EROR,"Archiving from suite %s stream %s. Exiting...",suite,t);
				return -10;
			}

			marslog(LOG_WARN,"Archiving from suite %s stream %s",suite,t);
		}
	}
	return NOERR;

}

static err dhs_grib_check(dhsdata *dhs,request* original_req)
{
	long length = 32768;
	char buffer[32768];
	int  cnt = 0;
	err e    = 0;
	FILE *f;
	char buf[80];
	int q = 0;
	const char* s,*t;
	int count,total;
	char *found;
	boolean isfg = false;
	static multidatabase once = { false, 0, 0};
	request *u = unwind_one_request(dhs->req);
	int dbcount = count_values(dhs->req,"DATABASE");

	timer *read_for_analyse = get_timer("Read GRIBs for analyse",NULL,true);

	s = get_value(dhs->req,"TYPE",0);
	t = get_value(dhs->req,"STREAM",0);

	isfg = (s && t && EQ(s,"FG") && !EQ(t,"WAVE") && !EQ(t,"SCWV"));

	if(isfg)
	{
		request *k = un_first_guess_all(u);
		free_all_requests(u);
		u = k;
		marslog(LOG_INFO,"Data is first-guess");
		if(mars.debug)
			print_one_request(u);
		set_value(dhs->req,"TYPE","FC");
	}


	/* If language accept multi TYPE/CLASS/..., update count_fields 
	   and fixed_check */

	count = count_fields(dhs->req);
	total = compute_total(dhs->req);

	/* Avoid performing the check for the same database */
	if(once.done)
	{
		marslog(LOG_INFO,"Source data already analysed");


		dhs->total_tosend = once.total_tosend;
		once.databases++;

		/* Reset it for further requests */
		if(once.databases == dbcount)
		{
			once.done = false;
			once.databases = 0;
			once.total_tosend = 0;
		}

		return NOERR;
	}

	found = NEW_ARRAY_CLEAR(char,total);

	dhs->total_tosend = 0;

	if(mars.debug)
		print_all_requests(u);

	start_timer();

	while(e == 0 && (s = get_value(dhs->req,"SOURCE",q++)) != NULL)
	{

		s = no_quotes(s);

		marslog(LOG_INFO,"Scanning and analysing %s",s);

		f = fopen(s,"r");
		if(f == NULL)
		{
			marslog(LOG_EROR|LOG_PERR,"ARCHIVE : '%s'",s);
			e = -6;
			break;
		}

		if(fseek(f,0,SEEK_END) != 0)
		{
			marslog(LOG_EROR|LOG_PERR,"fseek(%s)",s);
			e = -2;
		}

		dhs->total_tosend += ftell(f);

		if(fseek(f,0,SEEK_SET) != 0)
		{
			marslog(LOG_EROR|LOG_PERR,"fseek(%s)",s);
			e = -2;
		}


		length = sizeof(buffer);
		while( !e && (e = timed_readany(f,buffer,&length,read_for_analyse)) == NOERR || 
			(e == BUF_TO_SMALL))
		{
			int n = -1;
			request *g = empty_request("GRIB");

			if((e = grib_to_request(g,buffer,length)) != NOERR)
			{
				marslog(LOG_EROR,"Error while building MARS request from GRIB %d",cnt+1);
				marslog(LOG_EROR,"MARS description for Grib %d is:",cnt+1);
				print_all_requests(g);
			}

			if(mars.debug)
			{
				marslog(LOG_DBUG,"Field %d:",cnt);
				print_all_requests(g);
			}

			if((e == NOERR) && (e = fix_check(g,dhs->req,cnt+1)) == NOERR)
			{
				n = find_order(&u,g);

				if(n < 0 ) 
				{
					if(field_order(dhs->req,g) >= 0)
						marslog(LOG_EROR,"Grib %d in '%s' is duplicated",cnt+1,s);
					else
						marslog(LOG_EROR,"Grib %d is not in ARCHIVE request",cnt+1);
					marslog(LOG_EROR,"MARS description for Grib %d is:",cnt+1);
					print_all_requests(g);
					e = -6;
				} 
				else if(n >= total)
				{
					marslog(LOG_EROR,"MARS internal error analysing grib %d",cnt+1);
					marslog(LOG_EROR,"MARS description for Grib %d is:",cnt+1);
					print_all_requests(g);
					e = -6;
				}
				else if(found[n])
				{
					marslog(LOG_EROR,"Grib %d in '%s' is duplicated",cnt+1,s);
					marslog(LOG_EROR,"MARS description for Grib %d is:",cnt+1);
					print_all_requests(g);
					e = -6;
				}
			}

			if((e == NOERR) || (e == BUF_TO_SMALL))
			{
				found[n] = true;
				cnt++;
			}

			length = sizeof(buffer);
			free_all_requests(g);
		}

		if(e == EOF) e = 0;
		if(f) fclose(f);

	} /* Loop on sources */

	free_all_requests(u);
	FREE(found);

#ifdef fujitsu
	free(buffer);
#endif

	if(!e && cnt != count)
	{
		marslog(LOG_EROR,"Wrong number of fields :");
		marslog(LOG_EROR,"Request describes %d field(s)",count);
		marslog(LOG_EROR,"Source contains %d field(s)",cnt);
		e = -6;
	}
	else if(e == NOERR)
	{
		if(dbcount > 1)
		{
			once.done = true;
			once.total_tosend = dhs->total_tosend;
			once.databases++;
		}
		marslog(LOG_INFO,"Archiving %d field(s) from %d file(s)",cnt,q-1);
		dhs->count = cnt;
	}

	set_value(original_req,"_SIZE","%d",cnt);


	stop_timer(buf);
	if(*buf) marslog(LOG_INFO,"Analyse time: %s",buf);

	if(isfg) set_value(dhs->req,"TYPE","FG"); /* Reset type */

	return e;
}

static err dhs_image_check(dhsdata *dhs)
{
	err e    = 0;
	FILE *f;
	int q = 0;
	const char* s;
	char buf[80];


	dhs->total_tosend = 0;
	start_timer();

	while(e == 0 && (s = get_value(dhs->req,"SOURCE",q++)) != NULL)
	{
		s = no_quotes(s);
		marslog(LOG_INFO,"Scanning %s",s);

		f = fopen(s,"r");
		if(f == NULL)
		{
			marslog(LOG_EROR|LOG_PERR,"ARCHIVE : '%s'",s);
			e = -6;
			break;
		}

		if(fseek(f,0,SEEK_END) != 0)
		{
			marslog(LOG_EROR|LOG_PERR,"fseek(%s)",s);
			e = -2;
		}

		dhs->total_tosend += ftell(f);

		fclose(f);

	} /* Loop on sources */


	stop_timer(buf);
	if(*buf) marslog(LOG_INFO,"Scanning time: %s",buf);

	return e;
}

static err dhs_store_check(dhsdata *dhs)
{
	long length = 32768;
	char buffer[32768];
	err e    = 0;
	FILE *f;
	char buf[80];
	int q = 0;
	const char* s;

	dhs->total_tosend = 0;

	start_timer();

	while(e == 0 && (s = get_value(dhs->req,"SOURCE",q++)) != NULL)
	{

		s = no_quotes(s);

		marslog(LOG_INFO,"Scanning and analysing %s",s);

		f = fopen(s,"r");
		if(f == NULL)
		{
			marslog(LOG_EROR|LOG_PERR,"STORE : '%s'",s);
			e = -6;
			break;
		}

		if(fseek(f,0,SEEK_END) != 0)
		{
			marslog(LOG_EROR|LOG_PERR,"fseek(%s)",s);
			e = -2;
		}

		dhs->total_tosend += ftell(f);

		rewind(f);

		/*---------------------------*/

		/* Check for GRIB or BUFR */

		if(_readany(f,buffer,&length) != -1)
		{
			marslog(LOG_EROR,"STORE: %s contains GRIB or BUFR",s);
			e = -7;
		}

		rewind(f);

		/* check for something else....*/

		/*---------------------------*/

		fclose(f);

	} /* Loop on sources */

	stop_timer(buf);
	if(*buf) marslog(LOG_INFO,"Analyse time: %s",buf);

	return e;
}



source* new_source(request* r)
{
	source* s = NEW_CLEAR(source);
	s->cube = new_hypercube(r);
}


void free_source(source* s)
{
	free_hypercube(s->cube);
}

err source_open(source*)
{
}

err source_read(source*)
{
}

