"""
TAP: schema maintenance, job/parameter definition incl. upload and UWS actions.
"""

#c Copyright 2008-2020, the GAVO project
#c
#c This program is free software, covered by the GNU GPL.  See the
#c COPYING file in the source distribution.


import datetime
import functools
import os

from pyparsing import ParseException
from twisted.internet import threads

from gavo import base
from gavo import rsc
from gavo import svcs
from gavo import utils
from gavo.protocols import uws
from gavo.protocols import uwsactions
from gavo.utils import codetricks


RD_ID = "__system__/tap"

# used in the computation of quote
EST_TIME_PER_JOB = datetime.timedelta(minutes=10)

# A mapping of values of TAP's RESPONSEFORMAT parameter to our 
# formats.format codes, IANA media types and user-readable labels.
# Used below (1st element of value tuple) and for registry purposes.
FORMAT_CODES = {
	base.votableType:
		("votable", base.votableType, "VOTable, binary", 
			"ivo://ivoa.net/std/TAPRegExt#output-votable-binary"),
	"text/xml": 
		("votable", "text/xml", "VOTable, binary",
			"ivo://ivoa.net/std/TAPRegExt#output-votable-binary"),
	"votable": 
		("votable", base.votableType, "VOTable, binary",
			"ivo://ivoa.net/std/TAPRegEXT#output-votable-binary"),
	"application/x-votable+xml;serialization=binary2": 
		("votableb2", "application/x-votable+xml;serialization=binary2", 
			"VOTable, new binary", 
			"ivo://ivoa.net/std/TAPRegExt#output-votable-binary2"),
	"votable/b2": 
		("votableb2", "application/x-votable+xml;serialization=binary2", 
			"VOTable, new binary", 
			"ivo://ivoa.net/std/TAPRegExt#output-votable-binary2"),
	"vodml":
		("vodml", "application/x-votable+xml;version=1.4",
			"Experimental VOTable 1.4",
			"http://dc.g-vo.org/output-vodml"),
	"application/x-votable+xml;serialization=tabledata":
		("votabletd", "application/x-votable+xml;serialization=tabledata", 
			"VOTable, tabledata",
			"ivo://ivoa.net/std/TAPRegEXT#output-votable-td"),
	"votable/td":
		("votabletd", "application/x-votable+xml;serialization=tabledata", 
			"VOTable, tabledata",
			"ivo://ivoa.net/std/TAPRegEXT#output-votable-td"),
	"text/csv": 
		("csv", "text/csv", "CSV without column labels", None),
	"csv": ("csv_header", "text/csv;header=present", 
			"CSV with column labels", None),
	"text/csv;header=present": 
		("csv_header", "text/csv;header=present",
			"CSV with column labels", None),
	"text/tab-separated-values": 
		("tsv", "text/tab-separated-values", 
			"Tab separated values", None),
	"tsv": 
		("tsv", "text/tab-separated-values", 
			"Tab separated values", None),
	"text/plain": 
		("tsv", "text/plain",
			"Tab separated values", None),
	"application/fits": 
		("fits", "application/fits", "FITS binary table", None),
	"fits":
		("fits", "application/fits", "FITS binary table", None),
	"text/html": 
		("html", "text/html", "HTML table", None),
	"html": 
		("html", "text/html", "HTML table", None),
	"json": 
		("json", "application/json", "JSON", None),
	"application/json": 
		("json", "application/json", "JSON", None),
	"geojson":
		("geojson", "application/geo-json", "GeoJSON", None),
}


def _R(**kws): return kws

# this is used below in for registry purposes (values are pairs of
# IVOA id and a human-readable label).
SUPPORTED_LANGUAGES = {
	"ADQL": _R(versions={
			"2.0": "ivo://ivoa.net/std/ADQL#v2.0",
			"2.1": "ivo://ivoa.net/std/ADQL#v2.1"},
		description="The Astronomical Data Query Language is the standard"
			" IVOA dialect of SQL; it contains a very general SELECT statement"
			" as well as some extensions for spherical geometry and higher"
			" mathematics.")}


# A list of supported upload methods.  This is only used in the registry
# interface right now.
UPLOAD_METHODS = {
	"upload-inline": "POST inline upload",
	"upload-http": "http URL",
	"upload-https": "https URL",
	"upload-ftp": "ftp URL",
}


class TAPError(uws.UWSError):
	"""here for backward compatibility.

	Deprecated.
	"""


######################## registry interface helpers

def getSupportedLanguages():
	"""returns a list of tuples for the supported languages.

	This is tap.SUPPORTED_LANGUAGES in a format suitable for the
	TAP capabilities element.

	Each tuple returned is made up of 
	(name, description, [(version, ivo-id)...]).
	"""
	for name, desc in SUPPORTED_LANGUAGES.items():
		yield (name, desc["description"], list(desc["versions"].items()))


def getSupportedOutputFormats():
	"""yields tuples for the supported output formats.

	This is tap.OUTPUT_FORMATS in a format suitable for the
	TAP capabilities element.

	Each tuple is made up of (mime, aliases, description, ivoId).
	"""
	codes, descrs, ivoIds = {}, {}, {}
	for code, (_, outputMime, descr, ivoId) in FORMAT_CODES.items():
		codes.setdefault(outputMime, set()).add(code)
		descrs[outputMime] = descr
		ivoIds[outputMime] = ivoId
	for mime in codes:
		# mime never is an alias of itself
		codes[mime].discard(mime)
		yield mime, codes[mime], descrs[mime], ivoIds[mime]


######################## maintaining TAP schema

def _insertRDIntoTAP_SCHEMA(rd, connection):
	"""helps publishToTAP.

	Actually, it does all its work, except not rejecting //tap itself.
	This is an implementation detail of letting //tap#createSchema's
	postCreation script do its work.
	"""
	# first check if we have any adql tables at all, and don't attempt
	# anything if we don't (this is cheap optimizing and keeps TAP_SCHEMA
	# from being created on systems that don't do ADQL).
	for table in rd.tables:
		# the readProfile condition in the next line is a proxy for adql=hidden.
		if table.adql or "untrustedquery" in table.readProfiles:
			break
	else:
		return
	tapRD = base.caches.getRD(RD_ID)
	for ddId in ["importTablesFromRD", "importDMsFromRD", "importColumnsFromRD", 
			"importFkeysFromRD", "importGroupsFromRD"]:
		dd = tapRD.getById(ddId)
		rsc.makeData(dd, forceSource=rd, parseOptions=rsc.parseValidating,
			connection=connection, runCommit=False)
	
	# finally, remove schemas that don't have any tables (which happens
	# with adql-hidden tables, when moving RDs, etc.  Such leftovers
	# are trouble because they're in TAP_SCHEMA but not in tables any more.
	connection.execute("""
		delete from tap_schema.schemas 
		where
			schema_name in (
				select schema_name 
				from tap_schema.schemas 
					left outer join tap_schema.tables using (schema_name) 
				where table_name is null)""")


def publishToTAP(rd, connection):
	"""publishes info for all ADQL-enabled tables of rd to the TAP_SCHEMA.
	"""
	if rd.sourceId=='__system__/tap':
		# if we're being built ourselves, skip this; tap's tap_schema
		# maintenance is manual.
		return
	_insertRDIntoTAP_SCHEMA(rd, connection)


def unpublishFromTAP(rd, connection):
	"""removes all information originating from rd from TAP_SCHEMA.
	"""
	if rd.sourceId=='__system__/tap':
		# if we're being built ourselves, skip this; tap's tap_schema
		# maintenance is manual.
		return

	rd.setProperty("moribund", "True") # the embedded grammar take this
	                                   # to mean "kill this"
	publishToTAP(rd, connection)
	rd.clearProperty("moribund")


def getAccessibleTables():
	"""returns a list of qualified table names for the TAP-published tables.
	"""
	with base.getTableConn() as conn:
		return [r[0]
			for r in conn.query("select table_name from tap_schema.tables"
				" order by table_name")]


########################## Maintaining TAP jobs


class TAPTransitions(uws.ProcessBasedUWSTransitions):
	"""The transition function for TAP jobs.

	There's a hack here: After each transition, when you've released
	your lock on the job, call checkProcessQueue (in reality, only
	PhaseAction does this).
	"""
	def __init__(self):
		uws.SimpleUWSTransitions.__init__(self, "TAP")

	def getCommandLine(self, wjob):
		return "gavo", ["gavo", "--ui", "stingy", "tap", "--", str(wjob.jobId)]

	def queueJob(self, newState, wjob, ignored):
		"""puts a job on the queue.
		"""
		uws.ProcessBasedUWSTransitions.queueJob(self, newState, wjob, ignored)
		wjob.uws.scheduleProcessQueueCheck()

	def errorOutJob(self, newPhase, wjob, ignored):
		uws.SimpleUWSTransitions.errorOutJob(self, newPhase, wjob, ignored)
		wjob.uws.scheduleProcessQueueCheck()

	def completeJob(self, newPhase, wjob, ignored):
		uws.SimpleUWSTransitions.completeJob(self, newPhase, wjob, ignored)
		wjob.uws.scheduleProcessQueueCheck()

	def killJob(self, newPhase, wjob, ignored):
		try:
			uws.ProcessBasedUWSTransitions.killJob(self, newPhase, wjob, ignored)
		finally:
			wjob.uws.scheduleProcessQueueCheck()


########################## The TAP UWS job


@functools.lru_cache(1)
def getUploadGrammar():
	from pyparsing import (Word, ZeroOrMore, Suppress, StringEnd,
		alphas, alphanums, CharsNotIn)
	# Should we allow more tableNames?
	with utils.pyparsingWhitechars(" \t"):
		tableName = Word( alphas+"_", alphanums+"_" )
		# What should we allow/forbid in terms of URIs?
		uri = CharsNotIn(" ;,")
		uploadSpec = tableName("name") + "," + uri("uri")
		uploads = uploadSpec + ZeroOrMore(
			Suppress(";") + uploadSpec) + StringEnd()
		uploadSpec.addParseAction(lambda s,p,t: (t["name"], t["uri"]))
		return uploads


def parseUploadString(uploadString):
	"""iterates over pairs of tableName, uploadSource from a TAP upload string.
	"""
	try:
		res = utils.pyparseString(getUploadGrammar(), uploadString).asList()
		return res
	except ParseException as ex:
		raise base.ValidationError(
			"Syntax error in UPLOAD parameter (near %s)"%(ex.loc), "UPLOAD",
			hint="Note that we only allow regular SQL identifiers as table names,"
				" i.e., basically only alphanumerics are allowed.")


class LangParameter(uws.JobParameter):
	@classmethod
	def addPar(cls, name, value, job):
		uws.JobParameter.updatePar(name, value, job)


class MaxrecParameter(uws.JobParameter):
	name = "MAXREC"
	_serialize, _deserialize = staticmethod(uws.strOrEmpty), int


class LocalFile(object):
	"""A sentinel class representing a file within a job work directory
	(as resulting from an upload).
	"""
	def __init__(self, jobId, wd, fileName):
		self.jobId, self.fileName = jobId, fileName
		self.fullPath = os.path.join(wd, fileName)

	def __str__(self):
		# stringify to a URL for easy UPLOAD string generation.
		# This smells of a bad idea.  If you change it, change UPLOAD.getParam.
		return self.getURL()

	def getURL(self):
		"""returns the URL the file is retrievable under for the life time of
		the job.
		"""
		return base.caches.getRD(RD_ID).getById("run").getURL("tap",
			absolute=True)+"/async/%s/results/%s"%(
				self.jobId,
				self.fileName)


class UploadParameter(uws.JobParameter):
# the way this is specified, inline uploads are quite tricky. 
# To obtain the data, we must access the request, which we don't have
# here.  So, I just grab in from upstack (which of course is bound
# to fail if we're not being called from within a proper web request).
# It's not pretty, but then this kind of interdependency between
# HTTP parameters sucks whatever you do.
#
# We assume uploads come in the request's special files dictionary.
# This is created in taprender.TAPRenderer.gatherUploadFiles.

	_deserialize, _serialize = utils.identity, utils.identity

	@classmethod
	def addPar(cls, name, value, job):
		if not value.strip():
			return
		newUploads = []
		for tableName, upload in parseUploadString(value):
			if upload.startswith("param:"):
				newUploads.append(
					(tableName, cls._saveUpload(job, upload[6:])))
			else:
				newUploads.append((tableName, upload))
		newVal = job.parameters.get(name, [])+newUploads
		uws.JobParameter.updatePar(name, newVal, job)

	@classmethod
	def getPar(cls, name, job):
		return ";".join("%s,%s"%p for p in job.parameters.get("upload", ""))

	@classmethod
	def _saveUpload(cls, job, paramName):
		try:
			upload = codetricks.stealVar("request").fields[paramName]
			uploadName = utils.defuseFileName(upload.filename)
			uploadFile = upload.file
		except KeyError:
			raise base.ui.logOldExc(
				base.ValidationError("No inline upload '%s' found"%paramName, 
					"UPLOAD"))
		except AttributeError:
			raise base.ui.logOldExc(
				base.ValidationError("Upload parameter references non-file '%s'"
					%paramName, "UPLOAD"))

		uploadName = utils.defuseFileName(uploadName)
		with job.openFile(uploadName, "wb") as f:
			f.write(uploadFile.read())
		return LocalFile(job.jobId, job.getWD(), uploadName)


class TAPJob(uws.UWSJobWithWD):
	_jobsTDId = "//tap#tapjobs"
	_transitions = TAPTransitions()

	_parameter_maxrec = MaxrecParameter
	_parameter_lang = LangParameter
	_parameter_upload = UploadParameter
	_parameter_request = uws.JobParameter
	_parameter_query = uws.JobParameter
	_parameter_responseformat = uws.JobParameter

	@property
	def quote(self):
		"""returns an estimation of the job completion.

		This currently is very naive: we give each job that's going to run
		before this one 10 minutes.

		This method needs to be changed when the dequeueing algorithm
		is changed.
		"""
		with base.getTableConn() as conn:
			nBefore = self.uws.runCanned('countQueuedBefore',
				{'dt': self.destructionTime}, conn)[0]["count"]
		return datetime.datetime.utcnow()+nBefore*EST_TIME_PER_JOB



#################### The TAP worker system

class PlanAction(uwsactions.JobAction):
	"""retrieve a query plan.

	This is actually a TAP action; as we add UWSes, we'll need to think
	about how we can customize uwsactions my UWS type.
	"""
	name = "plan"

	def formatPlan(self, qTableAndPlan):
		qTable, plan = qTableAndPlan
		return plan.encode("utf-8")

	def getPlan(self, job, request):
		from gavo.protocols import taprunner
		qTable = taprunner.getQTableFromJob(job.parameters,
			job.jobId, "untrustedquery", 1)
		request.setHeader("content-type", "text/plain;charset=utf-8")
		return qTable, qTable.getPlan()

	def doGET(self, job, request):
		return threads.deferToThread(self.getPlan, job, request
			).addCallback(
			self.formatPlan)


class TAPUWS(uws.UWSWithQueueing):
	"""The UWS responsible for processing async TAP requests.
	"""
	_baseURLCache = None

	joblistPreamble = ("<?xml-stylesheet href='/static"
		"/xsl/tap-joblist-to-html.xsl' type='text/xsl'?>")
	jobdocPreamble = ("<?xml-stylesheet href='/static/xsl/"
		"tap-job-to-html.xsl' type='text/xsl'?>")

	def __init__(self):
		self.runcountGoal = base.getConfig("async", "maxTAPRunning")
		uws.UWSWithQueueing.__init__(self, TAPJob, uwsactions.JobActions(
			PlanAction))

	@property
	def baseURL(self):
		if self._baseURLCache is None:
			self._baseURLCache = base.caches.getRD(
				RD_ID).getById("run").getURL("tap")
		return self._baseURLCache

	def getURLForId(self, jobId):
		"""returns a fully qualified URL for the job with jobId.
		"""
		return "%s/%s/%s"%(self.baseURL, "async", jobId)

WORKER_SYSTEM = TAPUWS()


######################### The TAP core

class TAPCore(svcs.Core):
	"""A core for the TAP renderer.

	Right now, this is a no-op and not used by the renderer.

	This will change as we move to regularise the TAP system.
	"""
	name_ = "tapCore"
