"""
TAP 1.2 prototype persistent TAP uploads.

These are implemented as renderers that might work on stuff other than
TAP, but I'm not sure there is a case to be made for persistent *table*
uploads on anything else.  And anyway: these tables have extra features
to be TAP accessible, and they are managed through the
TAP_SCHEMA.persistent_uploads table.
"""

#c Copyright 2008-2025, the GAVO project <gavo@ari.uni-heidelberg.de>
#c
#c This program is free software, covered by the GNU GPL.  See the
#c COPYING file in the source distribution.


import datetime
import functools
import re

from twisted.internet import threads
from twisted.web import resource
from twisted.web import server

from gavo import base
from gavo import formats
from gavo import rsc
from gavo import rscdef
from gavo import svcs
from gavo import utils
from gavo.base import valuemappers
from gavo.formats import votableread
from gavo.protocols import dali
from gavo.protocols import uws
from gavo.protocols import useruws
from gavo.registry import tableset
from gavo.web import vodal
from gavo.web.vosi import VTM


USER_TABLE_NAME_PATTERN = re.compile(r"[a-z][a-z0-9_]*$")

MS = base.makeStruct


@functools.cache
def getIndexerUWS():
	return useruws.makeUWSForService(
		base.resolveCrossId("//tap_user#create-index"))


def createIndex(user, tableName, indexName):
	"""actually creates an index for an uploaded table.

	This is called from within //tap_user#create-index.
	"""
	td, physicalName = getUploadedTableDef(user, tableName)
	td.id = physicalName.split(".")[-1]
	td.onDisk = True
	for index in td.indices:
		if index.name==indexName:
			break
	else:
		raise base.NotFoundError(tableName, "Uploaded table" f"uploads for {user}")
	
	with base.getWritableAdminConn() as conn:
		table = rsc.TableForDef(td, connection=conn)
		index.create(table, notify=False)


def _getBaseUCD(ucd):
	"""returns a basis UCD for spatial UCDs.

	This is only useful for _customizeIndex.
	"""
	return ".".join(ucd.split(";")[0].split(".")[:-1])


def _customizeIndex(columns):
	"""returns extra DBIndex attributes to create indexes for specific sorts
	of columns.
	"""
	if len(columns)==1 and columns[0].type=="spoint":
		return {"cluster": True, "kind": "spoint", "method": "GIST"}
	
	elif len(columns)==2:
		latCol, lonCol = None, None
		for col in columns:
			if col.ucd.split(";")[0] in {
					"pos.eq.ra", "pos.galactic.lon", "pos.earth.lon",
					"pos.ecliptic.lon"}:
				if col.unit=="deg":
					lonCol = col
			if col.ucd.split(";")[0] in {
					"pos.eq.dec", "pos.galactic.lat", "pos.earth.lat",
					"pos.ecliptic.lat"}:
				if col.unit=="deg":
					latCol = col

		# The _getBaseUCD condition below is a bit questionable: Shouldn't
		# we error out on these?  Mixing different coordinate flavours
		# probably is never sensible, but silently refusing to do a
		# spatial index on them doesn't *quite* feel right either.
		if (latCol is not None and lonCol is not None
				and _getBaseUCD(latCol.ucd)==_getBaseUCD(lonCol.ucd)):
			indexType = base.getConfig("db", "splitSpatialIndex")
			if indexType=="q3c":
				return {"cluster": True,
					"content_": f"q3c_ang2ipix({lonCol.name}),{latCol.name})"}
			elif indexType=="pgsphere":
				return {"cluster": True, "method": "GIST", "kind": "spoint",
					"content_": f"spoint(radians({lonCol.name}),radians({latCol.name}))"}
			else:
				raise Exception(f"Unknown spatial index method {indexType}")

	return {}


def dropUserUploadedTable(conn, user, tableName):
	"""drops a user-uploaded table.

	This will drop both the entry in the management table and the table
	itself.  It is not an error to request dropping a non-existing table;
	that's a no-op, though.
	"""
	constraints = {"user": user, "tablename": tableName}
	res = list(conn.query("select physical_name from tap_user.tables"
			" WHERE user_name=%(user)s and table_name=%(tablename)s",
			constraints))
	if res:
		physicalName = res[0][0]
		conn.execute(f"DROP TABLE IF EXISTS {physicalName}")

		conn.execute("DELETE FROM tap_user.tables"
				" WHERE user_name=%(user)s and table_name=%(tablename)s",
				constraints)


def getUploadedTableDef(user, tableName):
	"""returns an rsc.TableDef for an uploaded table for user together with
	its physical name.

	This will raise an UnknownURI if no such table exists.
	"""
	with base.getTableConn() as conn:
		try:
			cursor = conn.query(
				"SELECT metadata, physical_name"
				" FROM tap_user.tables"
				" WHERE user_name=%(user)s and table_name=%(tablename)s",
				{"user": user, "tablename": tableName})
			rawTD, physicalName = next(cursor)
		except StopIteration:
			raise svcs.UnknownURI(f"No such uploaded table for user {user}")

	# Defensive programming: Make sure there's only one match
	try:
		next(cursor)
	except StopIteration:
		pass
	else:
		# this can't happen
		raise Exception("Two matches for uploaded table")

	td = base.parseFromString(rscdef.TableDef, rawTD)
	td.parent = base.caches.getRD("//tap_user")
	return td, physicalName


def uploadUserTable(conn, tableName, srcFile, user):
	"""actually performs a table ingestion.

	It arranges for the data in the VOTable readable from srcFile
	will be tap_user.tableName for user, while it is actually
	stored in tap_user.<something else>.  That something else,
	the physcial_name, is returned by this function.

	The function also performs the basic bookkeeping, including storing a
	VODataService tableset in tap_user.tables.
	"""
	if valuemappers.needsQuoting(tableName):
		raise base.ReportableError(f"'{tableName}' cannot be used as"
			" an upload table name (which must be regular ADQL identifiers,"
			" in particular not ADQL reserved words).")

	# this is where the physical name is defined.  Everyone else
	# must get it from the physcial_name column, as this rule
	# will probably change.
	physicalName = f"tap_user._{user}_{tableName}"
	if not re.match(r"tap_user\.[a-z_0-9]+$", physicalName):
		raise base.ReportableError("Bad generated table name.  Giving up.")
	if base.UnmanagedQuerier(conn).getTableType(physicalName):
		conn.execute(f"DROP table IF EXISTS {physicalName}")

	if srcFile:
		uploadedTable = votableread.uploadVOTable(
			physicalName.split(".")[-1],
			srcFile,
			conn,
			nameMaker=votableread.AutoQuotedNameMaker(),
			rd=base.caches.getRD("//tap_user"),
			temporary=False,
			adql=True)
		serialised = serialiseTD(uploadedTable.tableDef, tableName)
	
	else:
		# without a source file, this is just an allocation (for create table),
		# and we have no idea about the table structure.  Do a placeholder:
		serialised = f'<table id="{tableName}">'

	tablesTable = rsc.TableForDef(
		base.resolveCrossId("//tap_user#tables"),
		connection=conn)
	tablesTable.addRow({
		"user_name": user,
		"table_name": tableName,
		"physical_name": physicalName,
		"metadata": serialised,
		"expiry": datetime.datetime.utcnow()
			+datetime.timedelta(days=base.getConfig("ivoa", "userTableDays"))})
	
	return physicalName


class ResourceWithDALIErrors(resource.Resource):
	"""A class that tries to produce DALI errors on almost all that
	can go wrong in the sync case.

	To work this, don't override render and getChild but render_METHOD
	and _getChildReal.  Only sync errors will be handled, when you defer,
	you need to call addErrback(self._renderError, request) yourself.
	_renderError finishes the request itself.
	"""
	def render(self, request):
		try:
			return resource.Resource.render(self, request)
		except Exception as exc:
			if not isinstance(exc, svcs.Error):
				base.ui.notifyError(str(exc))
			return dali.serveDALIError(request, exc)

	def _renderError(self, flr, request):
		base.ui.notifyFailure(flr)
		dali.serveDALIError(request, flr.value)
		return server.NOT_DONE_YET

	def serveImmediate(self, request, mediaType, content):
		request.setHeader("content-type", mediaType)
		request.write(utils.bytify(content))
		request.finish()
		return server.NOT_DONE_YET

	def _getChildReal(self, name, request):
		raise svcs.UnknownURI("{self.__class__.__name__} has no children")

	def getChild(self, name, request):
		try:
			return self._getChildReal(utils.debytify(name), request)
		except Exception as exc:
			return dali.DALIErrorResource(exc)


def serialiseTD(tableDef, logicalName):
	"""returns RD XML for tableDef.
	"""
	# we need to manually give the table an id because structToXML
	# swallows these.  I suppose there should be a way to presere
	# id-s in that function, but knowing when to drop an id and
	# when to preserve it is not simple; note that you cannot
	# TableForDef these TDs because their physical name is differnt
	# from their id.
	return base.structToXML(tableDef).replace(
			"<table>", f'<table id="{logicalName}">')


class PersistentUploadResource(ResourceWithDALIErrors):
	"""a resource operating on a persistent table upload.

	These are all constructed with the table name in question, which
	is the logical name without the tap_user schema.  Note that
	depending on which user is encoded in the request handled, this
	may refer to different physical tables.
	"""
	def __init__(self, tableName):
		self.tableName = tableName
		ResourceWithDALIErrors.__init__(self)


	def _getCurrentTD(self, request):
		"""returns a TableDef for the user uploaded table for this request.
		"""
		user = request.getAuthUser() or "anonymous"
		return getUploadedTableDef(user, self.tableName)[0]


class DestructionTimeManager(PersistentUploadResource):
	"""a resource for setting and retrieving the life time of an
	uploaded table.

	This handles the destruction child of the table resource.
	"""
	def getGrammar(self):
		return MS(svcs.ContextGrammar,
			inputKeys=[MS(svcs.InputKey,
				name="DESTRUCTION",
				type="timestamp",
				tablehead="Delete at",
				description="DALI-type ISO timestamp for when this table should"
					" be dropped automatically.",
				multiplicity="single")])

	def render_POST(self, request):
		user = request.getAuthUser() or "anonymous"
		parsed = self.getGrammar().parseStrargs(request.strargs)

		if not parsed["DESTRUCTION"]:
			raise svcs.BadRequestion("DESTRUCTION parameter required on POST")
		
		destructionTime = parsed["DESTRUCTION"]

		# let's have a hard limit of a year for now
		limit = datetime.datetime.now()+datetime.timedelta(days=366)
		newExpiry = min(destructionTime, limit)

		with base.getWritableTableConn() as conn:
			conn.execute("UPDATE tap_user.tables"
				" SET expiry=%(expiry)s"
				" WHERE user_name=%(user)s and table_name=%(tablename)s", {
				"user": user,
				"tablename": self.tableName,
				"expiry": newExpiry})
	
		return self.render_GET(request)
	
	def render_GET(self, request):
		user = request.getAuthUser() or "anonymous"
		with base.getWritableTableConn() as conn:
			res = conn.query("SELECT expiry FROM tap_user.tables"
				" WHERE user_name=%(user)s and table_name=%(tablename)s", {
				"user": user,
				"tablename": self.tableName})

		expiry = next(res)[0]
		request.setHeader("content-type", "text/plain; charset=utf-8")
		return utils.bytify(utils.formatISODT(expiry))


class IndexManager(PersistentUploadResource):
	"""The index child of a persistently uploaded table.

	POST creates an index (possibly of special kinds), GET returns
	a human-readable non-standard text representation of the indexes.
	"""
	def getGrammar(self):
		return MS(svcs.ContextGrammar,
			inputKeys=[MS(svcs.InputKey,
				name="INDEX",
				type="text",
				tablehead="Index column(s)",
				description="A name of a column to index. This parameter can"
					" be repeated to create joint indexes; a particularly common"
					" reason to simulatneously index multiple columns is spherical"
					" coordinates.  DaCHS will use an appropriate spatial indexing"
					" method if it detects that situation (by using UCDs).",
					multiplicity="multiple")])

	def _startIndexCreation(self, request, indexNames, user):
		td = self._getCurrentTD(request)
		indexColumns = []
		for name in indexNames:
			try:
				indexColumns.append(td.getColumnByName(name))
			except base.NotFoundError:
				raise svcs.BadRequest(f"No such column: {name}")

		newIndex = MS(
			rscdef.DBIndex,
			columns=indexNames,
			**_customizeIndex(indexColumns))

		# we want to overwrite indexes with the same names so you can
		# update indexes rather than accumulate them
		for indind, ind in enumerate(td.indices):
			if ind.name==newIndex.name:
				with base.getWritableAdminConn() as conn:
					ind.drop(base.UnmanagedQuerier(conn))
				td.indices[indind] = newIndex
				break
		else:
			td.indices.append(newIndex)

		newMetadata = serialiseTD(td, self.tableName)

		with base.getWritableTableConn() as conn:
			res = conn.execute("UPDATE tap_user.tables"
				" SET metadata=%(metadata)s"
				" WHERE user_name=%(user)s and table_name=%(tablename)s", {
				"user": user,
				"tablename": self.tableName,
				"metadata": newMetadata})
		if res!=1:
			raise base.DBError(
				f"Failed to update metadata for table {self.tableName}")

		ws = getIndexerUWS()
		uwsUser = user if user!="anonymous" else None
		jobId = ws.getNewIdFromArgs(
			{},
			{"table_name": self.tableName, "index_name": newIndex.name},
			uwsUser)
		ws.changeToPhase(jobId, uws.QUEUED)

		raise svcs.SeeOther(ws.getJob(jobId).getURL())

	def render_POST(self, request):
		user = request.getAuthUser() or "anonymous"
		parsed = self.getGrammar().parseStrargs(request.strargs)
		if not parsed["INDEX"]:
			raise svcs.BadRequestion("INDEX parameter required on POST")
		return self._startIndexCreation(request, parsed["INDEX"], user)
	
	def render_GET(self, request):
		td, _ = getUploadedTableDef(
			request.getAuthUser() or "anonymous", self.tableName)
		res = [f"Indexes on table tap_user.{self.tableName}\n"]
		if td.indices:
			for ind in td.indices:
				for statement in ind.iterCode():
					res.append(statement)
		else:
			res.append("(none)")

		request.setHeader("content-type", "text/plain; charset=utf-8")
		return utils.bytify("\n".join(res)+"\n")


class PersistentUploadManager(PersistentUploadResource):
	"""A resource corresponding to a persistently uploaded table.

	It is created by a put request.  A GET will yield a VODataService tablset,
	a DELETE will remove the table.  POST is undefined so far,
	but there's management children destruction and index.

	These are exclusively created by the PersistentUploadRenderer.
	"""
	def render_PUT(self, request):
		user = request.getAuthUser() or "anonymous"
		# for now, we try to get away with a computed physical name:
		# _<user>_<user-table-name>.  This *could* work because we
		# forbid leading underscores in our <user-table-name>-s.  This
		# will break (without delimited table identifiers) when our user
		# names become complicated.
		mediaType = request.getHeader("content-type")
		if mediaType is None:
			raise svcs.BadRequest("Missing payload media type")

		# normalise the media type so we can more easily recognised VOTables
		try:
			incomingFormat = formats.getMIMEFor(formats.getKeyFor(mediaType))
			if not "votable" in incomingFormat:
				raise formats.CannotSerializeIn("Really, parse from non-VOTable")
		except formats.CannotSerializeIn:
			raise svcs.BadRequest(f"Do not know how to ingest {mediaType} tables",
					hint="We accept VOTables with many media types; if in"
						" doubt, use application/x-votable+xml")

		# defer the rest into a thread; it's hard to make this async with
		# what we have.
		d = threads.deferToThread(self._perform_upload, request, user)
		d.addErrback(self._renderError, request)
		return server.NOT_DONE_YET

	def _perform_upload(self, request, user):
		with base.getWritableAdminConn() as conn:
			uploadUserTable(conn, self.tableName, request.content, user)
			
		self.serveImmediate(request, "text/plain; charset=utf-8",
			f"Query this table as tap_user.{self.tableName}\n")

	def render_GET(self, request):
		td = self._getCurrentTD(request)
		et = tableset.getTableForTableDef(td, set(), rootElement=VTM.table)

		request.setHeader("content-type", "text/xml")
		utils.xmlwrite(et, request,
			prolog="<?xml-stylesheet href='/static/xsl/vosi.xsl' type='text/xsl'?>")
		return b""

	def render_DELETE(self, request):
		user = request.getAuthUser() or "anonymous"
		with base.getWritableAdminConn() as conn:
			try:
				# dropUserUploadedTable fails silently, but we want
				# to raise an error, so let's make sure the table
				# to drop actually exists.
				next(conn.query(
					"SELECT physical_name FROM tap_user.tables"
					" WHERE user_name=%(user)s and table_name=%(tablename)s",
					{"user": user, "tablename": self.tableName}))[0]
			except StopIteration:
				raise svcs.UnknownURI(f"No such uploaded table for user {user}")

			dropUserUploadedTable(conn, user, self.tableName)

		return self.serveImmediate(request,
			"text/plain; charset=utf-8", f"Dropped user table {self.tableName}\n")

	def getChild(self, name, request):
		name = name.decode("ascii")
		if name=="destruction":
			return DestructionTimeManager(self.tableName)
		elif name=="index":
			return IndexManager(self.tableName)
		else:
			raise svcs.UnknownURI("Unsupported child resource: {name}")


class PersistentUploadRenderer(vodal.UnifiedDALRenderer):
	"""A renderer for creating and deleting persistent TAP-queriable tables.

	This really only makes sense on the TAP service.
	"""
	name = "user_tables"
	standardId = "ivo://ivoa.net/std/tap#persistent-uploads-1.2"
	resultType = "text/html"

	def render(self, request):
		user = request.getAuthUser() or "anonymous"
		if user=="anonymous":
			return dali.serveDALIError(
				request,
				svcs.ForbiddenURI("No persistent tables listing for"
					" anonymous users."))

		# this returns a VOSI tableset for the uploads made by this user
		with base.getTableConn() as conn:
			rawTDs = conn.query("SELECT metadata FROM tap_user.tables"
					" WHERE user_name=%(user)s", {"user": user})

		rd = base.caches.getRD("//tap_user")
		tds = []
		for (rawTD,) in rawTDs:
			td = base.parseFromString(rscdef.TableDef, rawTD)
			td.parent = rd
			tds.append(td)

		et = tableset.getTablesetForSchemaCollection(
			[(rd, tds)], rootElement=VTM.tableset)
		request.setHeader("content-type", "text/xml")
		utils.xmlwrite(et, request,
			prolog="<?xml-stylesheet href='/static/xsl/vosi.xsl' type='text/xsl'?>")
		return b""

	def getChild(self, name, request):
		try:
			name = name.decode("ascii", "replace")
			if not USER_TABLE_NAME_PATTERN.match(name):
				raise svcs.UnknownURI("You must give a valid upload table name here")

			with base.getTableConn() as conn:
				if base.UnmanagedQuerier(conn).getTableType("tap_user.tables"
						)!="BASE TABLE":
					raise svcs.ForbiddenURI(
						"Persistent table uploads not enabled on this instance.")

			return PersistentUploadManager(name)
		except Exception as exc:
			if not isinstance(exc, svcs.Error):
				base.ui.notifyError(exc)
			return dali.DALIErrorResource(exc)
