<resource resdir="__system" schema="dc">
	<meta name="creationDate">2008-07-24T09:00:00</meta>
	<meta name="subject">astronomical-software</meta>
	<meta name="description">Table-related metadata for
		the tables within this data center.</meta>

	<table id="tablemeta" onDisk="True" system="True"
			dupePolicy="overwrite">
		<meta name="description">A table mapping table names and schemas to
			the resource descriptors they come from and whether they are open
			to ADQL queries.

			This is used wherever DaCHS needs to go from a database name to the
			resource description, e.g., when generating tableinfo.

			The table is maintained through gavo imp; to force things out
			of here, there's gavo drop (for RDs; use -f if the RD is gone or
			moved away) or gavo purge (for single tables).
			</meta>

		<primary>tableName</primary>

		<column name="tableName" description="Fully qualified table name"
			type="text" verbLevel="1"/>
		<column name="sourceRD" type="text"
			description="Id of the resource descriptor containing the
				table's definition"
			tablehead="RD" verbLevel="15"/>
		<column name="tableDesc" type="text"
			description="Description of the table content"
			tablehead="Table desc." verbLevel="1"/>
		<column name="resDesc" type="text"
			description="Description of the resource this table is part of"
			tablehead="Res desc." verbLevel="15"/>
		<column name="adql" type="boolean" required="True"
			description="True if this table may be accessed using ADQL"
			verbLevel="30"/>
		<column name="nrows" type="bigint"
			description="Estimated number of rows in the table.">
			<values nullLiteral="-1"/>
		</column>
	</table>

	<table id="simple_col_stats" onDisk="True" system="True">
		<meta name="description">Simple (one-column) statistics of
			orderable columns

			This is usually filled by dachs limits, which might use estimates rather
			than actual statistics for large tables.  Also, values elements on the
			column definitions themselves override what may be given here.
		</meta>
		<primary>tableName, column_name</primary>
		<index columns="tableName"/>
		<foreignKey inTable="tablemeta" source="tableName"/>

		<column original="tablemeta.tableName"/>

		<column name="column_name" type="text"
			description="Name of the column in question, DaCHS internal
				representation."/>

		<column name="min_value" type="text"
			description="Minimal value found in this column, tabledata
				representation."/>
		<column name="max_value" type="text"
			description="Maximal value found in this column, tabledata
				representation."/>

		<column name="percentile03" type="text"
			description="Value at the 3rd percentile of this column
				(representative of the lower bound of a '2-sigma interval').
				Tabledata representation."/>
		<column name="median" type="text"
			description="Median value in this column, tabledata representation."/>
		<column name="percentile97" type="text"
			description="Value at the 97th percentile of this column
				(representative of the upper bound of a '2-sigma interval').
				Tabledata representation."/>

		<column name="fill_factor"
			description="Ratio of non-null values in this column to the
				total number of rows in the table."/>
	</table>

	<STREAM id="fill-stats-table">
		<doc>A stream to fill a make element to importers for statistics tables.

		These do all the necessary mangling and extraction in their
		embedded grammars, so it's fairly stereotypical.  The main trick
		is a newSource script that purges previous annotations.
		</doc>
		<script type="newSource" lang="python" notify="False">
			table.connection.execute(
				f"DELETE FROM {table.tableDef.getQName()}"
				" WHERE tableName=%(annotatedTable)s",
				{"annotatedTable": sourceToken.getQName()})
		</script>
		<rowmaker idmaps="*"/>
	</STREAM>

	<data id="import_simple_col_stats" auto="False" updating="True">
		<!-- this data item is used by updateTableLevelMetadata and
		pulls numeric column annotations from the table descriptor
		passed in as sourceToken.
		-->
		<embeddedGrammar>
			<iterator><code>
			for col in self.sourceToken:
				if col.type in base.NUMERIC_TYPES or col.type=="timestamp":
					if hasattr(col, "annotations"):
						row = col.annotations.copy()
						if not row:
							continue
						row["tableName"] = self.sourceToken.getQName()
						row["column_name"] = col.name
						yield row
			</code></iterator>
		</embeddedGrammar>

		<make table="simple_col_stats">
			<FEED source="fill-stats-table"/>
		</make>
	</data>

	<table id="discrete_string_values" onDisk="True" system="True">
		<meta name="description">Discrete values found in string-valued
			columns.

			This is usually filled by dachs limits.  Only columns with
			a statistics property of "enumerate" are considered here.
			Values found here are
		</meta>
		<primary>tableName, column_name</primary>
		<index columns="tableName"/>
		<foreignKey inTable="tablemeta" source="tableName"/>

		<column original="tablemeta.tableName"/>

		<column name="column_name" type="text"
			description="Name of the column in question, DaCHS internal
				representation."/>
		<column name="vals" type="text[]"
			description="Values found in the column."/>
		<column name="freqs" type="real[]"
			description="Relative frequencies of the strings given in values."/>
	</table>
	
	<data id="import_discrete_string_values" auto="False" updating="True">
		<!-- this data item is used by updateTableLevelMetadata to update
		statistics for enumerated string columns.  The input is a td,
		rows are pulled out of columns annostated with discrete_values.
		-->
		<embeddedGrammar>
			<iterator><code>
			for col in self.sourceToken:
				if (hasattr(col, "annotations")
						and "discrete_values" in col.annotations):
					items = sorted(col.annotations["discrete_values"].items())
					yield {
						"vals": [i[0] for i in items],
						"freqs": [i[1] for i in items],
						"tableName": self.sourceToken.getQName(),
						"column_name": col.name,
					}
			</code></iterator>
		</embeddedGrammar>
		<make table="discrete_string_values">
			<FEED source="fill-stats-table"/>
		</make>
	</data>

	<table id="metastore" onDisk="True" system="True" primary="key"
			dupePolicy="overwrite">
		<meta name="description">A table for storing all kinds of key-value
			pairs.  Key starting with an underscore are for use by user RDs.

			Only one pair per key is supported, newer keys overwrite older ones.

			Currently, this is only used for schemaversion, the version of
			the DaCHS system tables as used by gavo upgrade to figure out
			what to change.  gavo upgrade manages this.

			From your code, you can use base.getDBMeta(key) and
			base.setDBMeta(connection, key, value) to put persistent, string-valued
			metadata in here; if you use this, would you tell us your
			use case?
		</meta>

		<column name="quoted/key" type="text" description="A key; everything that
			starts with an underscore is user defined."/>
		<column name="quoted/value" type="text" description="A value; no serialization
			 format is defined here, but you are encouraged to use python literals
			 for non-strings."/>
	</table>

	<rowmaker id="fromColumnList">
		<!-- turns a rawrec with column, colInd, tableName keys into a
		columnmeta row -->
		<apply name="makerow">
			<code>
				column = vars["column"]
				for key in ["description", "unit", "ucd", "tablehead",
						"utype", "verbLevel", "type"]:
					result[key] = getattr(column, key)
				result["displayHint"] = column.getDisplayHintAsString()
				result["fieldName"] = column.name
				result["sourceRD"] = column.parent.rd.sourceId
			</code>
		</apply>
		<map dest="colInd"/>
		<map dest="tableName"/>
	</rowmaker>

	<data id="import">
		<make table="tablemeta"/>
		<make table="simple_col_stats"/>
		<make table="discrete_string_values"/>
		<make table="metastore">
			<script lang="python" type="postCreation" notify="False">
				from gavo.user import upgrade
				from gavo import base
				base.setDBMeta(table.connection,
					"schemaversion", upgrade.CURRENT_SCHEMAVERSION)
			</script>
		</make>
	</data>

	<service id="show" allowed="tableinfo">
		<meta name="shortName">Table infos</meta>
		<meta name="description">This service formats metadata for tables
			within the \getConfig{web}{sitename}</meta>
		<meta name="title">\getConfig{web}{sitename} Table Infos</meta>
		<nullCore/>
	</service>

	<service id="list">
		<meta name="shortName">ADQL tables</meta>
		<meta name="description">An overview over the tables available for ADQL
			querying within the \getConfig{web}{sitename}</meta>
		<meta name="title">\getConfig{web}{sitename} Public Tables</meta>

		<fixedQueryCore
			query="SELECT tableName, tableName as tableinfo, tableDesc, resDesc
				FROM dc.tablemeta WHERE adql ORDER BY tableName">
			<outputTable namePath="tablemeta">
				<outputField original="tableName"/>
				<outputField name="tableinfo" original="tableName">
					<formatter>
						return T.a(href=base.makeSitePath("/__system__/dc_tables/"
							"show/tableinfo/"+urllib.parse.quote(data)))["Table Info"]
					</formatter>
				</outputField>
				<outputField original="tableDesc"/>
				<outputField original="resDesc"/>
			</outputTable>
		</fixedQueryCore>
	</service>
</resource>

