File: dali.py

package info (click to toggle)
gavodachs 2.3%2Bdfsg-3
  • links: PTS, VCS
  • area: main
  • in suites: bullseye
  • size: 7,260 kB
  • sloc: python: 58,359; xml: 8,882; javascript: 3,453; ansic: 661; sh: 158; makefile: 22
file content (236 lines) | stat: -rw-r--r-- 7,261 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
"""
Common code supporting functionality described in DALI.
"""

#c Copyright 2008-2020, the GAVO project
#c
#c This program is free software, covered by the GNU GPL.  See the
#c COPYING file in the source distribution.


import os

from twisted.web import resource
from twisted.web import server

from gavo import base
from gavo import formats
from gavo import utils
from gavo.base import sqlmunge
from gavo.votable import V

# Upload stuff -- note that TAP uploads are somewhat different from the
# DALI ones, as TAP allows multiple uploads in one string. Hence, we
# have a different (and simpler) implementation here.


def getUploadKeyFor(inputKey):
	"""returns an input key for file items in "PQL".

	This is actually specified by DALI.  In that scheme, the parameter
	is always called UPLOAD (there can thus only be one such parameter,
	but it can be present multiple times if necessary, except we've
	not figured out how to do the description right in that case).

	It contains a comma-separated pair of (key,source) pairs, where
	source is a URL; there's a special scheme param: for referring to 
	inline uploads by their name.

	This is used exclusively for metadata generation, and there's special
	code to handle it there.  There's also special code in 
	inputdef.ContextGrammar to magcially make UPLOAD into the file 
	parameters we use within DaCHS.  Sigh.
	"""
	return inputKey.change(
		name="INPUT:UPLOAD",
		type="pql-upload",
		description="An upload of the form '%s,URL'; the input for this"
			" parameter is then taken from URL, which may be param:name"
			" for pulling the content from the inline upload name.  Purpose"
			" of the upload: %s"%(inputKey.name, inputKey.description),
		values=None)

# pql-uploads never contribute to SQL queries
sqlmunge.registerSQLFactory("pql-upload", lambda field, val, sqlPars: None)


def parseUploadString(uploadString):
	"""returns resourceName, uploadSource from a DALI upload string.
	"""
	try:
		destName, uploadSource = uploadString.split(",", 1)
	except (TypeError, ValueError):
		raise base.ValidationError("Invalid UPLOAD string",
			"UPLOAD", hint="UPLOADs look like my_upload,http://foo.bar/up"
				" or inline_upload,param:foo.")
	return destName, uploadSource


class URLUpload(object):
	"""a somewhat FieldStorage-compatible facade to an upload coming from
	a URL.

	The filename for now is the complete upload URL, but that's likely
	to change.
	"""
	def __init__(self, uploadURL, uploadName):
		self.uploadURL, self.name = uploadURL, uploadName
		self.file = utils.urlopenRemote(self.uploadURL)
		self.filename = uploadURL
		self.headers = self.file.info()
		major, minor, parSet = formats.getMIMEKey(
			self.headers.get("content-type", "*/*"))
		self.type = "%s/%s"%(major, minor)
		self.type_options = dict(parSet)
	
	@property
	def value(self):
		try:
			f = utils.urlopenRemote(self.uploadURL)
			return f.read()
		finally:
			f.close()


def iterUploads(request):
	"""iterates over DALI uploads in request.

	This yields pairs of (file name, file object), where file name
	is the file name requested (sanitized to have no slashes and non-ASCII).
	The UPLOAD and inline-file keys are removed from request's args
	member.  The file object is a cgi-style thing with file, filename,
	etc. attributes.
	"""
	# UWS auto-downcases things (it probably shouldn't)
	uploads = request.strargs.pop("UPLOAD", [])
	if not uploads:
		return
		
	for uploadString in uploads:
		paramName, uploadSource = parseUploadString(uploadString)
		try:
			if uploadSource.startswith("param:"):
				fileKey = uploadSource[6:]
				upload = request.fields[fileKey]
			else:
				upload = URLUpload(uploadSource, paramName)

			yield paramName, upload
		except (KeyError, AttributeError):
			raise base.ui.logOldExc(base.ValidationError(
				"%s references a non-existing"
				" file upload."%uploadSource, "UPLOAD", 
				hint="If you pass UPLOAD=foo,param:x,"
				" you must pass a file upload under the key x."))


def mangleUploads(request):
	"""manipulates request such that DALI UPLOADs appear in strargs
	as pairs of name and file/URLUpload

	These are as in normal CGI: uploads are under "their names" (with
	DALI uploads, the resource names), with values being pairs of
	some name and a FieldStorage-compatible thing having name, filename, value,
	file, type, type_options, and headers.

	There's extra code for this in TAP because regrettably TAP works a bit
	differently from this.
	"""
	try:
		for paramName, fObject in iterUploads(request):
			request.strargs[paramName] = (
				utils.defuseFileName(fObject.filename), fObject.file)
	except base.Error:
		raise
	except Exception as ex:
		raise base.ui.logOldExc(
			base.ValidationError("Processing upload failed: %s"%ex,
				"UPLOAD", hint="If this was a URL upload, perhaps I"
				" just can't see the machine the URL is pointing to?"))
			

def writeUploadBytesTo(request, destDir):
	"""writes a file corresponding to a DALI upload to destDir.

	For the sake uws.UploadParameter, we return the names of the
	files we've been creating.
	"""
	created = []

	if not os.path.isdir(destDir):
		os.mkdir(destDir)

	for fName, fObject in iterUploads(request):
		with open(os.path.join(destDir, fName), "wb") as f:
			utils.cat(fObject.file, f)
		created.append(fName)

	return created


def getDALIError(errInfo, queryStatus="ERROR"):
	"""returns  a DALI-compliant error VOTable from an error info.

	errInfo can either be an exception, a UWS error info (a
	dict with keys msg, type, and hint), or a pre-formatted string.
	"""
	if isinstance(errInfo, Exception):
		errInfo = {
			"msg": str(errInfo),
			"type": errInfo.__class__.__name__,
			"hint": getattr(errInfo, "hint", None)}
	
	elif isinstance(errInfo, str):
		errInfo = {
			"msg": errInfo,
			"type": "ReportableError",
			"hint": None}

	res = V.RESOURCE(type="results") [
		V.INFO(name="QUERY_STATUS", value=queryStatus)[
			errInfo["msg"]]]
	if errInfo["hint"]:
		res[V.INFO(name="HINT", value="HINT")[
			errInfo["hint"]]]

	return V.VOTABLE[res]


def serveDALIError(request, errInfo, httpStatus=200, queryStatus="ERROR"):
	"""serves a DALI-compliant error message from errInfo.

	See getDALIError for what errInfo can be.

	This closes the request and returns NOT_DONE_YET, so you
	can write ``return serverDALIError`` in normal render functions

	DALI isn't quite clear on what httpCode ought to be, and protocols
	sometimes actually require it to be 200 even of errors (sigh).
	To return non-200, pass an httpCode
	"""
	try:
		request.setHeader("content-type", "text/xml")
		errVOT = getDALIError(errInfo, queryStatus)
		request.setResponseCode(httpStatus)
		request.write(
			utils.xmlrender(errVOT,
				prolog="<?xml-stylesheet href='/static/xsl/"
					"dalierror-to-html.xsl' type='text/xsl'?>"))
	finally:
		request.finish()
		return server.NOT_DONE_YET


class DALIErrorResource(resource.Resource):
	"""A DALI error (i.e., an INFO in a VOTable.

	This is constructed with an exception or a UWS errInfo (see getDALIError).

	Use this when you want to return a DALI error from within a getChild
	function.  Otherwise, just use serveDALIError.
	"""
	def __init__(self, errInfo):
		self.errInfo = errInfo
	
	def render(self, request):
		return serveDALIError(request, self.errInfo)