1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236
|
"""
Common code supporting functionality described in DALI.
"""
#c Copyright 2008-2020, the GAVO project
#c
#c This program is free software, covered by the GNU GPL. See the
#c COPYING file in the source distribution.
import os
from twisted.web import resource
from twisted.web import server
from gavo import base
from gavo import formats
from gavo import utils
from gavo.base import sqlmunge
from gavo.votable import V
# Upload stuff -- note that TAP uploads are somewhat different from the
# DALI ones, as TAP allows multiple uploads in one string. Hence, we
# have a different (and simpler) implementation here.
def getUploadKeyFor(inputKey):
"""returns an input key for file items in "PQL".
This is actually specified by DALI. In that scheme, the parameter
is always called UPLOAD (there can thus only be one such parameter,
but it can be present multiple times if necessary, except we've
not figured out how to do the description right in that case).
It contains a comma-separated pair of (key,source) pairs, where
source is a URL; there's a special scheme param: for referring to
inline uploads by their name.
This is used exclusively for metadata generation, and there's special
code to handle it there. There's also special code in
inputdef.ContextGrammar to magcially make UPLOAD into the file
parameters we use within DaCHS. Sigh.
"""
return inputKey.change(
name="INPUT:UPLOAD",
type="pql-upload",
description="An upload of the form '%s,URL'; the input for this"
" parameter is then taken from URL, which may be param:name"
" for pulling the content from the inline upload name. Purpose"
" of the upload: %s"%(inputKey.name, inputKey.description),
values=None)
# pql-uploads never contribute to SQL queries
sqlmunge.registerSQLFactory("pql-upload", lambda field, val, sqlPars: None)
def parseUploadString(uploadString):
"""returns resourceName, uploadSource from a DALI upload string.
"""
try:
destName, uploadSource = uploadString.split(",", 1)
except (TypeError, ValueError):
raise base.ValidationError("Invalid UPLOAD string",
"UPLOAD", hint="UPLOADs look like my_upload,http://foo.bar/up"
" or inline_upload,param:foo.")
return destName, uploadSource
class URLUpload(object):
"""a somewhat FieldStorage-compatible facade to an upload coming from
a URL.
The filename for now is the complete upload URL, but that's likely
to change.
"""
def __init__(self, uploadURL, uploadName):
self.uploadURL, self.name = uploadURL, uploadName
self.file = utils.urlopenRemote(self.uploadURL)
self.filename = uploadURL
self.headers = self.file.info()
major, minor, parSet = formats.getMIMEKey(
self.headers.get("content-type", "*/*"))
self.type = "%s/%s"%(major, minor)
self.type_options = dict(parSet)
@property
def value(self):
try:
f = utils.urlopenRemote(self.uploadURL)
return f.read()
finally:
f.close()
def iterUploads(request):
"""iterates over DALI uploads in request.
This yields pairs of (file name, file object), where file name
is the file name requested (sanitized to have no slashes and non-ASCII).
The UPLOAD and inline-file keys are removed from request's args
member. The file object is a cgi-style thing with file, filename,
etc. attributes.
"""
# UWS auto-downcases things (it probably shouldn't)
uploads = request.strargs.pop("UPLOAD", [])
if not uploads:
return
for uploadString in uploads:
paramName, uploadSource = parseUploadString(uploadString)
try:
if uploadSource.startswith("param:"):
fileKey = uploadSource[6:]
upload = request.fields[fileKey]
else:
upload = URLUpload(uploadSource, paramName)
yield paramName, upload
except (KeyError, AttributeError):
raise base.ui.logOldExc(base.ValidationError(
"%s references a non-existing"
" file upload."%uploadSource, "UPLOAD",
hint="If you pass UPLOAD=foo,param:x,"
" you must pass a file upload under the key x."))
def mangleUploads(request):
"""manipulates request such that DALI UPLOADs appear in strargs
as pairs of name and file/URLUpload
These are as in normal CGI: uploads are under "their names" (with
DALI uploads, the resource names), with values being pairs of
some name and a FieldStorage-compatible thing having name, filename, value,
file, type, type_options, and headers.
There's extra code for this in TAP because regrettably TAP works a bit
differently from this.
"""
try:
for paramName, fObject in iterUploads(request):
request.strargs[paramName] = (
utils.defuseFileName(fObject.filename), fObject.file)
except base.Error:
raise
except Exception as ex:
raise base.ui.logOldExc(
base.ValidationError("Processing upload failed: %s"%ex,
"UPLOAD", hint="If this was a URL upload, perhaps I"
" just can't see the machine the URL is pointing to?"))
def writeUploadBytesTo(request, destDir):
"""writes a file corresponding to a DALI upload to destDir.
For the sake uws.UploadParameter, we return the names of the
files we've been creating.
"""
created = []
if not os.path.isdir(destDir):
os.mkdir(destDir)
for fName, fObject in iterUploads(request):
with open(os.path.join(destDir, fName), "wb") as f:
utils.cat(fObject.file, f)
created.append(fName)
return created
def getDALIError(errInfo, queryStatus="ERROR"):
"""returns a DALI-compliant error VOTable from an error info.
errInfo can either be an exception, a UWS error info (a
dict with keys msg, type, and hint), or a pre-formatted string.
"""
if isinstance(errInfo, Exception):
errInfo = {
"msg": str(errInfo),
"type": errInfo.__class__.__name__,
"hint": getattr(errInfo, "hint", None)}
elif isinstance(errInfo, str):
errInfo = {
"msg": errInfo,
"type": "ReportableError",
"hint": None}
res = V.RESOURCE(type="results") [
V.INFO(name="QUERY_STATUS", value=queryStatus)[
errInfo["msg"]]]
if errInfo["hint"]:
res[V.INFO(name="HINT", value="HINT")[
errInfo["hint"]]]
return V.VOTABLE[res]
def serveDALIError(request, errInfo, httpStatus=200, queryStatus="ERROR"):
"""serves a DALI-compliant error message from errInfo.
See getDALIError for what errInfo can be.
This closes the request and returns NOT_DONE_YET, so you
can write ``return serverDALIError`` in normal render functions
DALI isn't quite clear on what httpCode ought to be, and protocols
sometimes actually require it to be 200 even of errors (sigh).
To return non-200, pass an httpCode
"""
try:
request.setHeader("content-type", "text/xml")
errVOT = getDALIError(errInfo, queryStatus)
request.setResponseCode(httpStatus)
request.write(
utils.xmlrender(errVOT,
prolog="<?xml-stylesheet href='/static/xsl/"
"dalierror-to-html.xsl' type='text/xsl'?>"))
finally:
request.finish()
return server.NOT_DONE_YET
class DALIErrorResource(resource.Resource):
"""A DALI error (i.e., an INFO in a VOTable.
This is constructed with an exception or a UWS errInfo (see getDALIError).
Use this when you want to return a DALI error from within a getChild
function. Otherwise, just use serveDALIError.
"""
def __init__(self, errInfo):
self.errInfo = errInfo
def render(self, request):
return serveDALIError(request, self.errInfo)
|