File: argparser.py

package info (click to toggle)
cwltool 1.0.20181217162649%2Bdfsg-10
  • links: PTS, VCS
  • area: main
  • in suites: buster
  • size: 2,632 kB
  • sloc: python: 11,008; makefile: 153; sh: 22
file content (427 lines) | stat: -rw-r--r-- 21,191 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
"""Command line argument parsing for cwltool."""
from __future__ import absolute_import, print_function

import argparse
import os
from typing import (Any, AnyStr, Dict, List, MutableMapping, MutableSequence,
                    Optional, Sequence, Union, cast)

from schema_salad.ref_resolver import file_uri
from typing_extensions import Text  # pylint: disable=unused-import
# move to a regular typing import when Python 3.3-3.6 is no longer supported

from .loghandler import _logger
from .process import Process, shortname  # pylint: disable=unused-import
from .resolver import ga4gh_tool_registries
from .software_requirements import SOFTWARE_REQUIREMENTS_ENABLED
from .utils import DEFAULT_TMP_PREFIX


def arg_parser():  # type: () -> argparse.ArgumentParser
    parser = argparse.ArgumentParser(
        description='Reference executor for Common Workflow Language standards.')
    parser.add_argument("--basedir", type=Text)
    parser.add_argument("--outdir", type=Text, default=os.path.abspath('.'),
                        help="Output directory, default current directory")

    parser.add_argument("--parallel", action="store_true", default=False,
                        help="[experimental] Run jobs in parallel. ")
    envgroup = parser.add_mutually_exclusive_group()
    envgroup.add_argument("--preserve-environment", type=Text, action="append",
                          help="Preserve specific environment variable when "
                          "running CommandLineTools.  May be provided multiple "
                          "times.", metavar="ENVVAR", default=["PATH"],
                          dest="preserve_environment")
    envgroup.add_argument("--preserve-entire-environment", action="store_true",
                          help="Preserve all environment variable when running "
                          "CommandLineTools.", default=False,
                          dest="preserve_entire_environment")

    exgroup = parser.add_mutually_exclusive_group()
    exgroup.add_argument("--rm-container", action="store_true", default=True,
                         help="Delete Docker container used by jobs after they exit (default)",
                         dest="rm_container")

    exgroup.add_argument(
        "--leave-container", action="store_false", default=True,
        help="Do not delete Docker container used by jobs after they exit",
        dest="rm_container")

    cidgroup = parser.add_argument_group(
        "Options for recording the Docker container identifier into a file.")
    # Disabled as containerid is now saved by default
    cidgroup.add_argument("--record-container-id", action="store_true",
                          default=False,
                          help = argparse.SUPPRESS,
                          dest="record_container_id")

    cidgroup.add_argument(
        "--cidfile-dir", type=Text, help="Store the Docker "
        "container ID into a file in the specified directory.",
        default=None, dest="cidfile_dir")

    cidgroup.add_argument(
        "--cidfile-prefix", type=Text,
        help="Specify a prefix to the container ID filename. "
        "Final file name will be followed by a timestamp. "
        "The default is no prefix.",
        default=None, dest="cidfile_prefix")

    parser.add_argument("--tmpdir-prefix", type=Text,
                        help="Path prefix for temporary directories",
                        default=DEFAULT_TMP_PREFIX)

    exgroup = parser.add_mutually_exclusive_group()
    exgroup.add_argument("--tmp-outdir-prefix", type=Text,
                         help="Path prefix for intermediate output directories",
                         default=DEFAULT_TMP_PREFIX)

    exgroup.add_argument(
        "--cachedir", type=Text, default="",
        help="Directory to cache intermediate workflow outputs to avoid recomputing steps.")

    exgroup = parser.add_mutually_exclusive_group()
    exgroup.add_argument("--rm-tmpdir", action="store_true", default=True,
                         help="Delete intermediate temporary directories (default)",
                         dest="rm_tmpdir")

    exgroup.add_argument("--leave-tmpdir", action="store_false",
                         default=True, help="Do not delete intermediate temporary directories",
                         dest="rm_tmpdir")

    exgroup = parser.add_mutually_exclusive_group()
    exgroup.add_argument(
        "--move-outputs", action="store_const", const="move", default="move",
        help="Move output files to the workflow output directory and delete "
        "intermediate output directories (default).", dest="move_outputs")

    exgroup.add_argument("--leave-outputs", action="store_const", const="leave", default="move",
                         help="Leave output files in intermediate output directories.",
                         dest="move_outputs")

    exgroup.add_argument("--copy-outputs", action="store_const", const="copy", default="move",
                         help="Copy output files to the workflow output directory, don't delete intermediate output directories.",
                         dest="move_outputs")

    exgroup = parser.add_mutually_exclusive_group()
    exgroup.add_argument("--enable-pull", default=True, action="store_true",
                         help="Try to pull Docker images", dest="enable_pull")

    exgroup.add_argument("--disable-pull", default=True, action="store_false",
                         help="Do not try to pull Docker images", dest="enable_pull")

    parser.add_argument("--rdf-serializer",
                        help="Output RDF serialization format used by --print-rdf (one of turtle (default), n3, nt, xml)",
                        default="turtle")

    parser.add_argument("--eval-timeout",
                        help="Time to wait for a Javascript expression to evaluate before giving an error, default 20s.",
                        type=float,
                        default=20)
    parser.add_argument("--provenance", help=argparse.SUPPRESS)
    exgroup = parser.add_mutually_exclusive_group()
    exgroup.add_argument("--print-rdf", action="store_true",
                         help="Print corresponding RDF graph for workflow and exit")
    exgroup.add_argument("--print-dot", action="store_true",
                         help="Print workflow visualization in graphviz format and exit")
    exgroup.add_argument("--print-pre", action="store_true", help="Print CWL document after preprocessing.")
    exgroup.add_argument("--print-deps", action="store_true", help="Print CWL document dependencies.")
    exgroup.add_argument("--print-input-deps", action="store_true", help="Print input object document dependencies.")
    exgroup.add_argument("--pack", action="store_true", help="Combine components into single document and print.")
    exgroup.add_argument("--version", action="store_true", help="Print version and exit")
    exgroup.add_argument("--validate", action="store_true", help="Validate CWL document only.")
    exgroup.add_argument("--print-supported-versions", action="store_true", help="Print supported CWL specs.")
    exgroup.add_argument("--print-subgraph", action="store_true",
                         help="Print workflow subgraph that will execute "
                         "(can combine with --target)")
    exgroup.add_argument("--print-targets", action="store_true", help="Print targets (output parameters)")

    exgroup = parser.add_mutually_exclusive_group()
    exgroup.add_argument("--strict", action="store_true",
                         help="Strict validation (unrecognized or out of place fields are error)",
                         default=True, dest="strict")
    exgroup.add_argument("--non-strict", action="store_false", help="Lenient validation (ignore unrecognized fields)",
                         default=True, dest="strict")

    parser.add_argument("--skip-schemas", action="store_true",
            help="Skip loading of schemas", default=False, dest="skip_schemas")

    exgroup = parser.add_mutually_exclusive_group()
    exgroup.add_argument("--verbose", action="store_true", help="Default logging")
    exgroup.add_argument("--quiet", action="store_true", help="Only print warnings and errors.")
    exgroup.add_argument("--debug", action="store_true", help="Print even more logging")

    parser.add_argument(
        "--strict-memory-limit", action="store_true", help="When running with "
        "software containers and the Docker engine, pass either the "
        "calculated memory allocation from ResourceRequirements or the "
        "default of 1 gigabyte to Docker's --memory option.")

    parser.add_argument("--timestamps", action="store_true", help="Add "
                        "timestamps to the errors, warnings, and "
                        "notifications.")
    parser.add_argument("--js-console", action="store_true", help="Enable javascript console output")
    parser.add_argument("--disable-js-validation", help=argparse.SUPPRESS)
    dockergroup = parser.add_mutually_exclusive_group()
    dockergroup.add_argument("--user-space-docker-cmd", metavar="CMD",
                        help="(Linux/OS X only) Specify a user space docker "
                        "command (like udocker or dx-docker) that will be "
                        "used to call 'pull' and 'run'")
    dockergroup.add_argument("--singularity", action="store_true",
                             default=False, help="[experimental] Use "
                             "Singularity runtime for running containers. "
                             "Requires Singularity v2.3.2+ and Linux with kernel "
                             "version v3.18+ or with overlayfs support "
                             "backported.")
    dockergroup.add_argument("--no-container", action="store_false",
                             default=True, help="Do not execute jobs in a "
                             "Docker container, even when `DockerRequirement` "
                             "is specified under `hints`.",
                             dest="use_container")

    dependency_resolvers_configuration_help = argparse.SUPPRESS
    dependencies_directory_help = argparse.SUPPRESS
    use_biocontainers_help = argparse.SUPPRESS
    conda_dependencies = argparse.SUPPRESS

    if SOFTWARE_REQUIREMENTS_ENABLED:
        dependency_resolvers_configuration_help = "Dependency resolver configuration file describing how to adapt 'SoftwareRequirement' packages to current system."
        dependencies_directory_help = "Defaut root directory used by dependency resolvers configuration."
        use_biocontainers_help = "Use biocontainers for tools without an explicitly annotated Docker container."
        conda_dependencies = "Short cut to use Conda to resolve 'SoftwareRequirement' packages."

    parser.add_argument("--beta-dependency-resolvers-configuration", default=None, help=dependency_resolvers_configuration_help)
    parser.add_argument("--beta-dependencies-directory", default=None, help=dependencies_directory_help)
    parser.add_argument("--beta-use-biocontainers", default=None, help=use_biocontainers_help, action="store_true")
    parser.add_argument("--beta-conda-dependencies", default=None, help=conda_dependencies, action="store_true")

    parser.add_argument("--tool-help", action="store_true", help="Print command line help for tool")

    parser.add_argument("--relative-deps", choices=['primary', 'cwd'],
                        default="primary", help="When using --print-deps, print paths "
                                                "relative to primary file or current working directory.")

    parser.add_argument("--enable-dev", action="store_true",
                        help="Enable loading and running development versions "
                             "of CWL spec.", default=False)

    parser.add_argument("--enable-ext", action="store_true",
                        help="Enable loading and running cwltool extensions "
                             "to CWL spec.", default=False)

    parser.add_argument("--default-container",
                        help="Specify a default docker container that will be used if the workflow fails to specify one.")
    parser.add_argument("--no-match-user", action="store_true",
                        help="Disable passing the current uid to `docker run --user`")
    parser.add_argument("--custom-net", type=Text,
                        help="Passed to `docker run` as the '--net' "
                             "parameter when NetworkAccess is true.")
    parser.add_argument("--disable-validate", dest="do_validate",
                        action="store_false", default=True,
                        help=argparse.SUPPRESS)

    exgroup = parser.add_mutually_exclusive_group()
    exgroup.add_argument("--enable-ga4gh-tool-registry", action="store_true", help="Enable resolution using GA4GH tool registry API",
                        dest="enable_ga4gh_tool_registry", default=True)
    exgroup.add_argument("--disable-ga4gh-tool-registry", action="store_false", help="Disable resolution using GA4GH tool registry API",
                        dest="enable_ga4gh_tool_registry", default=True)

    parser.add_argument("--add-ga4gh-tool-registry", action="append", help="Add a GA4GH tool registry endpoint to use for resolution, default %s" % ga4gh_tool_registries,
                        dest="ga4gh_tool_registries", default=[])

    parser.add_argument("--on-error",
                        help="Desired workflow behavior when a step fails.  One of 'stop' (do not submit any more steps) or "
                        "'continue' (may submit other steps that are not downstream from the error). Default is 'stop'.",
                        default="stop", choices=("stop", "continue"))

    exgroup = parser.add_mutually_exclusive_group()
    exgroup.add_argument("--compute-checksum", action="store_true", default=True,
                         help="Compute checksum of contents while collecting outputs",
                         dest="compute_checksum")
    exgroup.add_argument("--no-compute-checksum", action="store_false",
                         help="Do not compute checksum of contents while collecting outputs",
                         dest="compute_checksum")

    parser.add_argument("--relax-path-checks", action="store_true",
                        default=False, help="Relax requirements on path names to permit "
                        "spaces and hash characters.", dest="relax_path_checks")
    exgroup.add_argument("--make-template", action="store_true",
                         help="Generate a template input object")

    parser.add_argument("--force-docker-pull", action="store_true",
                        default=False, help="Pull latest docker image even if"
                                            " it is locally present", dest="force_docker_pull")
    parser.add_argument("--no-read-only", action="store_true",
                        default=False, help="Do not set root directory in the"
                                            " container as read-only", dest="no_read_only")

    parser.add_argument("--overrides", type=str,
                        default=None, help="Read process requirement overrides from file.")

    parser.add_argument("--target", "-t", action="append",
                        help="Only execute steps that contribute to "
                        "listed targets (can provide more than once).")

    parser.add_argument("workflow", type=Text, nargs="?", default=None,
            metavar='cwl_document', help="path or URL to a CWL Workflow, "
            "CommandLineTool, or ExpressionTool. If the `inputs_object` has a "
            "`cwl:tool` field indicating the path or URL to the cwl_document, "
            " then the `workflow` argument is optional.")
    parser.add_argument("job_order", nargs=argparse.REMAINDER,
            metavar='inputs_object', help="path or URL to a YAML or JSON "
            "formatted description of the required input values for the given "
            "`cwl_document`.")

    return parser


def get_default_args():
    # type: () -> Dict[str, Any]
    """
    Get default values of cwltool's command line options
    """
    ap = arg_parser()
    args = ap.parse_args([])
    return vars(args)


class FSAction(argparse.Action):
    objclass = None  # type: Text

    def __init__(self, option_strings, dest, nargs=None, **kwargs):
        # type: (List[Text], Text, Any, **Any) -> None
        if nargs is not None:
            raise ValueError("nargs not allowed")
        super(FSAction, self).__init__(option_strings, dest, **kwargs)

    def __call__(self, parser, namespace, values, option_string=None):
        # type: (argparse.ArgumentParser, argparse.Namespace, Union[AnyStr, Sequence[Any], None], AnyStr) -> None
        setattr(namespace,
                self.dest,  # type: ignore
                {"class": self.objclass,
                 "location": file_uri(str(os.path.abspath(cast(AnyStr, values))))})


class FSAppendAction(argparse.Action):
    objclass = None  # type: Text

    def __init__(self, option_strings, dest, nargs=None, **kwargs):
        # type: (List[Text], Text, Any, **Any) -> None
        if nargs is not None:
            raise ValueError("nargs not allowed")
        super(FSAppendAction, self).__init__(option_strings, dest, **kwargs)

    def __call__(self, parser, namespace, values, option_string=None):
        # type: (argparse.ArgumentParser, argparse.Namespace, Union[AnyStr, Sequence[Any], None], AnyStr) -> None
        g = getattr(namespace,
                    self.dest  # type: ignore
                    )
        if not g:
            g = []
            setattr(namespace,
                    self.dest,  # type: ignore
                    g)
        g.append(
            {"class": self.objclass,
             "location": file_uri(str(os.path.abspath(cast(AnyStr, values))))})


class FileAction(FSAction):
    objclass = "File"


class DirectoryAction(FSAction):
    objclass = "Directory"


class FileAppendAction(FSAppendAction):
    objclass = "File"


class DirectoryAppendAction(FSAppendAction):
    objclass = "Directory"


def add_argument(toolparser, name, inptype, records, description="",
                 default=None):
    # type: (argparse.ArgumentParser, Text, Any, List[Text], Text, Any) -> None
    if len(name) == 1:
        flag = "-"
    else:
        flag = "--"

    required = default is None
    if isinstance(inptype, MutableSequence):
        if inptype[0] == "null":
            required = False
            if len(inptype) == 2:
                inptype = inptype[1]
            else:
                _logger.debug(u"Can't make command line argument from %s", inptype)
                return None

    ahelp = description.replace("%", "%%")
    action = None  # type: Optional[Union[argparse.Action, Text]]
    atype = None  # type: Any

    if inptype == "File":
        action = cast(argparse.Action, FileAction)
    elif inptype == "Directory":
        action = cast(argparse.Action, DirectoryAction)
    elif isinstance(inptype, MutableMapping) and inptype["type"] == "array":
        if inptype["items"] == "File":
            action = cast(argparse.Action, FileAppendAction)
        elif inptype["items"] == "Directory":
            action = cast(argparse.Action, DirectoryAppendAction)
        else:
            action = "append"
    elif isinstance(inptype, MutableMapping) and inptype["type"] == "enum":
        atype = Text
    elif isinstance(inptype, MutableMapping) and inptype["type"] == "record":
        records.append(name)
        for field in inptype['fields']:
            fieldname = name + "." + shortname(field['name'])
            fieldtype = field['type']
            fielddescription = field.get("doc", "")
            add_argument(
                toolparser, fieldname, fieldtype, records,
                fielddescription)
        return
    elif inptype == "string":
        atype = Text
    elif inptype == "int":
        atype = int
    elif inptype == "double":
        atype = float
    elif inptype == "float":
        atype = float
    elif inptype == "boolean":
        action = "store_true"
    else:
        _logger.debug(u"Can't make command line argument from %s", inptype)
        return None

    if inptype != "boolean":
        typekw = {'type': atype}
    else:
        typekw = {}

    toolparser.add_argument(  # type: ignore
        flag + name, required=required, help=ahelp, action=action,
        default=default, **typekw)


def generate_parser(toolparser, tool, namemap, records):
    # type: (argparse.ArgumentParser, Process, Dict[Text, Text], List[Text]) -> argparse.ArgumentParser
    toolparser.add_argument("job_order", nargs="?", help="Job input json file")
    namemap["job_order"] = "job_order"

    for inp in tool.tool["inputs"]:
        name = shortname(inp["id"])
        namemap[name.replace("-", "_")] = name
        inptype = inp["type"]
        description = inp.get("doc", "")
        default = inp.get("default", None)
        add_argument(toolparser, name, inptype, records, description, default)

    return toolparser