1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384
|
# coding: utf-8
from __future__ import absolute_import, division, print_function, unicode_literals
import sys
from collections import OrderedDict
from itertools import chain
from click.utils import LazyFile
from pip._internal.req.constructors import install_req_from_line
from pip._internal.utils.misc import redact_auth_from_url
from pip._internal.vcs import is_url
from pip._vendor import six
from pip._vendor.six.moves import shlex_quote
from .click import style
UNSAFE_PACKAGES = {"setuptools", "distribute", "pip"}
COMPILE_EXCLUDE_OPTIONS = {
"--dry-run",
"--quiet",
"--rebuild",
"--upgrade",
"--upgrade-package",
"--verbose",
"--cache-dir",
"--no-reuse-hashes",
}
def key_from_ireq(ireq):
"""Get a standardized key for an InstallRequirement."""
if ireq.req is None and ireq.link is not None:
return str(ireq.link)
else:
return key_from_req(ireq.req)
def key_from_req(req):
"""Get an all-lowercase version of the requirement's name."""
if hasattr(req, "key"):
# from pkg_resources, such as installed dists for pip-sync
key = req.key
else:
# from packaging, such as install requirements from requirements.txt
key = req.name
key = key.replace("_", "-").lower()
return key
def comment(text):
return style(text, fg="green")
def make_install_requirement(name, version, extras, constraint=False):
# If no extras are specified, the extras string is blank
extras_string = ""
if extras:
# Sort extras for stability
extras_string = "[{}]".format(",".join(sorted(extras)))
return install_req_from_line(
str("{}{}=={}".format(name, extras_string, version)), constraint=constraint
)
def is_url_requirement(ireq):
"""
Return True if requirement was specified as a path or URL.
ireq.original_link will have been set by InstallRequirement.__init__
"""
return bool(ireq.original_link)
def format_requirement(ireq, marker=None, hashes=None):
"""
Generic formatter for pretty printing InstallRequirements to the terminal
in a less verbose way than using its `__str__` method.
"""
if ireq.editable:
line = "-e {}".format(ireq.link.url)
elif is_url_requirement(ireq):
line = ireq.link.url
else:
line = str(ireq.req).lower()
if marker:
line = "{} ; {}".format(line, marker)
if hashes:
for hash_ in sorted(hashes):
line += " \\\n --hash={}".format(hash_)
return line
def format_specifier(ireq):
"""
Generic formatter for pretty printing the specifier part of
InstallRequirements to the terminal.
"""
# TODO: Ideally, this is carried over to the pip library itself
specs = ireq.specifier if ireq.req is not None else []
specs = sorted(specs, key=lambda x: x.version)
return ",".join(str(s) for s in specs) or "<any>"
def is_pinned_requirement(ireq):
"""
Returns whether an InstallRequirement is a "pinned" requirement.
An InstallRequirement is considered pinned if:
- Is not editable
- It has exactly one specifier
- That specifier is "=="
- The version does not contain a wildcard
Examples:
django==1.8 # pinned
django>1.8 # NOT pinned
django~=1.8 # NOT pinned
django==1.* # NOT pinned
"""
if ireq.editable:
return False
if ireq.req is None or len(ireq.specifier) != 1:
return False
spec = next(iter(ireq.specifier))
return spec.operator in {"==", "==="} and not spec.version.endswith(".*")
def as_tuple(ireq):
"""
Pulls out the (name: str, version:str, extras:(str)) tuple from
the pinned InstallRequirement.
"""
if not is_pinned_requirement(ireq):
raise TypeError("Expected a pinned InstallRequirement, got {}".format(ireq))
name = key_from_ireq(ireq)
version = next(iter(ireq.specifier)).version
extras = tuple(sorted(ireq.extras))
return name, version, extras
def flat_map(fn, collection):
"""Map a function over a collection and flatten the result by one-level"""
return chain.from_iterable(map(fn, collection))
def lookup_table(values, key=None, keyval=None, unique=False, use_lists=False):
"""
Builds a dict-based lookup table (index) elegantly.
Supports building normal and unique lookup tables. For example:
>>> assert lookup_table(
... ['foo', 'bar', 'baz', 'qux', 'quux'], lambda s: s[0]) == {
... 'b': {'bar', 'baz'},
... 'f': {'foo'},
... 'q': {'quux', 'qux'}
... }
For key functions that uniquely identify values, set unique=True:
>>> assert lookup_table(
... ['foo', 'bar', 'baz', 'qux', 'quux'], lambda s: s[0],
... unique=True) == {
... 'b': 'baz',
... 'f': 'foo',
... 'q': 'quux'
... }
For the values represented as lists, set use_lists=True:
>>> assert lookup_table(
... ['foo', 'bar', 'baz', 'qux', 'quux'], lambda s: s[0],
... use_lists=True) == {
... 'b': ['bar', 'baz'],
... 'f': ['foo'],
... 'q': ['qux', 'quux']
... }
The values of the resulting lookup table will be lists, not sets.
For extra power, you can even change the values while building up the LUT.
To do so, use the `keyval` function instead of the `key` arg:
>>> assert lookup_table(
... ['foo', 'bar', 'baz', 'qux', 'quux'],
... keyval=lambda s: (s[0], s[1:])) == {
... 'b': {'ar', 'az'},
... 'f': {'oo'},
... 'q': {'uux', 'ux'}
... }
"""
if keyval is None:
if key is None:
def keyval(v):
return v
else:
def keyval(v):
return (key(v), v)
if unique:
return dict(keyval(v) for v in values)
lut = {}
for value in values:
k, v = keyval(value)
try:
s = lut[k]
except KeyError:
if use_lists:
s = lut[k] = list()
else:
s = lut[k] = set()
if use_lists:
s.append(v)
else:
s.add(v)
return dict(lut)
def dedup(iterable):
"""Deduplicate an iterable object like iter(set(iterable)) but
order-preserved.
"""
return iter(OrderedDict.fromkeys(iterable))
def name_from_req(req):
"""Get the name of the requirement"""
if hasattr(req, "project_name"):
# from pkg_resources, such as installed dists for pip-sync
return req.project_name
else:
# from packaging, such as install requirements from requirements.txt
return req.name
def fs_str(string):
"""
Convert given string to a correctly encoded filesystem string.
On Python 2, if the input string is unicode, converts it to bytes
encoded with the filesystem encoding.
On Python 3 returns the string as is, since Python 3 uses unicode
paths and the input string shouldn't be bytes.
:type string: str|unicode
:rtype: str
"""
if isinstance(string, str):
return string
if isinstance(string, bytes):
raise TypeError("fs_str() argument must not be bytes")
return string.encode(_fs_encoding)
_fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
def get_hashes_from_ireq(ireq):
"""
Given an InstallRequirement, return a list of string hashes in
the format "{algorithm}:{hash}". Return an empty list if there are no hashes
in the requirement options.
"""
result = []
for algorithm, hexdigests in ireq.hash_options.items():
for hash_ in hexdigests:
result.append("{}:{}".format(algorithm, hash_))
return result
def force_text(s):
"""
Return a string representing `s`.
"""
if s is None:
return ""
if not isinstance(s, six.string_types):
return six.text_type(s)
return s
def get_compile_command(click_ctx):
"""
Returns a normalized compile command depending on cli context.
The command will be normalized by:
- expanding options short to long
- removing values that are already default
- sorting the arguments
- removing one-off arguments like '--upgrade'
- removing arguments that don't change build behaviour like '--verbose'
"""
from piptools.scripts.compile import cli
# Map of the compile cli options (option name -> click.Option)
compile_options = {option.name: option for option in cli.params}
left_args = []
right_args = []
for option_name, value in click_ctx.params.items():
option = compile_options[option_name]
# Collect variadic args separately, they will be added
# at the end of the command later
if option.nargs < 0:
# These will necessarily be src_files
# Re-add click-stripped '--' if any start with '-'
if any(val.startswith("-") and val != "-" for val in value):
right_args.append("--")
right_args.extend([shlex_quote(force_text(val)) for val in value])
continue
# Get the latest option name (usually it'll be a long name)
option_long_name = option.opts[-1]
# Exclude one-off options (--upgrade/--upgrade-package/--rebuild/...)
# or options that don't change compile behaviour (--verbose/--dry-run/...)
if option_long_name in COMPILE_EXCLUDE_OPTIONS:
continue
# Skip options without a value
if option.default is None and not value:
continue
# Skip options with a default value
if option.default == value:
continue
# Use a file name for file-like objects
if isinstance(value, LazyFile):
value = value.name
# Convert value to the list
if not isinstance(value, (tuple, list)):
value = [value]
for val in value:
# Flags don't have a value, thus add to args true or false option long name
if option.is_flag:
# If there are false-options, choose an option name depending on a value
if option.secondary_opts:
# Get the latest false-option
secondary_option_long_name = option.secondary_opts[-1]
arg = option_long_name if val else secondary_option_long_name
# There are no false-options, use true-option
else:
arg = option_long_name
left_args.append(shlex_quote(arg))
# Append to args the option with a value
else:
if isinstance(val, six.string_types) and is_url(val):
val = redact_auth_from_url(val)
if option.name == "pip_args":
# shlex_quote would produce functional but noisily quoted results,
# e.g. --pip-args='--cache-dir='"'"'/tmp/with spaces'"'"''
# Instead, we try to get more legible quoting via repr:
left_args.append(
"{option}={value}".format(
option=option_long_name, value=repr(fs_str(force_text(val)))
)
)
else:
left_args.append(
"{option}={value}".format(
option=option_long_name, value=shlex_quote(force_text(val))
)
)
return " ".join(["pip-compile"] + sorted(left_args) + sorted(right_args))
|