forked from mirrors/gecko-dev
6d154c1ed3
Vendoring wheels has three benefits: * There's far less files, so Firefox checkouts will be smaller. * It works around `zipp` not allowing `pip install` from extracted source `tar.gz` files. Now, we should be able to use the pip resolver against vendored packages, which will be needed for future mach virtualenv work. * `./mach vendor python` takes far less time to execute. Since we need the raw Python to be available to add to the `sys.path`, we extract the wheels before putting them in tree. Due to the structure of some wheels being less nested than of a source `tar.gz`, `common_virtualenv_packages` needed to be adjusted accordingly. `install_pip_package()` had to be tweaked as well since you can't `pip install` an extracted wheel. So, we "re-bundle" the wheel before installing from a vendored package. Replace python packages with wheels where possible This contains the vendoring changes caused by the last patch. For reviewing, there's a couple things to note: * A bunch of files are deleted, since there's generally less files in a wheel than in a source archive. * There's a new `.dist-info` directory for each extracted wheel, so expect roughly 5 or 6 new files for each wheel'd package. * There should be no source code changes other than moves from package names changing from having `-` to having `_`. Differential Revision: https://phabricator.services.mozilla.com/D116512
384 lines
11 KiB
Python
384 lines
11 KiB
Python
# coding: utf-8
|
|
from __future__ import absolute_import, division, print_function, unicode_literals
|
|
|
|
import sys
|
|
from collections import OrderedDict
|
|
from itertools import chain
|
|
|
|
from click.utils import LazyFile
|
|
from pip._internal.req.constructors import install_req_from_line
|
|
from pip._internal.utils.misc import redact_auth_from_url
|
|
from pip._internal.vcs import is_url
|
|
from pip._vendor import six
|
|
from pip._vendor.six.moves import shlex_quote
|
|
|
|
from .click import style
|
|
|
|
UNSAFE_PACKAGES = {"setuptools", "distribute", "pip"}
|
|
COMPILE_EXCLUDE_OPTIONS = {
|
|
"--dry-run",
|
|
"--quiet",
|
|
"--rebuild",
|
|
"--upgrade",
|
|
"--upgrade-package",
|
|
"--verbose",
|
|
"--cache-dir",
|
|
"--no-reuse-hashes",
|
|
}
|
|
|
|
|
|
def key_from_ireq(ireq):
|
|
"""Get a standardized key for an InstallRequirement."""
|
|
if ireq.req is None and ireq.link is not None:
|
|
return str(ireq.link)
|
|
else:
|
|
return key_from_req(ireq.req)
|
|
|
|
|
|
def key_from_req(req):
|
|
"""Get an all-lowercase version of the requirement's name."""
|
|
if hasattr(req, "key"):
|
|
# from pkg_resources, such as installed dists for pip-sync
|
|
key = req.key
|
|
else:
|
|
# from packaging, such as install requirements from requirements.txt
|
|
key = req.name
|
|
|
|
key = key.replace("_", "-").lower()
|
|
return key
|
|
|
|
|
|
def comment(text):
|
|
return style(text, fg="green")
|
|
|
|
|
|
def make_install_requirement(name, version, extras, constraint=False):
|
|
# If no extras are specified, the extras string is blank
|
|
extras_string = ""
|
|
if extras:
|
|
# Sort extras for stability
|
|
extras_string = "[{}]".format(",".join(sorted(extras)))
|
|
|
|
return install_req_from_line(
|
|
str("{}{}=={}".format(name, extras_string, version)), constraint=constraint
|
|
)
|
|
|
|
|
|
def is_url_requirement(ireq):
|
|
"""
|
|
Return True if requirement was specified as a path or URL.
|
|
ireq.original_link will have been set by InstallRequirement.__init__
|
|
"""
|
|
return bool(ireq.original_link)
|
|
|
|
|
|
def format_requirement(ireq, marker=None, hashes=None):
|
|
"""
|
|
Generic formatter for pretty printing InstallRequirements to the terminal
|
|
in a less verbose way than using its `__str__` method.
|
|
"""
|
|
if ireq.editable:
|
|
line = "-e {}".format(ireq.link.url)
|
|
elif is_url_requirement(ireq):
|
|
line = ireq.link.url
|
|
else:
|
|
line = str(ireq.req).lower()
|
|
|
|
if marker:
|
|
line = "{} ; {}".format(line, marker)
|
|
|
|
if hashes:
|
|
for hash_ in sorted(hashes):
|
|
line += " \\\n --hash={}".format(hash_)
|
|
|
|
return line
|
|
|
|
|
|
def format_specifier(ireq):
|
|
"""
|
|
Generic formatter for pretty printing the specifier part of
|
|
InstallRequirements to the terminal.
|
|
"""
|
|
# TODO: Ideally, this is carried over to the pip library itself
|
|
specs = ireq.specifier if ireq.req is not None else []
|
|
specs = sorted(specs, key=lambda x: x.version)
|
|
return ",".join(str(s) for s in specs) or "<any>"
|
|
|
|
|
|
def is_pinned_requirement(ireq):
|
|
"""
|
|
Returns whether an InstallRequirement is a "pinned" requirement.
|
|
|
|
An InstallRequirement is considered pinned if:
|
|
|
|
- Is not editable
|
|
- It has exactly one specifier
|
|
- That specifier is "=="
|
|
- The version does not contain a wildcard
|
|
|
|
Examples:
|
|
django==1.8 # pinned
|
|
django>1.8 # NOT pinned
|
|
django~=1.8 # NOT pinned
|
|
django==1.* # NOT pinned
|
|
"""
|
|
if ireq.editable:
|
|
return False
|
|
|
|
if ireq.req is None or len(ireq.specifier) != 1:
|
|
return False
|
|
|
|
spec = next(iter(ireq.specifier))
|
|
return spec.operator in {"==", "==="} and not spec.version.endswith(".*")
|
|
|
|
|
|
def as_tuple(ireq):
|
|
"""
|
|
Pulls out the (name: str, version:str, extras:(str)) tuple from
|
|
the pinned InstallRequirement.
|
|
"""
|
|
if not is_pinned_requirement(ireq):
|
|
raise TypeError("Expected a pinned InstallRequirement, got {}".format(ireq))
|
|
|
|
name = key_from_ireq(ireq)
|
|
version = next(iter(ireq.specifier)).version
|
|
extras = tuple(sorted(ireq.extras))
|
|
return name, version, extras
|
|
|
|
|
|
def flat_map(fn, collection):
|
|
"""Map a function over a collection and flatten the result by one-level"""
|
|
return chain.from_iterable(map(fn, collection))
|
|
|
|
|
|
def lookup_table(values, key=None, keyval=None, unique=False, use_lists=False):
|
|
"""
|
|
Builds a dict-based lookup table (index) elegantly.
|
|
|
|
Supports building normal and unique lookup tables. For example:
|
|
|
|
>>> assert lookup_table(
|
|
... ['foo', 'bar', 'baz', 'qux', 'quux'], lambda s: s[0]) == {
|
|
... 'b': {'bar', 'baz'},
|
|
... 'f': {'foo'},
|
|
... 'q': {'quux', 'qux'}
|
|
... }
|
|
|
|
For key functions that uniquely identify values, set unique=True:
|
|
|
|
>>> assert lookup_table(
|
|
... ['foo', 'bar', 'baz', 'qux', 'quux'], lambda s: s[0],
|
|
... unique=True) == {
|
|
... 'b': 'baz',
|
|
... 'f': 'foo',
|
|
... 'q': 'quux'
|
|
... }
|
|
|
|
For the values represented as lists, set use_lists=True:
|
|
|
|
>>> assert lookup_table(
|
|
... ['foo', 'bar', 'baz', 'qux', 'quux'], lambda s: s[0],
|
|
... use_lists=True) == {
|
|
... 'b': ['bar', 'baz'],
|
|
... 'f': ['foo'],
|
|
... 'q': ['qux', 'quux']
|
|
... }
|
|
|
|
The values of the resulting lookup table will be lists, not sets.
|
|
|
|
For extra power, you can even change the values while building up the LUT.
|
|
To do so, use the `keyval` function instead of the `key` arg:
|
|
|
|
>>> assert lookup_table(
|
|
... ['foo', 'bar', 'baz', 'qux', 'quux'],
|
|
... keyval=lambda s: (s[0], s[1:])) == {
|
|
... 'b': {'ar', 'az'},
|
|
... 'f': {'oo'},
|
|
... 'q': {'uux', 'ux'}
|
|
... }
|
|
|
|
"""
|
|
if keyval is None:
|
|
if key is None:
|
|
|
|
def keyval(v):
|
|
return v
|
|
|
|
else:
|
|
|
|
def keyval(v):
|
|
return (key(v), v)
|
|
|
|
if unique:
|
|
return dict(keyval(v) for v in values)
|
|
|
|
lut = {}
|
|
for value in values:
|
|
k, v = keyval(value)
|
|
try:
|
|
s = lut[k]
|
|
except KeyError:
|
|
if use_lists:
|
|
s = lut[k] = list()
|
|
else:
|
|
s = lut[k] = set()
|
|
if use_lists:
|
|
s.append(v)
|
|
else:
|
|
s.add(v)
|
|
return dict(lut)
|
|
|
|
|
|
def dedup(iterable):
|
|
"""Deduplicate an iterable object like iter(set(iterable)) but
|
|
order-preserved.
|
|
"""
|
|
return iter(OrderedDict.fromkeys(iterable))
|
|
|
|
|
|
def name_from_req(req):
|
|
"""Get the name of the requirement"""
|
|
if hasattr(req, "project_name"):
|
|
# from pkg_resources, such as installed dists for pip-sync
|
|
return req.project_name
|
|
else:
|
|
# from packaging, such as install requirements from requirements.txt
|
|
return req.name
|
|
|
|
|
|
def fs_str(string):
|
|
"""
|
|
Convert given string to a correctly encoded filesystem string.
|
|
|
|
On Python 2, if the input string is unicode, converts it to bytes
|
|
encoded with the filesystem encoding.
|
|
|
|
On Python 3 returns the string as is, since Python 3 uses unicode
|
|
paths and the input string shouldn't be bytes.
|
|
|
|
:type string: str|unicode
|
|
:rtype: str
|
|
"""
|
|
if isinstance(string, str):
|
|
return string
|
|
if isinstance(string, bytes):
|
|
raise TypeError("fs_str() argument must not be bytes")
|
|
return string.encode(_fs_encoding)
|
|
|
|
|
|
_fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
|
|
|
|
|
|
def get_hashes_from_ireq(ireq):
|
|
"""
|
|
Given an InstallRequirement, return a list of string hashes in
|
|
the format "{algorithm}:{hash}". Return an empty list if there are no hashes
|
|
in the requirement options.
|
|
"""
|
|
result = []
|
|
for algorithm, hexdigests in ireq.hash_options.items():
|
|
for hash_ in hexdigests:
|
|
result.append("{}:{}".format(algorithm, hash_))
|
|
return result
|
|
|
|
|
|
def force_text(s):
|
|
"""
|
|
Return a string representing `s`.
|
|
"""
|
|
if s is None:
|
|
return ""
|
|
if not isinstance(s, six.string_types):
|
|
return six.text_type(s)
|
|
return s
|
|
|
|
|
|
def get_compile_command(click_ctx):
|
|
"""
|
|
Returns a normalized compile command depending on cli context.
|
|
|
|
The command will be normalized by:
|
|
- expanding options short to long
|
|
- removing values that are already default
|
|
- sorting the arguments
|
|
- removing one-off arguments like '--upgrade'
|
|
- removing arguments that don't change build behaviour like '--verbose'
|
|
"""
|
|
from piptools.scripts.compile import cli
|
|
|
|
# Map of the compile cli options (option name -> click.Option)
|
|
compile_options = {option.name: option for option in cli.params}
|
|
|
|
left_args = []
|
|
right_args = []
|
|
|
|
for option_name, value in click_ctx.params.items():
|
|
option = compile_options[option_name]
|
|
|
|
# Collect variadic args separately, they will be added
|
|
# at the end of the command later
|
|
if option.nargs < 0:
|
|
# These will necessarily be src_files
|
|
# Re-add click-stripped '--' if any start with '-'
|
|
if any(val.startswith("-") and val != "-" for val in value):
|
|
right_args.append("--")
|
|
right_args.extend([shlex_quote(force_text(val)) for val in value])
|
|
continue
|
|
|
|
# Get the latest option name (usually it'll be a long name)
|
|
option_long_name = option.opts[-1]
|
|
|
|
# Exclude one-off options (--upgrade/--upgrade-package/--rebuild/...)
|
|
# or options that don't change compile behaviour (--verbose/--dry-run/...)
|
|
if option_long_name in COMPILE_EXCLUDE_OPTIONS:
|
|
continue
|
|
|
|
# Skip options without a value
|
|
if option.default is None and not value:
|
|
continue
|
|
|
|
# Skip options with a default value
|
|
if option.default == value:
|
|
continue
|
|
|
|
# Use a file name for file-like objects
|
|
if isinstance(value, LazyFile):
|
|
value = value.name
|
|
|
|
# Convert value to the list
|
|
if not isinstance(value, (tuple, list)):
|
|
value = [value]
|
|
|
|
for val in value:
|
|
# Flags don't have a value, thus add to args true or false option long name
|
|
if option.is_flag:
|
|
# If there are false-options, choose an option name depending on a value
|
|
if option.secondary_opts:
|
|
# Get the latest false-option
|
|
secondary_option_long_name = option.secondary_opts[-1]
|
|
arg = option_long_name if val else secondary_option_long_name
|
|
# There are no false-options, use true-option
|
|
else:
|
|
arg = option_long_name
|
|
left_args.append(shlex_quote(arg))
|
|
# Append to args the option with a value
|
|
else:
|
|
if isinstance(val, six.string_types) and is_url(val):
|
|
val = redact_auth_from_url(val)
|
|
if option.name == "pip_args":
|
|
# shlex_quote would produce functional but noisily quoted results,
|
|
# e.g. --pip-args='--cache-dir='"'"'/tmp/with spaces'"'"''
|
|
# Instead, we try to get more legible quoting via repr:
|
|
left_args.append(
|
|
"{option}={value}".format(
|
|
option=option_long_name, value=repr(fs_str(force_text(val)))
|
|
)
|
|
)
|
|
else:
|
|
left_args.append(
|
|
"{option}={value}".format(
|
|
option=option_long_name, value=shlex_quote(force_text(val))
|
|
)
|
|
)
|
|
|
|
return " ".join(["pip-compile"] + sorted(left_args) + sorted(right_args))
|