if dependency.find("@") > 0 and (

dependency.find("@") < dependency.find("://") or

"://" not in dependency

):

# WORKAROUND FOR UPSTREAM BUG:

# https://github.com/pypa/pip/issues/6097

# (Please REMOVE workaround once that is fixed & released upstream!)

#

# Basically, setup_requires() can contain a format pip won't install

# from a requirements.txt (PEP 508 URLs).

# To avoid this, translate to an #egg= reference:

if dependency.endswith("#"):

dependency = dependency[:-1]

url = (dependency.partition("@")[2].strip().partition("#egg")[0] +

"#egg=" +

dependency.partition("@")[0].strip()

)

return url

package,

output_folder,

debug=False

):

""" Extracts metdata files from the given package to the given folder,

if package is None:

raise ValueError("package cannot be None")

if not os.path.exists(output_folder) or os.path.isfile(output_folder):

raise ValueError("output folder needs to be existing folder")

if debug:

print("extract_metainfo_files_from_package: extracting for " +

"package: " + str(package))

# A temp folder for making a package copy in case it's a local folder,

# because extracting metadata might modify files

# (creating sdists/wheels...)

temp_folder = tempfile.mkdtemp(prefix="pythonpackage-package-copy-")

try:

# Package is indeed a folder! Get a temp copy to work on:

if is_filesystem_path(package):

shutil.copytree(

parse_as_folder_reference(package),

os.path.join(temp_folder, "package"),

ignore=shutil.ignore_patterns(".tox")

)

package = os.path.join(temp_folder, "package")

# Because PEP517 can be noisy and contextlib.redirect_* fails to

# contain it, we will run the actual analysis in a separate process:

try:

subprocess.check_output([

sys.executable,

"-c",

"import importlib\n"

"import json\n"

"import os\n"

"import sys\n"

"sys.path = [os.path.dirname(sys.argv[3])] + sys.path\n"

"m = importlib.import_module(\n"

" os.path.basename(sys.argv[3]).partition('.')[0]\n"

")\n"

"m._extract_metainfo_files_from_package_unsafe("

" sys.argv[1],"

" sys.argv[2],"

")",

package, output_folder, os.path.abspath(__file__)],

stderr=subprocess.STDOUT, # make sure stderr is muted.

cwd=os.path.join(os.path.dirname(__file__), "..")

)

except subprocess.CalledProcessError as e:

output = e.output.decode("utf-8", "replace")

if debug:

print("Got error obtaining meta info.")

print("Detail output:")

print(output)

print("End of Detail output.")

raise ValueError(

"failed to obtain meta info - "

"is '{}' a valid package? "

"Detailed output:\n{}".format(package, output)

)

finally:

""" Returns the path the system-wide python binary.

# This function is required by get_package_as_folder() to work

# inside a virtualenv, since venv creation will fail with

# the virtualenv's local python binary.

# (venv/virtualenv incompatibility)

# Abort if not in virtualenv or venv:

if not hasattr(sys, "real_prefix") and (

not hasattr(sys, "base_prefix") or

os.path.normpath(sys.base_prefix) ==

os.path.normpath(sys.prefix)):

return sys.executable

# Extract prefix we need to look in:

if hasattr(sys, "real_prefix"):

search_prefix = sys.real_prefix # virtualenv

else:

search_prefix = sys.base_prefix # venv

def python_binary_from_folder(path):

def binary_is_usable(python_bin):

""" Helper function to see if a given binary name refers

# Abort if path isn't present at all or a directory:

if not os.path.exists(

os.path.join(path, python_bin)

) or os.path.isdir(os.path.join(path, python_bin)):

return

# We should check file not found anyway trying to run it,

# since it might be a dead symlink:

try:

filenotfounderror = FileNotFoundError

except NameError: # Python 2

filenotfounderror = OSError

try:

# Run it and see if version output works with no error:

subprocess.check_output([

os.path.join(path, python_bin), "--version"

], stderr=subprocess.STDOUT)

return True

except (subprocess.CalledProcessError, filenotfounderror):

return False

python_name = "python" + sys.version

while (not binary_is_usable(python_name) and

python_name.find(".") > 0):

# Try less specific binary name:

python_name = python_name.rpartition(".")[0]

if binary_is_usable(python_name):

return os.path.join(path, python_name)

return None

# Return from sys.real_prefix if present:

result = python_binary_from_folder(search_prefix)

if result is not None:

return result

# Check out all paths in $PATH:

bad_candidates = []

good_candidates = []

ever_had_nonvenv_path = False

ever_had_path_starting_with_prefix = False

for p in os.environ.get("PATH", "").split(":"):

# Skip if not possibly the real system python:

if not os.path.normpath(p).startswith(

os.path.normpath(search_prefix)

):

continue

ever_had_path_starting_with_prefix = True

# First folders might be virtualenv/venv we want to avoid:

if not ever_had_nonvenv_path:

sep = os.path.sep

if (

("system32" not in p.lower() and

"usr" not in p and

not p.startswith("/opt/python")) or

{"home", ".tox"}.intersection(set(p.split(sep))) or

"users" in p.lower()

):

# Doesn't look like bog-standard system path.

if (p.endswith(os.path.sep + "bin") or

p.endswith(os.path.sep + "bin" + os.path.sep)):

# Also ends in "bin" -> likely virtualenv/venv.

# Add as unfavorable / end of candidates:

bad_candidates.append(p)

continue

ever_had_nonvenv_path = True

good_candidates.append(p)

# If we have a bad env with PATH not containing any reference to our

# real python (travis, why would you do that to me?) then just guess

# based from the search prefix location itself:

if not ever_had_path_starting_with_prefix:

# ... and yes we're scanning all the folders for that, it's dumb

# but i'm not aware of a better way: (@JonasT)

for root, dirs, files in os.walk(search_prefix, topdown=True):

for name in dirs:

bad_candidates.append(os.path.join(root, name))

# Sort candidates by length (to prefer shorter ones):

def candidate_cmp(a, b):

return len(a) - len(b)

good_candidates = sorted(

good_candidates, key=functools.cmp_to_key(candidate_cmp)

)

bad_candidates = sorted(

bad_candidates, key=functools.cmp_to_key(candidate_cmp)

)

# See if we can now actually find the system python:

for p in good_candidates + bad_candidates:

result = python_binary_from_folder(p)

if result is not None:

return result

raise RuntimeError(

"failed to locate system python in: {}"

" - checked candidates were: {}, {}"

.format(sys.real_prefix, good_candidates, bad_candidates)

""" This function downloads the given package / dependency and extracts

venv_parent = tempfile.mkdtemp(

prefix="pythonpackage-venv-"

)

try:

# Create a venv to install into:

try:

if int(sys.version.partition(".")[0]) < 3:

# Python 2.x has no venv.

subprocess.check_output([

sys.executable, # no venv conflict possible,

# -> no need to use system python

"-m", "virtualenv",

"--python=" + _get_system_python_executable(),

os.path.join(venv_parent, 'venv')

], cwd=venv_parent)

else:

# On modern Python 3, use venv.

subprocess.check_output([

_get_system_python_executable(), "-m", "venv",

os.path.join(venv_parent, 'venv')

], cwd=venv_parent)

except subprocess.CalledProcessError as e:

output = e.output.decode('utf-8', 'replace')

raise ValueError(

'venv creation unexpectedly ' +

'failed. error output: ' + str(output)

)

venv_path = os.path.join(venv_parent, "venv")

# Update pip and wheel in venv for latest feature support:

try:

filenotfounderror = FileNotFoundError

except NameError: # Python 2.

filenotfounderror = OSError

try:

subprocess.check_output([

os.path.join(venv_path, "bin", "pip"),

"install", "-U", "pip", "wheel",

])

except filenotfounderror:

raise RuntimeError(

"venv appears to be missing pip. "

"did we fail to use a proper system python??\n"

"system python path detected: {}\n"

"os.environ['PATH']: {}".format(

_get_system_python_executable(),

os.environ.get("PATH", "")

)

)

# Create download subfolder:

os.mkdir(os.path.join(venv_path, "download"))

# Write a requirements.txt with our package and download:

with open(os.path.join(venv_path, "requirements.txt"),

"w", encoding="utf-8"

) as f:

def to_unicode(s): # Needed for Python 2.

try:

return s.decode("utf-8")

except AttributeError:

return s

f.write(to_unicode(transform_dep_for_pip(dependency)))

try:

subprocess.check_output(

[

os.path.join(venv_path, "bin", "pip"),

"download", "--no-deps", "-r", "../requirements.txt",

"-d", os.path.join(venv_path, "download")

],

stderr=subprocess.STDOUT,

cwd=os.path.join(venv_path, "download")

)

except subprocess.CalledProcessError as e:

raise RuntimeError("package download failed: " + str(e.output))

if len(os.listdir(os.path.join(venv_path, "download"))) == 0:

# No download. This can happen if the dependency has a condition

# which prohibits install in our environment.

# (the "package ; ... conditional ... " type of condition)

return (None, None)

# Get the result and make sure it's an extracted directory:

result_folder_or_file = os.path.join(

venv_path, "download",

os.listdir(os.path.join(venv_path, "download"))[0]

)

dl_type = "source"

if not os.path.isdir(result_folder_or_file):

# Must be an archive.

if result_folder_or_file.endswith((".zip", ".whl")):

if result_folder_or_file.endswith(".whl"):

dl_type = "wheel"

with zipfile.ZipFile(result_folder_or_file) as f:

f.extractall(os.path.join(venv_path,

"download", "extracted"

))

result_folder_or_file = os.path.join(

venv_path, "download", "extracted"

)

elif result_folder_or_file.find(".tar.") > 0:

# Probably a tarball.

with tarfile.open(result_folder_or_file) as f:

f.extractall(os.path.join(venv_path,

"download", "extracted"

))

result_folder_or_file = os.path.join(

venv_path, "download", "extracted"

)

else:

raise RuntimeError(

"unknown archive or download " +

"type: " + str(result_folder_or_file)

)

# If the result is hidden away in an additional subfolder,

# descend into it:

while os.path.isdir(result_folder_or_file) and \

len(os.listdir(result_folder_or_file)) == 1 and \

os.path.isdir(os.path.join(

result_folder_or_file,

os.listdir(result_folder_or_file)[0]

)):

result_folder_or_file = os.path.join(

result_folder_or_file,

os.listdir(result_folder_or_file)[0]

)

# Copy result to new dedicated folder so we can throw away

# our entire virtualenv nonsense after returning:

result_path = tempfile.mkdtemp()

shutil.rmtree(result_path)

shutil.copytree(result_folder_or_file, result_path)

return (dl_type, result_path)

finally:

package,

output_path

):

# This is the unwrapped function that will

# 1. make lots of stdout/stderr noise

# 2. possibly modify files (if the package source is a local folder)

# Use extract_metainfo_files_from_package_folder instead which avoids

# these issues.

clean_up_path = False

path_type = "source"

path = parse_as_folder_reference(package)

if path is None:

# This is not a path. Download it:

(path_type, path) = get_package_as_folder(package)

if path_type is None:

# Download failed.

raise ValueError(

"cannot get info for this package, " +

"pip says it has no downloads (conditional dependency?)"

)

clean_up_path = True

try:

build_requires = []

metadata_path = None

if path_type != "wheel":

# We need to process this first to get the metadata.

# Ensure pyproject.toml is available (pep517 expects it)

if not os.path.exists(os.path.join(path, "pyproject.toml")):

with open(os.path.join(path, "pyproject.toml"), "w") as f:

f.write(textwrap.dedent(u"""\

"""))

# Copy the pyproject.toml:

shutil.copyfile(

os.path.join(path, 'pyproject.toml'),

os.path.join(output_path, 'pyproject.toml')

)

# Get build backend and requirements from pyproject.toml:

with open(os.path.join(path, 'pyproject.toml')) as f:

build_sys = toml.load(f)['build-system']

backend = build_sys["build-backend"]

build_requires.extend(build_sys["requires"])

# Get a virtualenv with build requirements and get all metadata:

env = BuildEnvironment()

metadata = None

with env:

hooks = Pep517HookCaller(path, backend)

env.pip_install(

[transform_dep_for_pip(req) for req in build_requires]

)

reqs = hooks.get_requires_for_build_wheel({})

env.pip_install([transform_dep_for_pip(req) for req in reqs])

try:

metadata = hooks.prepare_metadata_for_build_wheel(path)

except Exception: # sadly, pep517 has no good error here

pass

if metadata is not None:

metadata_path = os.path.join(

path, metadata, "METADATA"

)

else:

# This is a wheel, so metadata should be in *.dist-info folder:

metadata_path = os.path.join(

path,

[f for f in os.listdir(path) if f.endswith(".dist-info")][0],

"METADATA"

)

# Store type of metadata source. Can be "wheel", "source" for source

# distribution, and others get_package_as_folder() may support

# in the future.

with open(os.path.join(output_path, "metadata_source"), "w") as f:

try:

f.write(path_type)

except TypeError: # in python 2 path_type may be str/bytes:

f.write(path_type.decode("utf-8", "replace"))

# Copy the metadata file:

shutil.copyfile(metadata_path, os.path.join(output_path, "METADATA"))

finally:

if clean_up_path:

""" Convenience function around parse_as_folder_reference() to

""" See if a dependency reference refers to a folder path.

# Special case: pep508 urls

if dep.find("@") > 0 and (

(dep.find("@") < dep.find("/") or "/" not in dep) and

(dep.find("@") < dep.find(":") or ":" not in dep)

):

# This should be a 'pkgname @ https://...' style path, or

# 'pkname @ /local/file/path'.

return parse_as_folder_reference(dep.partition("@")[2].lstrip())

# Check if this is either not an url, or a file URL:

if dep.startswith(("/", "file://")) or (

dep.find("/") > 0 and

dep.find("://") < 0) or (dep in ["", "."]):

if dep.startswith("file://"):

dep = urlunquote(urlparse(dep).path)

return dep

extract_type=None,

debug=False,

include_build_requirements=False

):

""" Internal function to extract metainfo from a package.

if debug:

print("_extract_info_from_package called with "

"extract_type={} include_build_requirements={}".format(

extract_type, include_build_requirements,

))

output_folder = tempfile.mkdtemp(prefix="pythonpackage-metafolder-")

try:

extract_metainfo_files_from_package(

dependency, output_folder, debug=debug

)

# Extract the type of data source we used to get the metadata:

with open(os.path.join(output_folder,

"metadata_source"), "r") as f:

metadata_source_type = f.read().strip()

# Extract main METADATA file:

with open(os.path.join(output_folder, "METADATA"),

"r", encoding="utf-8"

) as f:

# Get metadata and cut away description (is after 2 linebreaks)

metadata_entries = f.read().partition("\n\n")[0].splitlines()

if extract_type == "name":

name = None

for meta_entry in metadata_entries:

if meta_entry.lower().startswith("name:"):

return meta_entry.partition(":")[2].strip()

if name is None:

raise ValueError("failed to obtain package name")

return name

elif extract_type == "dependencies":

# First, make sure we don't attempt to return build requirements

# for wheels since they usually come without pyproject.toml

# and we haven't implemented another way to get them:

if include_build_requirements and \

metadata_source_type == "wheel":

if debug:

print("_extract_info_from_package: was called "

"with include_build_requirements=True on "

"package obtained as wheel, raising error...")

raise NotImplementedError(

"fetching build requirements for "

"wheels is not implemented"

)

# Get build requirements from pyproject.toml if requested:

requirements = []

if os.path.exists(os.path.join(output_folder,

'pyproject.toml')

) and include_build_requirements:

# Read build system from pyproject.toml file: (PEP518)

with open(os.path.join(output_folder, 'pyproject.toml')) as f:

build_sys = toml.load(f)['build-system']

if "requires" in build_sys:

requirements += build_sys["requires"]

elif include_build_requirements:

# For legacy packages with no pyproject.toml, we have to

# add setuptools as default build system.

requirements.append("setuptools")

# Add requirements from metadata:

requirements += [

entry.rpartition("Requires-Dist:")[2].strip()

for entry in metadata_entries

if entry.startswith("Requires-Dist")

]

return list(set(requirements)) # remove duplicates

finally:

use_cache=True):

def timestamp():

try:

return time.monotonic()

except AttributeError:

return time.time() # Python 2.

try:

value = package_name_cache[dependency]

if value[0] + 600.0 > timestamp() and use_cache:

return value[1]

except KeyError:

pass

result = _extract_info_from_package(dependency, extract_type="name")

package_name_cache[dependency] = (timestamp(), result)

recursive=False,

verbose=False,

include_build_requirements=False):

""" Obtain the dependencies from a package. Please note this

packages_processed = set()

package_queue = [package]

reqs = set()

reqs_as_names = set()

while len(package_queue) > 0:

current_queue = package_queue

package_queue = []

for package_dep in current_queue:

new_reqs = set()

if verbose:

print("get_package_dependencies: resolving dependency "

"to package name: ".format(package_dep))

package = get_package_name(package_dep)

if package.lower() in packages_processed:

continue

if verbose:

print("get_package_dependencies: "

"processing package: {}".format(package))

print("get_package_dependencies: "

"Packages seen so far: {}".format(

packages_processed

))

packages_processed.add(package.lower())

# Use our regular folder processing to examine:

new_reqs = new_reqs.union(_extract_info_from_package(

package_dep, extract_type="dependencies",

debug=verbose,

include_build_requirements=include_build_requirements,

))

# Process new requirements:

if verbose:

print('get_package_dependencies: collected '

"deps of '{}': {}".format(

package_dep, str(new_reqs),

))

for new_req in new_reqs:

try:

req_name = get_package_name(new_req)

except ValueError as e:

if new_req.find(";") >= 0:

# Conditional dep where condition isn't met?

# --> ignore it

continue

if verbose:

print("get_package_dependencies: " +

"unexpected failure to get name " +

"of '" + str(new_req) + "': " +

str(e))

raise RuntimeError(

"failed to get " +

"name of dependency: " + str(e)

)

if req_name.lower() in reqs_as_names:

continue

if req_name.lower() not in packages_processed:

package_queue.append(new_req)

reqs.add(new_req)

reqs_as_names.add(req_name.lower())

# Bail out here if we're not scanning recursively:

if not recursive:

package_queue[:] = [] # wipe queue

break

if verbose:

print("get_package_dependencies: returning result: {}".format(reqs))

package,

keep_version_pins=False,

recursive=False,

verbose=False,

include_build_requirements=False

):

""" Gets the dependencies from the package in the given folder,

# First, obtain the dependencies:

dependencies = get_package_dependencies(

package, recursive=recursive, verbose=verbose,

include_build_requirements=include_build_requirements,

)

if verbose:

print("get_dep_names_of_package_folder: " +

"processing dependency list to names: " +

str(dependencies))

# Transform dependencies to their stripped down names:

# (they can still have version pins/restrictions, conditionals, ...)

dependency_names = set()

for dep in dependencies:

# If we are supposed to keep exact version pins, extract first:

pin_to_append = ""

if keep_version_pins and "(==" in dep and dep.endswith(")"):

# This is a dependency of the format: 'pkg (==1.0)'

pin_to_append = "==" + dep.rpartition("==")[2][:-1]

elif keep_version_pins and "==" in dep and not dep.endswith(")"):

# This is a dependency of the format: 'pkg==1.0'

pin_to_append = "==" + dep.rpartition("==")[2]

# Now get true (and e.g. case-corrected) dependency name:

dep_name = get_package_name(dep) + pin_to_append

dependency_names.add(dep_name)