diff --git a/contrib/deterministic-build/find_restricted_dependencies.py b/contrib/deterministic-build/find_restricted_dependencies.py new file mode 100755 index 000000000..846a0c75d --- /dev/null +++ b/contrib/deterministic-build/find_restricted_dependencies.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +import sys + +import requests + + +def check_restriction(p, r): + # See: https://www.python.org/dev/peps/pep-0496/ + # Hopefully we don't need to parse the whole microlanguage + if "extra" in r and "[" not in p: + return False + for marker in ["os_name", "platform_release", "sys_platform", "platform_system"]: + if marker in r: + return True + + +for p in sys.stdin.read().split(): + p = p.strip() + if not p: + continue + assert "==" in p, "This script expects a list of packages with pinned version, e.g. package==1.2.3, not {}".format(p) + p, v = p.rsplit("==", 1) + try: + data = requests.get("https://pypi.org/pypi/{}/{}/json".format(p, v)).json()["info"] + except ValueError: + raise BaseException("Package could not be found: {}=={}".format(p, v)) + try: + for r in data["requires_dist"]: + if ";" not in r: + continue + d, restricted = r.split(";", 1) + if check_restriction(d, restricted): + print(d, sep=" ") + print("Installing {} from {} although it is only needed for {}".format(d, p, restricted), file=sys.stderr) + except TypeError: + # Has no dependencies at all + continue + diff --git a/contrib/deterministic-build/requirements.txt b/contrib/deterministic-build/requirements.txt index 3c400d955..58b0e83dc 100644 --- a/contrib/deterministic-build/requirements.txt +++ b/contrib/deterministic-build/requirements.txt @@ -55,3 +55,6 @@ six==1.11.0 \ urllib3==1.22 \ --hash=sha256:06330f386d6e4b195fbfc736b297f58c5a892e4440e54d294d7004e3a9bbea1b \ --hash=sha256:cc44da8e1145637334317feebd728bd869a35285b93cbb4cca2577da7e62db4f +colorama==0.3.9 \ + --hash=sha256:463f8483208e921368c9f306094eb6f725c6ca42b0f97e313cb5d5512459feda \ + --hash=sha256:48eb22f4f8461b1df5734a074b57042430fb06e1d61bd1e11b078c0fe6d7a1f1 diff --git a/contrib/freeze_packages.sh b/contrib/freeze_packages.sh index 840825199..623eddaad 100755 --- a/contrib/freeze_packages.sh +++ b/contrib/freeze_packages.sh @@ -21,9 +21,14 @@ for i in '' '-hw' '-binaries'; do echo "OK." requirements=$(pip freeze) + restricted=$(echo $requirements | $other_python ./deterministic-build/find_restricted_dependencies.py) + requirements="$requirements $restricted" + echo "Generating package hashes..." + rm $contrib/deterministic-build/requirements${i}.txt + touch $contrib/deterministic-build/requirements${i}.txt - echo "$requirements" | while IFS= read -r requirement ; do + for requirement in $requirements; do echo -e "\r Hashing $requirement..." $other_python -m hashin -r $contrib/deterministic-build/requirements${i}.txt ${requirement} done