From 784435030a463018fe589b8a78b7d8c75f219832 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 13 Sep 2018 23:59:32 -0400 Subject: [PATCH 01/32] Restructure and reorganization - Separation of concerns - actions are now separated from CLI interface - Models are separated from 'internals' - May want to think about what to do with internals to organize them a bit? - Introduced `models` for most of the Object Oriented stuff - Introduced `options` as a central aggregation of options and argument groups - this has 2 classes: `Option` and `ArgumentGroup`, each with the method `add_to_parser` for ease of use - Command subclasses now simply take a list of options or argument groups and have a defined method to override called `run(self, options)` -- this way we can pass parameters to the corresponding `actions` rather than just blanketing the options in Signed-off-by: Dan Ryan --- Pipfile.lock | 124 ++++++++-------- src/passa/actions/__init__.py | 0 src/passa/actions/add.py | 57 ++++++++ src/passa/actions/clean.py | 15 ++ src/passa/actions/freeze.py | 93 ++++++++++++ src/passa/actions/install.py | 32 ++++ src/passa/actions/lock.py | 15 ++ src/passa/actions/remove.py | 37 +++++ src/passa/actions/sync.py | 18 +++ src/passa/actions/upgrade.py | 49 +++++++ src/passa/cli/__init__.py | 2 +- src/passa/cli/_base.py | 41 ++++-- src/passa/cli/add.py | 93 ++---------- src/passa/cli/clean.py | 29 +--- src/passa/cli/freeze.py | 130 +---------------- src/passa/cli/install.py | 57 +------- src/passa/cli/lock.py | 21 +-- src/passa/cli/options.py | 137 ++++++++++++++++++ src/passa/cli/remove.py | 67 +-------- src/passa/cli/sync.py | 39 +---- src/passa/cli/upgrade.py | 84 +---------- src/passa/internals/_pip.py | 2 +- src/passa/internals/dependencies.py | 2 +- src/passa/models/__init__.py | 0 src/passa/{internals => models}/caches.py | 4 +- src/passa/{internals => models}/lockers.py | 11 +- src/passa/{internals => models}/metadata.py | 4 +- src/passa/{internals => models}/providers.py | 9 +- .../{internals => models}/synchronizers.py | 2 +- 29 files changed, 617 insertions(+), 557 deletions(-) create mode 100644 src/passa/actions/__init__.py create mode 100644 src/passa/actions/add.py create mode 100644 src/passa/actions/clean.py create mode 100644 src/passa/actions/freeze.py create mode 100644 src/passa/actions/install.py create mode 100644 src/passa/actions/lock.py create mode 100644 src/passa/actions/remove.py create mode 100644 src/passa/actions/sync.py create mode 100644 src/passa/actions/upgrade.py create mode 100644 src/passa/cli/options.py create mode 100644 src/passa/models/__init__.py rename src/passa/{internals => models}/caches.py (98%) rename src/passa/{internals => models}/lockers.py (97%) rename src/passa/{internals => models}/metadata.py (97%) rename src/passa/{internals => models}/providers.py (97%) rename src/passa/{internals => models}/synchronizers.py (98%) diff --git a/Pipfile.lock b/Pipfile.lock index ff1f65c..c83e057 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -24,11 +24,11 @@ }, "attrs": { "hashes": [ - "sha256:4b90b09eeeb9b88c35bc642cbac057e45a5fd85367b985bd2809c62b7b939265", - "sha256:e0d0eb91441a3b53dab4d9b743eafc1ac44476296a2053b6ca3af0b139faf87b" + "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", + "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '3.6'", - "version": "==18.1.0" + "version": "==18.2.0" }, "backports-shutil-get-terminal-size": { "hashes": [ @@ -115,22 +115,6 @@ "markers": "python_version >= '2.6' and python_version >= '3.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.0.0" }, - "more-itertools": { - "hashes": [ - "sha256:c187a73da93e7a8acc0001572aebc7e3c69daf7bf6881a2cea10650bd4420092", - "sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e", - "sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==4.3.0" - }, - "ordereddict": { - "hashes": [ - "sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f" - ], - "markers": "python_version < '3.0' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.1" - }, "packaging": { "hashes": [ "sha256:e9215d2d2535d3ae866c3d6efc77d5b24a0192cce0ff20e42896cc0664f889c0", @@ -149,7 +133,7 @@ "sha256:8eb170f8d0d61825e09a95b38be068299ddeda82f35e96c3301a8a5e7604cb83", "sha256:d1aa2a11ba7b8f7b21ab852b1fb5afb277e1bb99d5dfc663380b5015c0d80c5a" ], - "markers": "python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.6' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.3.2" }, "pip-shims": { @@ -189,19 +173,19 @@ }, "requirementslib": { "hashes": [ - "sha256:7288ec80fa62b78c6193eca85882605c4cdbb60361fc62478bd4a913c0b54400", - "sha256:8bb1b156d8b13337eab359273605b036d60735b350019addabb054e8a89a96f6" + "sha256:90151d8963f814e17190e067b60e92fb35fd1bc46c99f8dba3d7b0d93a3dd958", + "sha256:c3aeaa4e0b80843ba65a68878293e07ea52a8d0706dbba86b02dad6cd20ef2dd" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.1.5" + "version": "==1.1.6" }, "resolvelib": { "hashes": [ - "sha256:d52f2c0762deeb2a4cc34a84371a7a5ac85e111bdc69ce9ae729d8d636606ad6", - "sha256:eb759d43bbf50de9bf36afb9f6c269fabf9ff49084dbfad4ba67252d134bf4b5" + "sha256:6c4c6690b0bdd78bcc002e1a5d1b6abbde58c694a6ea1838f165b20d2c943db7", + "sha256:8734e53271ef98f38a2c99324d5e7905bc00c97dc3fc5bb7d83c82a979e71c04" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.2.1" + "version": "==0.2.2" }, "scandir": { "hashes": [ @@ -217,7 +201,7 @@ "sha256:c9009c527929f6e25604aec39b0a43c3f831d2947d89d6caaab22f057b7055c8", "sha256:f5c71e29b4e2af7ccdc03a020c626ede51da471173b4a6ad1e904f2b2e04b4bd" ], - "markers": "python_version < '3.5' and python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version < '3.6' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.9.0" }, "six": { @@ -225,7 +209,7 @@ "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" ], - "markers": "python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.6' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.11.0" }, "toml": { @@ -237,11 +221,11 @@ }, "tomlkit": { "hashes": [ - "sha256:618b057b96b488bb858102bdb38dd9d451c0617667031e320efea2392a39bec4", - "sha256:6856b5395f7c509baad1911aa3940b3c2fb33f53aff968ed1596c393ccea98e5" + "sha256:8ab16e93162fc44d3ad83d2aa29a7140b8f7d996ae1790a73b9a7aed6fb504ac", + "sha256:ca181cee7aee805d455628f7c94eb8ae814763769a93e69157f250fe4ebe1926" ], "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.4.3" + "version": "==0.4.4" }, "typing": { "hashes": [ @@ -278,6 +262,14 @@ ], "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.31.1" + }, + "yaspin": { + "hashes": [ + "sha256:36fdccc5e0637b5baa8892fe2c3d927782df7d504e9020f40eb2c1502518aa5a", + "sha256:8e52bf8079a48e2a53f3dfeec9e04addb900c101d1591c85df69cf677d3237e7" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.14.0" } }, "develop": { @@ -331,11 +323,11 @@ }, "attrs": { "hashes": [ - "sha256:4b90b09eeeb9b88c35bc642cbac057e45a5fd85367b985bd2809c62b7b939265", - "sha256:e0d0eb91441a3b53dab4d9b743eafc1ac44476296a2053b6ca3af0b139faf87b" + "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", + "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '3.6'", - "version": "==18.1.0" + "version": "==18.2.0" }, "babel": { "hashes": [ @@ -502,11 +494,11 @@ }, "imagesize": { "hashes": [ - "sha256:3620cc0cadba3f7475f9940d22431fc4d407269f1be59ec9b8edcca26440cf18", - "sha256:5b326e4678b6925158ccc66a9fa3122b6106d7c876ee32d7de6ce59385b96315" + "sha256:3f349de3eb99145973fefb7dbe38554414e5c30abd0c8e4b970a7c9d09f3a1d8", + "sha256:f3832918bc3c66617f92e35f5d70729187676313caa60c187eb0f28b8fe5e3b5" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.0.0" + "version": "==1.1.0" }, "importlib": { "hashes": [ @@ -524,11 +516,11 @@ }, "invoke": { "hashes": [ - "sha256:1c2cf54c9b9af973ad9704d8ba81b225117cab612568cacbfb3fc42958cc20a9", - "sha256:334495ea16e73948894e9535019f87a88a44b73e7977492b12c2d1b5085f8197", - "sha256:54bdd3fd0245abd1185e05359fd2e4f26be0657cfe7d7bb1bed735e054fa53ab" + "sha256:4f4de934b15c2276caa4fbc5a3b8a61c0eb0b234f2be1780d2b793321995c2d6", + "sha256:dc492f8f17a0746e92081aec3f86ae0b4750bf41607ea2ad87e5a7b5705121b7", + "sha256:eb6f9262d4d25b40330fb21d1e99bf0f85011ccc3526980f8a3eaedd4b43892e" ], - "version": "==1.1.1" + "version": "==1.2.0" }, "jinja2": { "hashes": [ @@ -596,7 +588,7 @@ "sha256:8eb170f8d0d61825e09a95b38be068299ddeda82f35e96c3301a8a5e7604cb83", "sha256:d1aa2a11ba7b8f7b21ab852b1fb5afb277e1bb99d5dfc663380b5015c0d80c5a" ], - "markers": "python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.6' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.3.2" }, "pip-shims": { @@ -659,19 +651,19 @@ }, "pytest": { "hashes": [ - "sha256:2d7c49e931316cc7d1638a3e5f54f5d7b4e5225972b3c9838f3584788d27f349", - "sha256:ad0c7db7b5d4081631e0155f5c61b80ad76ce148551aaafe3a718d65a7508b18" + "sha256:453cbbbe5ce6db38717d282b758b917de84802af4288910c12442984bde7b823", + "sha256:a8a07f84e680482eb51e244370aaf2caa6301ef265f37c2bdefb3dd3b663f99d" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==3.7.4" + "version": "==3.8.0" }, "pytest-cov": { "hashes": [ - "sha256:03aa752cf11db41d281ea1d807d954c4eda35cfa1b21d6971966cc041bbf6e2d", - "sha256:890fe5565400902b0c78b5357004aab1c814115894f4f21370e2433256a3eeec" + "sha256:513c425e931a0344944f84ea47f3956be0e416d95acbd897a44970c8d926d5d7", + "sha256:e360f048b7dae3f2f2a9a4d067b2dd6b6a015d384d1577c994a43f3f7cbad762" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.5.1" + "version": "==2.6.0" }, "pytest-forked": { "hashes": [ @@ -697,12 +689,6 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.23.0" }, - "pytoml": { - "hashes": [ - "sha256:42f76a696182570e93581e763da033c1de484973bad82458387b6dee6c184cbc" - ], - "version": "==0.1.19" - }, "pytz": { "hashes": [ "sha256:a061aa0a9e06881eb8b3b2b43f05b9439d6583c206d0a6c340ff72a7b6669053", @@ -728,19 +714,19 @@ }, "requirementslib": { "hashes": [ - "sha256:7288ec80fa62b78c6193eca85882605c4cdbb60361fc62478bd4a913c0b54400", - "sha256:8bb1b156d8b13337eab359273605b036d60735b350019addabb054e8a89a96f6" + "sha256:90151d8963f814e17190e067b60e92fb35fd1bc46c99f8dba3d7b0d93a3dd958", + "sha256:c3aeaa4e0b80843ba65a68878293e07ea52a8d0706dbba86b02dad6cd20ef2dd" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.1.5" + "version": "==1.1.6" }, "resolvelib": { "hashes": [ - "sha256:d52f2c0762deeb2a4cc34a84371a7a5ac85e111bdc69ce9ae729d8d636606ad6", - "sha256:eb759d43bbf50de9bf36afb9f6c269fabf9ff49084dbfad4ba67252d134bf4b5" + "sha256:6c4c6690b0bdd78bcc002e1a5d1b6abbde58c694a6ea1838f165b20d2c943db7", + "sha256:8734e53271ef98f38a2c99324d5e7905bc00c97dc3fc5bb7d83c82a979e71c04" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.2.1" + "version": "==0.2.2" }, "scandir": { "hashes": [ @@ -756,7 +742,7 @@ "sha256:c9009c527929f6e25604aec39b0a43c3f831d2947d89d6caaab22f057b7055c8", "sha256:f5c71e29b4e2af7ccdc03a020c626ede51da471173b4a6ad1e904f2b2e04b4bd" ], - "markers": "python_version < '3.5' and python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version < '3.6' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.9.0" }, "six": { @@ -764,7 +750,7 @@ "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" ], - "markers": "python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.6' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.11.0" }, "snowballstemmer": { @@ -807,11 +793,11 @@ }, "tomlkit": { "hashes": [ - "sha256:618b057b96b488bb858102bdb38dd9d451c0617667031e320efea2392a39bec4", - "sha256:6856b5395f7c509baad1911aa3940b3c2fb33f53aff968ed1596c393ccea98e5" + "sha256:8ab16e93162fc44d3ad83d2aa29a7140b8f7d996ae1790a73b9a7aed6fb504ac", + "sha256:ca181cee7aee805d455628f7c94eb8ae814763769a93e69157f250fe4ebe1926" ], "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.4.3" + "version": "==0.4.4" }, "towncrier": { "hashes": [ @@ -870,6 +856,14 @@ ], "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.31.1" + }, + "yaspin": { + "hashes": [ + "sha256:36fdccc5e0637b5baa8892fe2c3d927782df7d504e9020f40eb2c1502518aa5a", + "sha256:8e52bf8079a48e2a53f3dfeec9e04addb900c101d1591c85df69cf677d3237e7" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.14.0" } } } diff --git a/src/passa/actions/__init__.py b/src/passa/actions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/passa/actions/add.py b/src/passa/actions/add.py new file mode 100644 index 0000000..6338466 --- /dev/null +++ b/src/passa/actions/add.py @@ -0,0 +1,57 @@ +# -*- coding=utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + +import itertools +import sys + + +def add_packages(packages=[], editables=[], project=None, dev=False, sync=False, clean=False): + from passa.models.lockers import PinReuseLocker + from passa.operations.lock import lock + + lines = list(itertools.chain( + packages, + ("-e {}".format(e) for e in editables), + )) + + project = project + for line in lines: + try: + project.add_line_to_pipfile(line, develop=dev) + except (TypeError, ValueError) as e: + print("Cannot add {line!r} to Pipfile: {error}".format( + line=line, error=str(e), + ), file=sys.stderr) + return 2 + + prev_lockfile = project.lockfile + + locker = PinReuseLocker(project) + success = lock(locker) + if not success: + return 1 + + project._p.write() + project._l.write() + print("Written to project at", project.root) + + if not sync: + return + + from passa.models.synchronizers import Synchronizer + from passa.operations.sync import sync + + lockfile_diff = project.difference_lockfile(prev_lockfile) + default = any(lockfile_diff.default) + develop = any(lockfile_diff.develop) + + syncer = Synchronizer( + project, default=default, develop=develop, + clean_unneeded=clean, + ) + success = sync(syncer) + if not success: + return 1 + + print("Synchronized project at", project.root) diff --git a/src/passa/actions/clean.py b/src/passa/actions/clean.py new file mode 100644 index 0000000..976f11e --- /dev/null +++ b/src/passa/actions/clean.py @@ -0,0 +1,15 @@ +# -*- coding=utf-8 -*- +from __future__ import unicode_literals, absolute_import + + +def clean(project, dev=False): + from passa.models.synchronizers import Cleaner + from passa.operations.sync import clean + + cleaner = Cleaner(project, default=True, develop=dev) + + success = clean(cleaner) + if not success: + return 1 + + print("Cleaned project at", project.root) diff --git a/src/passa/actions/freeze.py b/src/passa/actions/freeze.py new file mode 100644 index 0000000..946543e --- /dev/null +++ b/src/passa/actions/freeze.py @@ -0,0 +1,93 @@ +# -*- coding=utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + +import contextlib +import io +import itertools +import sys + +import vistir.misc + + +def _source_as_lines(source, extra): + url = source["url"] + if extra: + lines = ["--extra-index-url {}".format(url)] + else: + lines = ["--index-url {}".format(url)] + if not source.get("verify_ssl", True): + lines = ["--trusted-host {}".format(url)] + return lines + + +def _requirement_as_line(requirement, sources, include_hashes): + if requirement.index: + sources = sources + else: + sources = None + line = vistir.misc.to_text( + requirement.as_line(sources=sources, include_hashes=include_hashes) + ) + return line + + +@contextlib.contextmanager +def open_for_output(filename): + if filename is None: + yield sys.stdout + return + with io.open(filename, "w", encoding="utf-8", newline="\n") as f: + yield f + + +def freeze(project=None, default=True, dev=True, include_hashes=None, target=None): + from requirementslib import Requirement + + lockfile = project.lockfile + if not lockfile: + print("Pipfile.lock is required to export.", file=sys.stderr) + return 1 + + section_names = [] + if default: + section_names.append("default") + if dev: + section_names.append("develop") + requirements = [ + Requirement.from_pipfile(key, entry._data) + for key, entry in itertools.chain.from_iterable( + lockfile.get(name, {}).items() + for name in section_names + ) + ] + + if include_hashes is None: + include_hashes = all(r.is_named for r in requirements) + + sources = lockfile.meta.sources._data + + source_lines = list(vistir.misc.dedup(itertools.chain( + itertools.chain.from_iterable( + _source_as_lines(source, False) + for source in sources[:1] + ), + itertools.chain.from_iterable( + _source_as_lines(source, True) + for source in sources[1:] + ), + ))) + + requirement_lines = sorted(vistir.misc.dedup( + _requirement_as_line(requirement, sources, include_hashes) + for requirement in requirements + )) + + with open_for_output(target) as f: + for line in source_lines: + f.write(line) + f.write("\n") + f.write("\n") + for line in requirement_lines: + f.write(line) + f.write("\n\n") diff --git a/src/passa/actions/install.py b/src/passa/actions/install.py new file mode 100644 index 0000000..1728dae --- /dev/null +++ b/src/passa/actions/install.py @@ -0,0 +1,32 @@ +# -*- coding=utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + + +def install(project=None, check=True, dev=False, clean=True): + from passa.models.lockers import BasicLocker + from passa.operations.lock import lock + + project = project + + if not check or not project.is_synced(): + locker = BasicLocker(project) + success = lock(locker) + if not success: + return 1 + project._l.write() + print("Written to project at", project.root) + + from passa.models.synchronizers import Synchronizer + from passa.operations.sync import sync + + syncer = Synchronizer( + project, default=True, develop=dev, + clean_unneeded=clean, + ) + + success = sync(syncer) + if not success: + return 1 + + print("Synchronized project at", project.root) diff --git a/src/passa/actions/lock.py b/src/passa/actions/lock.py new file mode 100644 index 0000000..5c04a4a --- /dev/null +++ b/src/passa/actions/lock.py @@ -0,0 +1,15 @@ +# -*- coding=utf-8 -*- + + +def lock(project=None): + from passa.models.lockers import BasicLocker + from passa.operations.lock import lock + + project = project + locker = BasicLocker(project) + success = lock(locker) + if not success: + return + + project._l.write() + print("Written to project at", project.root) diff --git a/src/passa/actions/remove.py b/src/passa/actions/remove.py new file mode 100644 index 0000000..0b6281b --- /dev/null +++ b/src/passa/actions/remove.py @@ -0,0 +1,37 @@ +# -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals + + +def remove(project=None, only="default", packages=[], clean=True): + from passa.models.lockers import PinReuseLocker + from passa.operations.lock import lock + + default = (only != "dev") + develop = (only != "default") + + project = project + project.remove_keys_from_pipfile( + packages, default=default, develop=develop, + ) + + locker = PinReuseLocker(project) + success = lock(locker) + if not success: + return 1 + + project._p.write() + project._l.write() + print("Written to project at", project.root) + + if not clean: + return + + from passa.models.synchronizers import Cleaner + from passa.operations.sync import clean + + cleaner = Cleaner(project, default=True, develop=True) + success = clean(cleaner) + if not success: + return 1 + + print("Cleaned project at", project.root) diff --git a/src/passa/actions/sync.py b/src/passa/actions/sync.py new file mode 100644 index 0000000..1a2d84b --- /dev/null +++ b/src/passa/actions/sync.py @@ -0,0 +1,18 @@ +# -*- coding=utf-8 -*- + + +def sync(project=None, dev=False, clean=True): + from passa.models.synchronizers import Synchronizer + from passa.operations.sync import sync + + project = project + syncer = Synchronizer( + project, default=True, develop=dev, + clean_unneeded=clean, + ) + + success = sync(syncer) + if not success: + return 1 + + print("Synchronized project at", project.root) diff --git a/src/passa/actions/upgrade.py b/src/passa/actions/upgrade.py new file mode 100644 index 0000000..a1aea8c --- /dev/null +++ b/src/passa/actions/upgrade.py @@ -0,0 +1,49 @@ +# -*- coding=utf-8 -*- +import sys + + +def upgrade(project=None, strategy="only-if-needed", sync=True, packages=[]): + from passa.models.lockers import EagerUpgradeLocker, PinReuseLocker + from passa.operations.lock import lock + + for package in packages: + if not project.contains_key_in_pipfile(package): + print("{package!r} not found in Pipfile".format( + package=package, + ), file=sys.stderr) + return 2 + + project.remove_keys_from_lockfile(packages) + + prev_lockfile = project.lockfile + + if strategy == "eager": + locker = EagerUpgradeLocker(project, packages) + else: + locker = PinReuseLocker(project) + success = lock(locker) + if not success: + return 1 + + project._l.write() + print("Written to project at", project.root) + + if not sync: + return + + from passa.operations.sync import sync + from passa.models.synchronizers import Synchronizer + + lockfile_diff = project.difference_lockfile(prev_lockfile) + default = bool(any(lockfile_diff.default)) + develop = bool(any(lockfile_diff.develop)) + + syncer = Synchronizer( + project, default=default, develop=develop, + clean_unneeded=False, + ) + success = sync(syncer) + if not success: + return 1 + + print("Synchronized project at", project.root) diff --git a/src/passa/cli/__init__.py b/src/passa/cli/__init__.py index cb503e8..b6891d3 100644 --- a/src/passa/cli/__init__.py +++ b/src/passa/cli/__init__.py @@ -34,7 +34,7 @@ def main(argv=None): continue parser = subparsers.add_parser(klass.name, help=klass.description) command = klass(parser) - parser.set_defaults(func=command.main) + parser.set_defaults(func=command.run) options = root_parser.parse_args(argv) diff --git a/src/passa/cli/_base.py b/src/passa/cli/_base.py index 68e0e34..71c7cc7 100644 --- a/src/passa/cli/_base.py +++ b/src/passa/cli/_base.py @@ -8,6 +8,8 @@ import tomlkit.exceptions +from .options import project + def build_project(root): # This is imported lazily to reduce import overhead. Not evey command @@ -39,18 +41,30 @@ class BaseCommand(object): name = None description = None parsed_main = None + arguments = [] - def __init__(self, parser): + def __init__(self, parser=None): + if not parser: + parser = argparse.ArgumentParser( + prog=os.path.basename(sys.argv[0]), + description="Base argument parser for passa" + ) self.parser = parser + self.default_aguments = [project] self.add_arguments() @classmethod - def run_current_module(cls): + def build_parser(cls): parser = argparse.ArgumentParser( prog="passa {}".format(cls.name), description=cls.description, ) - cls(parser)() + return cls(parser) + + @classmethod + def run_parser(cls): + parser = cls.build_parser() + parser() def __call__(self, argv=None): options = self.parser.parse_args(argv) @@ -58,16 +72,17 @@ def __call__(self, argv=None): if result is not None: sys.exit(result) + def add_default_arguments(self): + for arg in self.default_aguments: + arg.add_to_parser(self.parser) + def add_arguments(self): - self.parser.add_argument( - "--project", - metavar="project", - default=os.getcwd(), - type=build_project, - help="path to project root (directory containing Pipfile)", - ) + self.add_default_arguments() + for arg in self.arguments: + arg.add_to_parser(self.parser) def main(self, options): - # This __dict__ access is needed for Python 2 to prevent Python from - # wrapping parsed_main into an unbounded method. - return type(self).__dict__["parsed_main"](options) + return self.run(options) + + def run(self, options): + raise NotImplementedError diff --git a/src/passa/cli/add.py b/src/passa/cli/add.py index 26ce0ed..62f9e70 100644 --- a/src/passa/cli/add.py +++ b/src/passa/cli/add.py @@ -2,98 +2,27 @@ from __future__ import absolute_import, print_function, unicode_literals -import itertools -import sys - +from ..actions.add import add_packages from ._base import BaseCommand - - -def main(options): - from passa.internals.lockers import PinReuseLocker - from passa.operations.lock import lock - - lines = list(itertools.chain( - options.requirement_lines, - ("-e {}".format(e) for e in options.editable_lines), - )) - - project = options.project - for line in lines: - try: - project.add_line_to_pipfile(line, develop=options.dev) - except (TypeError, ValueError) as e: - print("Cannot add {line!r} to Pipfile: {error}".format( - line=line, error=str(e), - ), file=sys.stderr) - return 2 - - prev_lockfile = project.lockfile - - locker = PinReuseLocker(project) - success = lock(locker) - if not success: - return 1 - - project._p.write() - project._l.write() - print("Written to project at", project.root) - - if not options.sync: - return - - from passa.internals.synchronizers import Synchronizer - from passa.operations.sync import sync - - lockfile_diff = project.difference_lockfile(prev_lockfile) - default = bool(any(lockfile_diff.default)) - develop = bool(any(lockfile_diff.develop)) - - syncer = Synchronizer( - project, default=default, develop=develop, - clean_unneeded=False, - ) - success = sync(syncer) - if not success: - return 1 - - print("Synchronized project at", project.root) +from .options import package_group class Command(BaseCommand): name = "add" description = "Add packages to project." - parsed_main = main + arguments = [package_group] - def add_arguments(self): - super(Command, self).add_arguments() - self.parser.add_argument( - "requirement_lines", metavar="requirement", - nargs="*", - help="requirement to add (can be used multiple times)", - ) - self.parser.add_argument( - "-e", "--editable", - metavar="requirement", dest="editable_lines", - action="append", default=[], - help="editable requirement to add (can be used multiple times)", - ) - self.parser.add_argument( - "--dev", - action="store_true", - help="add packages to [dev-packages]", - ) - self.parser.add_argument( - "--no-sync", dest="sync", - action="store_false", default=True, - help="do not synchronize the environment", - ) - - def main(self, options): + def run(self, options): if not options.editable_lines and not options.requirement_lines: self.parser.error("Must supply either a requirement or --editable") - return super(Command, self).main(options) + return add_packages( + packages=options.packages, + editables=options.editables, + project=options.project, + dev=options.dev + ) if __name__ == "__main__": - Command.run_current_module() + Command.run_parser() diff --git a/src/passa/cli/clean.py b/src/passa/cli/clean.py index cd1b679..e23d5ee 100644 --- a/src/passa/cli/clean.py +++ b/src/passa/cli/clean.py @@ -2,37 +2,20 @@ from __future__ import absolute_import, print_function, unicode_literals +from ..actions.clean import clean from ._base import BaseCommand - - -def main(options): - from passa.internals.synchronizers import Cleaner - from passa.operations.sync import clean - - project = options.project - cleaner = Cleaner(project, default=True, develop=options.dev) - - success = clean(cleaner) - if not success: - return 1 - - print("Cleaned project at", project.root) +from .options import dev, no_default class Command(BaseCommand): name = "clean" description = "Uninstall unlisted packages from the environment." - parsed_main = main + arguments = [dev, no_default] - def add_arguments(self): - super(Command, self).add_arguments() - self.parser.add_argument( - "--no-dev", dest="dev", - action="store_false", default=True, - help="uninstall develop packages, only keep default ones", - ) + def run(self, options): + return clean(project=options.project, default=options.default, dev=options.dev) if __name__ == "__main__": - Command.run_current_module() + Command.run_parser() diff --git a/src/passa/cli/freeze.py b/src/passa/cli/freeze.py index 6ca5717..053c727 100644 --- a/src/passa/cli/freeze.py +++ b/src/passa/cli/freeze.py @@ -2,137 +2,23 @@ from __future__ import absolute_import, print_function, unicode_literals -import contextlib -import io -import itertools -import sys - -import six -import vistir.misc - +from ..actions.freeze import freeze from ._base import BaseCommand - - -def _source_as_lines(source, extra): - url = source["url"] - if extra: - lines = ["--extra-index-url {}".format(url)] - else: - lines = ["--index-url {}".format(url)] - if not source.get("verify_ssl", True): - lines = ["--trusted-host {}".format(url)] - return lines - - -def _requirement_as_line(requirement, sources, include_hashes): - if requirement.index: - sources = sources - else: - sources = None - line = requirement.as_line(sources=sources, include_hashes=include_hashes) - if not isinstance(line, six.text_type): - line = line.decode("utf-8") - return line - - -@contextlib.contextmanager -def open_for_output(filename): - if filename is None: - yield sys.stdout - return - with io.open(filename, "w", encoding="utf-8", newline="\n") as f: - yield f - - -def main(options): - from requirementslib import Requirement - - lockfile = options.project.lockfile - if not lockfile: - print("Pipfile.lock is required to export.", file=sys.stderr) - return 1 - - section_names = [] - if options.default: - section_names.append("default") - if options.dev: - section_names.append("develop") - requirements = [ - Requirement.from_pipfile(key, entry._data) - for key, entry in itertools.chain.from_iterable( - lockfile.get(name, {}).items() - for name in section_names - ) - ] - - include_hashes = options.include_hashes - if include_hashes is None: - include_hashes = all(r.is_named for r in requirements) - - sources = lockfile.meta.sources._data - - source_lines = list(vistir.misc.dedup(itertools.chain( - itertools.chain.from_iterable( - _source_as_lines(source, False) - for source in sources[:1] - ), - itertools.chain.from_iterable( - _source_as_lines(source, True) - for source in sources[1:] - ), - ))) - - requirement_lines = sorted(vistir.misc.dedup( - _requirement_as_line(requirement, sources, include_hashes) - for requirement in requirements - )) - - with open_for_output(options.target) as f: - for line in source_lines: - f.write(line) - f.write("\n") - f.write("\n") - for line in requirement_lines: - f.write(line) - f.write("\n\n") +from .options import dev, include_hashes_group, no_default, target class Command(BaseCommand): name = "freeze" description = "Export project depenencies to requirements.txt." - parsed_main = main + arguments = [dev, no_default, target, include_hashes_group] - def add_arguments(self): - super(Command, self).add_arguments() - self.parser.add_argument( - "--target", - default=None, - help="file to export into (default is to print to stdout)", - ) - self.parser.add_argument( - "--dev", - action="store_true", default=False, - help="include development packages in requirements.txt", - ) - self.parser.add_argument( - "--no-default", dest="default", - action="store_false", default=True, - help="do not include default packages in requirements.txt", - ) - include_hashes_group = self.parser.add_mutually_exclusive_group() - include_hashes_group.add_argument( - "--include-hashes", dest="include_hashes", - action="store_true", - help="output hashes in requirements.txt (default is to guess)", - ) - include_hashes_group.add_argument( - "--no-include-hashes", dest="include_hashes", - action="store_false", - help=("do not output hashes in requirements.txt " - "(default is to guess)"), + def run(self, options): + return freeze( + project=options.project, default=options.default, dev=options.dev, + include_hashes=options.include_hashes ) if __name__ == "__main__": - Command.run_current_module() + Command.run_parser() diff --git a/src/passa/cli/install.py b/src/passa/cli/install.py index f47377b..1c0b459 100644 --- a/src/passa/cli/install.py +++ b/src/passa/cli/install.py @@ -2,62 +2,21 @@ from __future__ import absolute_import, print_function, unicode_literals +from ..actions.install import install from ._base import BaseCommand - - -def main(options): - from passa.internals.lockers import BasicLocker - from passa.operations.lock import lock - - project = options.project - - if not options.check or not project.is_synced(): - locker = BasicLocker(project) - success = lock(locker) - if not success: - return 1 - project._l.write() - print("Written to project at", project.root) - - from passa.internals.synchronizers import Synchronizer - from passa.operations.sync import sync - - syncer = Synchronizer( - project, default=True, develop=options.dev, - clean_unneeded=options.clean, - ) - - success = sync(syncer) - if not success: - return 1 - - print("Synchronized project at", project.root) +from .options import dev, no_check, no_clean class Command(BaseCommand): name = "install" description = "Generate Pipfile.lock to synchronize the environment." - parsed_main = main - - def add_arguments(self): - super(Command, self).add_arguments() - self.parser.add_argument( - "--no-check", dest="check", - action="store_false", default=True, - help="do not check if Pipfile.lock is update, always resolve", - ) - self.parser.add_argument( - "--dev", - action="store_true", - help="install develop packages", - ) - self.parser.add_argument( - "--no-clean", dest="clean", - action="store_false", default=True, - help="do not uninstall packages not specified in Pipfile.lock", - ) + arguments = [no_check, dev, no_clean] + + def run(self, options): + return install(project=options.project, check=options.check, dev=options.dev, + clean=options.clean) if __name__ == "__main__": - Command.run_current_module() + Command.run_parser() diff --git a/src/passa/cli/lock.py b/src/passa/cli/lock.py index 67b1d11..9b0651a 100644 --- a/src/passa/cli/lock.py +++ b/src/passa/cli/lock.py @@ -2,28 +2,17 @@ from __future__ import absolute_import, print_function, unicode_literals +from ..actions.lock import lock from ._base import BaseCommand -def main(options): - from passa.internals.lockers import BasicLocker - from passa.operations.lock import lock - - project = options.project - locker = BasicLocker(project) - success = lock(locker) - if not success: - return - - project._l.write() - print("Written to project at", project.root) - - class Command(BaseCommand): name = "lock" description = "Generate Pipfile.lock." - parsed_main = main + + def run(self, options): + return lock(project=options.project) if __name__ == "__main__": - Command.run_current_module() + Command.run_parser() diff --git a/src/passa/cli/options.py b/src/passa/cli/options.py new file mode 100644 index 0000000..a70ffd6 --- /dev/null +++ b/src/passa/cli/options.py @@ -0,0 +1,137 @@ +# -*- coding=utf-8 -*- +from __future__ import absolute_import + +import argparse +import os + +import tomlkit.exceptions + +import passa.internals.projects +import vistir + + +class Project(passa.internals.projects.Project): + def __init__(self, root, *args, **kwargs): + root = vistir.compat.Path(root).absolute() + pipfile = root.joinpath("Pipfile") + if not pipfile.is_file(): + raise argparse.ArgumentError( + "{0!r} is not a Pipfile project".format(root), + ) + try: + super(Project, self).__init__(root.as_posix(), *args, **kwargs) + except tomlkit.exceptions.ParseError as e: + raise argparse.ArgumentError( + "failed to parse Pipfile: {0!r}".format(str(e)), + ) + + def __name__(self): + return "Project Root" + + +class Option(object): + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + + def add_to_parser(self, parser): + parser.add_argument(*self.args, **self.kwargs) + + def add_to_group(self, group): + group.add_argument(*self.args, **self.kwargs) + + +class ArgumentGroup(object): + def __init__(self, name, parser=None, is_mutually_exclusive=False, required=None, options=[]): + self.name = name + self.options = options + self.parser = parser + self.required = required + self.is_mutually_exclusive = is_mutually_exclusive + self.argument_group = None + + def add_to_parser(self, parser): + group = None + if self.is_mutually_exclusive: + group = parser.add_mutually_exclusive_group(required=self.required) + else: + group = parser.add_argument_group() + for option in self.options: + option.add_to_group(group) + self.argument_group = group + self.parser = parser + + +project = Option( + "--project", metavar="project", default=os.getcwd(), type=Project, + help="path to project root (directory containing Pipfile)", +) + +packages = Option( + "packages", metavar="package", nargs="*", + help="requirement to add (can be used multiple times)", +) + +editable = Option( + '-e', '--editable', dest='editables', nargs="*", default=[], metavar='path/vcs', + help="editable requirement to add (can be used multiple times)", +) + +dev = Option( + "--dev", action="store_true", default=False, + help="Use [dev-packages] for install/freeze/uninstall operations", +) + +no_sync = Option( + "--no-sync", dest="sync", action="store_false", default=True, + help="do not synchronize the environment", +) + +target = Option( + "-t", "--target", default=None, + help="file to export into (default is to print to stdout)" +) + +no_default = Option( + "--no-default", dest="default", action="store_false", default=True, + help="do not include default packages when exporting, importing, or cleaning" +) + +include_hashes = Option( + "--include-hashes", dest="include_hashes", action="store_true", + help="output hashes in requirements.txt (default is to guess)", +) + +no_include_hashes = Option( + "--no-include-hashes", dest="include_hashes", action="store_false", + help="do not output hashes in requirements.txt (default is to guess)", +) + +no_check = Option( + "--no-check", dest="check", action="store_false", default=True, + help="do not check if Pipfile.lock is up to date, always resolve", +) + +no_clean = Option( + "--no-clean", dest="clean", action="store_false", default=True, + help="do not remove packages not specified in Pipfile.lock", +) + +dev_only = Option( + "--dev", dest="only", action="store_const", const="dev", + help="only try to modify [dev-packages]", +) + +default_only = Option( + "--default", dest="only", action="store_const", const="default", + help="only try to modify [default]", +) + +strategy = Option( + "--strategy", choices=["eager", "only-if-needed"], default="only-if-needed", + help="how dependency upgrading is handled", +) + +include_hashes_group = ArgumentGroup("include_hashes", is_mutually_exclusive=True, options=[include_hashes, no_include_hashes]) +dev_group = ArgumentGroup("dev", is_mutually_exclusive="True", options=[dev_only, default_only]) +package_group = ArgumentGroup("packages", options=[packages, editable, dev, no_sync]) diff --git a/src/passa/cli/remove.py b/src/passa/cli/remove.py index b1dbfd7..538acbf 100644 --- a/src/passa/cli/remove.py +++ b/src/passa/cli/remove.py @@ -2,74 +2,21 @@ from __future__ import absolute_import, print_function, unicode_literals +from ..actions.remove import remove from ._base import BaseCommand - - -def main(options): - from passa.internals.lockers import PinReuseLocker - from passa.operations.lock import lock - - default = (options.only != "dev") - develop = (options.only != "default") - - project = options.project - project.remove_keys_from_pipfile( - options.packages, default=default, develop=develop, - ) - - locker = PinReuseLocker(project) - success = lock(locker) - if not success: - return 1 - - project._p.write() - project._l.write() - print("Written to project at", project.root) - - if not options.clean: - return - - from passa.internals.synchronizers import Cleaner - from passa.operations.sync import clean - - cleaner = Cleaner(project, default=True, develop=True) - success = clean(cleaner) - if not success: - return 1 - - print("Cleaned project at", project.root) +from .options import dev_group, no_clean, packages class Command(BaseCommand): name = "remove" description = "Remove packages from project." - parsed_main = main + arguments = [dev_group, no_clean, packages] - def add_arguments(self): - super(Command, self).add_arguments() - self.parser.add_argument( - "packages", metavar="package", - nargs="+", - help="package to remove (can be used multiple times)", - ) - dev_group = self.parser.add_mutually_exclusive_group() - dev_group.add_argument( - "--dev", dest="only", - action="store_const", const="dev", - help="only try to remove from [dev-packages]", - ) - dev_group.add_argument( - "--default", dest="only", - action="store_const", const="default", - help="only try to remove from [packages]", - ) - self.parser.add_argument( - "--no-clean", dest="clean", - action="store_false", default=True, - help="do not uninstall packages not specified in Pipfile.lock", - ) + def run(self, options): + return remove(project=options.project, only=options.only, + packages=options.packages, clean=options.clean) if __name__ == "__main__": - Command.run_current_module() + Command.run_parser() diff --git a/src/passa/cli/sync.py b/src/passa/cli/sync.py index ade81e0..a09b784 100644 --- a/src/passa/cli/sync.py +++ b/src/passa/cli/sync.py @@ -2,45 +2,20 @@ from __future__ import absolute_import, print_function, unicode_literals +from ..actions.sync import sync from ._base import BaseCommand - - -def main(options): - from passa.internals.synchronizers import Synchronizer - from passa.operations.sync import sync - - project = options.project - syncer = Synchronizer( - project, default=True, develop=options.dev, - clean_unneeded=options.clean, - ) - - success = sync(syncer) - if not success: - return 1 - - print("Synchronized project at", project.root) +from .options import dev, no_clean class Command(BaseCommand): name = "sync" description = "Install Pipfile.lock into the environment." - parsed_main = main - - def add_arguments(self): - super(Command, self).add_arguments() - self.parser.add_argument( - "--dev", - action="store_true", - help="install develop packages", - ) - self.parser.add_argument( - "--no-clean", dest="clean", - action="store_false", default=True, - help="do not uninstall packages not specified in Pipfile.lock", - ) + arguments = [dev, no_clean] + + def run(self, options): + return sync(project=options.project, dev=options.dev, clean=options.clean) if __name__ == "__main__": - Command.run_current_module() + Command.run_parser() diff --git a/src/passa/cli/upgrade.py b/src/passa/cli/upgrade.py index 011fff6..cf7f502 100644 --- a/src/passa/cli/upgrade.py +++ b/src/passa/cli/upgrade.py @@ -1,91 +1,21 @@ # -*- coding=utf-8 -*- - from __future__ import absolute_import, print_function, unicode_literals -import sys - +from ..actions.upgrade import upgrade from ._base import BaseCommand - - -def main(options): - from passa.internals.lockers import EagerUpgradeLocker, PinReuseLocker - from passa.operations.lock import lock - - project = options.project - packages = options.packages - for package in packages: - if not project.contains_key_in_pipfile(package): - print("{package!r} not found in Pipfile".format( - package=package, - ), file=sys.stderr) - return 2 - - project.remove_keys_from_lockfile(packages) - - prev_lockfile = project.lockfile - - if options.strategy == "eager": - locker = EagerUpgradeLocker(project, packages) - else: - locker = PinReuseLocker(project) - success = lock(locker) - if not success: - return 1 - - project._l.write() - print("Written to project at", project.root) - - if not options.sync: - return - - from passa.operations.sync import sync - from passa.internals.synchronizers import Synchronizer - - lockfile_diff = project.difference_lockfile(prev_lockfile) - default = bool(any(lockfile_diff.default)) - develop = bool(any(lockfile_diff.develop)) - - syncer = Synchronizer( - project, default=default, develop=develop, - clean_unneeded=False, - ) - success = sync(syncer) - if not success: - return 1 - - print("Synchronized project at", project.root) +from .options import no_clean, no_sync, packages, strategy class Command(BaseCommand): name = "upgrade" description = "Upgrade packages in project." - parsed_main = main + arguments = [packages, strategy, no_clean, no_sync] - def add_arguments(self): - super(Command, self).add_arguments() - self.parser.add_argument( - "packages", metavar="package", - nargs="+", - help="package to upgrade (can be used multiple times)", - ) - self.parser.add_argument( - "--strategy", - choices=["eager", "only-if-needed"], - default="only-if-needed", - help="how dependency upgrading is handled", - ) - self.parser.add_argument( - "--no-sync", dest="sync", - action="store_false", default=True, - help="do not synchronize the environment", - ) - self.parser.add_argument( - "--no-clean", dest="clean", - action="store_false", default=True, - help="do not uninstall packages not specified in Pipfile.lock", - ) + def run(self, options): + return upgrade(project=options.project, strategy=options.strategy, + sync=options.sync, packages=options.packages) if __name__ == "__main__": - Command.run_current_module() + Command.run_parser() diff --git a/src/passa/internals/_pip.py b/src/passa/internals/_pip.py index f73e241..2aa143a 100644 --- a/src/passa/internals/_pip.py +++ b/src/passa/internals/_pip.py @@ -17,8 +17,8 @@ import six import vistir +from ..models.caches import CACHE_DIR from ._pip_shims import VCS_SUPPORT, build_wheel as _build_wheel, unpack_url -from .caches import CACHE_DIR from .utils import filter_sources diff --git a/src/passa/internals/dependencies.py b/src/passa/internals/dependencies.py index 1fa3b0b..410a5e6 100644 --- a/src/passa/internals/dependencies.py +++ b/src/passa/internals/dependencies.py @@ -13,8 +13,8 @@ import requirementslib import six +from ..models.caches import DependencyCache, RequiresPythonCache from ._pip import WheelBuildError, build_wheel, read_sdist_metadata -from .caches import DependencyCache, RequiresPythonCache from .markers import contains_extra, get_contained_extras, get_without_extra from .utils import get_pinned_version, is_pinned diff --git a/src/passa/models/__init__.py b/src/passa/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/passa/internals/caches.py b/src/passa/models/caches.py similarity index 98% rename from src/passa/internals/caches.py rename to src/passa/models/caches.py index 6d3131f..c6d29b5 100644 --- a/src/passa/internals/caches.py +++ b/src/passa/models/caches.py @@ -13,8 +13,8 @@ import requests import vistir -from ._pip_shims import VCS_SUPPORT -from .utils import get_pinned_version +from ..internals._pip_shims import VCS_SUPPORT +from ..internals.utils import get_pinned_version CACHE_DIR = os.environ.get("PASSA_CACHE_DIR", appdirs.user_cache_dir("passa")) diff --git a/src/passa/internals/lockers.py b/src/passa/models/lockers.py similarity index 97% rename from src/passa/internals/lockers.py rename to src/passa/models/lockers.py index b567c46..c25ca60 100644 --- a/src/passa/internals/lockers.py +++ b/src/passa/models/lockers.py @@ -4,18 +4,19 @@ import itertools +import resolvelib + import plette import requirementslib -import resolvelib import vistir +from ..internals.hashes import get_hashes +from ..internals.reporters import StdOutReporter +from ..internals.traces import trace_graph +from ..internals.utils import identify_requirment from .caches import HashCache -from .hashes import get_hashes from .metadata import set_metadata from .providers import BasicProvider, EagerUpgradeProvider, PinReuseProvider -from .reporters import StdOutReporter -from .traces import trace_graph -from .utils import identify_requirment def _get_requirements(model, section_name): diff --git a/src/passa/internals/metadata.py b/src/passa/models/metadata.py similarity index 97% rename from src/passa/internals/metadata.py rename to src/passa/models/metadata.py index 9709c53..a949f1e 100644 --- a/src/passa/internals/metadata.py +++ b/src/passa/models/metadata.py @@ -10,8 +10,8 @@ import vistir import vistir.misc -from .markers import get_without_extra -from .specifiers import cleanup_pyspecs, pyspec_from_markers +from ..internals.markers import get_without_extra +from ..internals.specifiers import cleanup_pyspecs, pyspec_from_markers def dedup_markers(s): diff --git a/src/passa/internals/providers.py b/src/passa/models/providers.py similarity index 97% rename from src/passa/internals/providers.py rename to src/passa/models/providers.py index bf4fd22..36b2f2e 100644 --- a/src/passa/internals/providers.py +++ b/src/passa/models/providers.py @@ -6,11 +6,10 @@ import resolvelib -from .candidates import find_candidates -from .dependencies import get_dependencies -from .utils import ( - filter_sources, get_allow_prereleases, - are_requirements_equal, identify_requirment, strip_extras, +from ..internals.candidates import find_candidates +from ..internals.dependencies import get_dependencies +from ..internals.utils import ( + filter_sources, get_allow_prereleases, identify_requirment, strip_extras, ) diff --git a/src/passa/internals/synchronizers.py b/src/passa/models/synchronizers.py similarity index 98% rename from src/passa/internals/synchronizers.py rename to src/passa/models/synchronizers.py index 7ccce23..bad4905 100644 --- a/src/passa/internals/synchronizers.py +++ b/src/passa/models/synchronizers.py @@ -14,7 +14,7 @@ import packaging.version import requirementslib -from ._pip import uninstall, EditableInstaller, WheelInstaller +from ..internals._pip import uninstall, EditableInstaller, WheelInstaller def _is_installation_local(name): From cadad159c2a5fff083caefc00e7fd3d329b889c0 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 14 Sep 2018 00:18:02 -0400 Subject: [PATCH 02/32] Add forgotten __future__ imports - Also move `projects` and fix imports Signed-off-by: Dan Ryan --- src/passa/actions/clean.py | 3 ++- src/passa/actions/lock.py | 2 ++ src/passa/actions/remove.py | 1 + src/passa/actions/sync.py | 2 ++ src/passa/actions/upgrade.py | 3 +++ src/passa/cli/_base.py | 24 --------------------- src/passa/{internals => models}/projects.py | 0 7 files changed, 10 insertions(+), 25 deletions(-) rename src/passa/{internals => models}/projects.py (100%) diff --git a/src/passa/actions/clean.py b/src/passa/actions/clean.py index 976f11e..3570e4d 100644 --- a/src/passa/actions/clean.py +++ b/src/passa/actions/clean.py @@ -1,5 +1,6 @@ # -*- coding=utf-8 -*- -from __future__ import unicode_literals, absolute_import + +from __future__ import absolute_import, print_function, unicode_literals def clean(project, dev=False): diff --git a/src/passa/actions/lock.py b/src/passa/actions/lock.py index 5c04a4a..7c09469 100644 --- a/src/passa/actions/lock.py +++ b/src/passa/actions/lock.py @@ -1,5 +1,7 @@ # -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals + def lock(project=None): from passa.models.lockers import BasicLocker diff --git a/src/passa/actions/remove.py b/src/passa/actions/remove.py index 0b6281b..158f5e6 100644 --- a/src/passa/actions/remove.py +++ b/src/passa/actions/remove.py @@ -1,4 +1,5 @@ # -*- coding=utf-8 -*- + from __future__ import absolute_import, print_function, unicode_literals diff --git a/src/passa/actions/sync.py b/src/passa/actions/sync.py index 1a2d84b..23e36ee 100644 --- a/src/passa/actions/sync.py +++ b/src/passa/actions/sync.py @@ -1,5 +1,7 @@ # -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals + def sync(project=None, dev=False, clean=True): from passa.models.synchronizers import Synchronizer diff --git a/src/passa/actions/upgrade.py b/src/passa/actions/upgrade.py index a1aea8c..fb3ad7f 100644 --- a/src/passa/actions/upgrade.py +++ b/src/passa/actions/upgrade.py @@ -1,4 +1,7 @@ # -*- coding=utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import sys diff --git a/src/passa/cli/_base.py b/src/passa/cli/_base.py index 71c7cc7..aeaefa1 100644 --- a/src/passa/cli/_base.py +++ b/src/passa/cli/_base.py @@ -11,30 +11,6 @@ from .options import project -def build_project(root): - # This is imported lazily to reduce import overhead. Not evey command - # needs the project instance. - from passa.internals.projects import Project - root = os.path.abspath(root) - if not os.path.isfile(os.path.join(root, "Pipfile")): - raise argparse.ArgumentError( - "{0!r} is not a Pipfile project".format(root), - ) - try: - project = Project(root) - except tomlkit.exceptions.ParseError as e: - raise argparse.ArgumentError( - "failed to parse Pipfile: {0!r}".format(str(e)), - ) - return project - - -# Better error reporting. Recent argparse would emit something like -# "invalid project root value: 'xxxxxx'". The str() wrapper is needed to -# keep Python 2 happy :( -build_project.__name__ = str("project root") - - class BaseCommand(object): """A CLI command. """ diff --git a/src/passa/internals/projects.py b/src/passa/models/projects.py similarity index 100% rename from src/passa/internals/projects.py rename to src/passa/models/projects.py From edc773464ce86a0fa355700d9d0ab9945e4d3e96 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 14 Sep 2018 00:20:09 -0400 Subject: [PATCH 03/32] drop unused import Signed-off-by: Dan Ryan --- src/passa/cli/_base.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/passa/cli/_base.py b/src/passa/cli/_base.py index aeaefa1..a6ea33e 100644 --- a/src/passa/cli/_base.py +++ b/src/passa/cli/_base.py @@ -6,8 +6,6 @@ import os import sys -import tomlkit.exceptions - from .options import project From 9afcc4ec4f248cedc3428fc380d59e0ab17b29e4 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 14 Sep 2018 00:25:06 -0400 Subject: [PATCH 04/32] Fix missed import Signed-off-by: Dan Ryan --- src/passa/cli/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/passa/cli/options.py b/src/passa/cli/options.py index a70ffd6..988183d 100644 --- a/src/passa/cli/options.py +++ b/src/passa/cli/options.py @@ -6,7 +6,7 @@ import tomlkit.exceptions -import passa.internals.projects +import passa.models.projects import vistir From 76ca23e60d2ca37893e5f20927f99e9c3f3e1f69 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 14 Sep 2018 00:33:43 -0400 Subject: [PATCH 05/32] sigh Signed-off-by: Dan Ryan --- src/passa/cli/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/passa/cli/options.py b/src/passa/cli/options.py index 988183d..5f454b3 100644 --- a/src/passa/cli/options.py +++ b/src/passa/cli/options.py @@ -10,7 +10,7 @@ import vistir -class Project(passa.internals.projects.Project): +class Project(passa.models.projects.Project): def __init__(self, root, *args, **kwargs): root = vistir.compat.Path(root).absolute() pipfile = root.joinpath("Pipfile") From 702853d838a78dfada0baccb2c5466518c7d14d0 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 17 Sep 2018 01:08:36 -0400 Subject: [PATCH 06/32] Update license, add news - A few minor fixes with the restructure Signed-off-by: Dan Ryan --- LICENSE | 2 +- news/53.feature.rst | 1 + src/passa/actions/freeze.py | 2 +- src/passa/actions/init.py | 59 +++++++++++++++++++++++++++++++++++++ src/passa/cli/_base.py | 5 ++-- src/passa/cli/add.py | 2 +- src/passa/cli/init.py | 32 ++++++++++++++++++++ src/passa/cli/options.py | 16 ++++++++++ 8 files changed, 113 insertions(+), 6 deletions(-) create mode 100644 news/53.feature.rst create mode 100644 src/passa/actions/init.py create mode 100644 src/passa/cli/init.py diff --git a/LICENSE b/LICENSE index e1a278e..cd41e27 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2018, Dan Ryan +Copyright (c) 2018, Dan Ryan and Tzu-ping Chung Permission to use, copy, modify, and distribute this software for any purpose with or without fee is hereby granted, provided that the above diff --git a/news/53.feature.rst b/news/53.feature.rst new file mode 100644 index 0000000..2589034 --- /dev/null +++ b/news/53.feature.rst @@ -0,0 +1 @@ +Refactored and restructured the internals for improved organization and separation of concerns. diff --git a/src/passa/actions/freeze.py b/src/passa/actions/freeze.py index 946543e..ca4dbb2 100644 --- a/src/passa/actions/freeze.py +++ b/src/passa/actions/freeze.py @@ -90,4 +90,4 @@ def freeze(project=None, default=True, dev=True, include_hashes=None, target=Non f.write("\n") for line in requirement_lines: f.write(line) - f.write("\n\n") + f.write("\n") diff --git a/src/passa/actions/init.py b/src/passa/actions/init.py new file mode 100644 index 0000000..1d9f592 --- /dev/null +++ b/src/passa/actions/init.py @@ -0,0 +1,59 @@ +# -*- coding=utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + +import io +import os +from pip_shims import Command as PipCommand, cmdoptions +import plette +import six +import vistir + + +class PipCmd(PipCommand): + name = "PipCmd" + + +def get_sources(urls, trusted_hosts): + trusted_hosts = [six.moves.urllib.parse.urlparse(url).netloc for url in trusted_hosts] + sources = [] + for url in urls: + parsed_url = six.moves.urllib.parse.urlparse(url) + netloc = parsed_url.netloc + if '@' in netloc: + _, _, netloc = netloc.rpartition('@') + name, _, _ = netloc.partition('.') # Just use the domain name as the source name + verify_ssl = True + if netloc in trusted_hosts: + verify_ssl = False + sources.append({"url": url, "name": name, "verify_ssl": verify_ssl}) + return sources + + +def init_project(root=None, python_version=None): + pipfile_path = os.path.join(root, "Pipfile") + if os.path.isfile(pipfile_path): + raise RuntimeError("{0!r} is already a Pipfile project".format(root)) + if not os.path.exists(root): + vistir.path.mkdir_p(root, mode=0o755) + pip_command = PipCmd() + cmdoptions.make_option_group(cmdoptions.index_group, pip_command.parser) + parsed, _ = pip_command.parser.parse_args([]) + index_urls = [parsed.index_url] + parsed.extra_index_urls + sources = get_sources(index_urls, parsed.trusted_hosts) + data = { + "sources": sources, + "packages": {}, + "dev-packages": {}, + } + if python_version: + data["requires"] = {"python_version": python_version} + return create_project(pipfile_path=pipfile_path, data=data) + + +def create_project(pipfile_path, data={}): + pipfile = plette.pipfiles.Pipfile(data=data) + with io.open(pipfile_path, "w") as fh: + pipfile.dump(fh) + print("Successfully created new pipfile at {0!r}".format(pipfile_path)) + return 0 diff --git a/src/passa/cli/_base.py b/src/passa/cli/_base.py index a6ea33e..0ca4868 100644 --- a/src/passa/cli/_base.py +++ b/src/passa/cli/_base.py @@ -14,7 +14,7 @@ class BaseCommand(object): """ name = None description = None - parsed_main = None + default_arguments = [project] arguments = [] def __init__(self, parser=None): @@ -24,7 +24,6 @@ def __init__(self, parser=None): description="Base argument parser for passa" ) self.parser = parser - self.default_aguments = [project] self.add_arguments() @classmethod @@ -47,7 +46,7 @@ def __call__(self, argv=None): sys.exit(result) def add_default_arguments(self): - for arg in self.default_aguments: + for arg in self.default_arguments: arg.add_to_parser(self.parser) def add_arguments(self): diff --git a/src/passa/cli/add.py b/src/passa/cli/add.py index 62f9e70..d5596cd 100644 --- a/src/passa/cli/add.py +++ b/src/passa/cli/add.py @@ -14,7 +14,7 @@ class Command(BaseCommand): arguments = [package_group] def run(self, options): - if not options.editable_lines and not options.requirement_lines: + if not options.editables and not options.packages: self.parser.error("Must supply either a requirement or --editable") return add_packages( packages=options.packages, diff --git a/src/passa/cli/init.py b/src/passa/cli/init.py new file mode 100644 index 0000000..95ce8d8 --- /dev/null +++ b/src/passa/cli/init.py @@ -0,0 +1,32 @@ +# -*- coding=utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + +import argparse +import os + +from ..actions.init import init_project +from ._base import BaseCommand +from .options import new_project_group + + +class Command(BaseCommand): + + name = "init" + description = "Create a new project." + default_arguments = [] + arguments = [new_project_group] + + def run(self, options): + pipfile_path = os.path.join(options.project, "Pipfile") + if os.path.exists(pipfile_path): + raise argparse.ArgumentError( + "{0!r} is already a Pipfile project".format(options.project), + ) + return init_project( + root=options.project, python_version=options.python_version + ) + + +if __name__ == "__main__": + Command.run_parser() diff --git a/src/passa/cli/options.py b/src/passa/cli/options.py index 5f454b3..da89a3b 100644 --- a/src/passa/cli/options.py +++ b/src/passa/cli/options.py @@ -3,6 +3,7 @@ import argparse import os +import sys import tomlkit.exceptions @@ -10,6 +11,9 @@ import vistir +PYTHON_VERSION = ".".join(str(v) for v in sys.version_info[:2]) + + class Project(passa.models.projects.Project): def __init__(self, root, *args, **kwargs): root = vistir.compat.Path(root).absolute() @@ -67,6 +71,17 @@ def add_to_parser(self, parser): help="path to project root (directory containing Pipfile)", ) +new_project = Option( + "--project", metavar="project", default=os.getcwd(), type=str, + help="path to project root (directory containing Pipfile)", +) + +python_version = Option( + "--py-version", "--python-version", "--requires-python", metavar="python-version", + dest="python_version", default=PYTHON_VERSION, type=str, + help="required minor python version for the project" +) + packages = Option( "packages", metavar="package", nargs="*", help="requirement to add (can be used multiple times)", @@ -135,3 +150,4 @@ def add_to_parser(self, parser): include_hashes_group = ArgumentGroup("include_hashes", is_mutually_exclusive=True, options=[include_hashes, no_include_hashes]) dev_group = ArgumentGroup("dev", is_mutually_exclusive="True", options=[dev_only, default_only]) package_group = ArgumentGroup("packages", options=[packages, editable, dev, no_sync]) +new_project_group = ArgumentGroup("new-project", options=[new_project, python_version]) From ec156a5b67b47955c79774cfdb68bab1947adb64 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 17 Sep 2018 01:16:01 -0400 Subject: [PATCH 07/32] Add initial tests Signed-off-by: Dan Ryan --- tests/actions/__init__.py | 0 tests/actions/test_add.py | 20 ++++++++++++++++++++ tests/actions/test_clean.py | 18 ++++++++++++++++++ tests/actions/test_freeze.py | 21 +++++++++++++++++++++ tests/actions/test_init.py | 19 +++++++++++++++++++ tests/actions/test_install.py | 0 tests/actions/test_lock.py | 0 tests/actions/test_remove.py | 0 tests/actions/test_sync.py | 0 tests/actions/test_upgrade.py | 0 tests/cli/__init__.py | 0 tests/cli/test_add.py | 1 + tests/cli/test_clean.py | 0 tests/cli/test_freeze.py | 0 tests/cli/test_init.py | 1 + tests/cli/test_install.py | 0 tests/cli/test_lock.py | 0 tests/cli/test_remove.py | 0 tests/cli/test_sync.py | 0 tests/cli/test_upgrade.py | 0 tests/conftest.py | 21 +++++++++++++++++++++ tests/integration/__init__.py | 0 22 files changed, 101 insertions(+) create mode 100644 tests/actions/__init__.py create mode 100644 tests/actions/test_add.py create mode 100644 tests/actions/test_clean.py create mode 100644 tests/actions/test_freeze.py create mode 100644 tests/actions/test_init.py create mode 100644 tests/actions/test_install.py create mode 100644 tests/actions/test_lock.py create mode 100644 tests/actions/test_remove.py create mode 100644 tests/actions/test_sync.py create mode 100644 tests/actions/test_upgrade.py create mode 100644 tests/cli/__init__.py create mode 100644 tests/cli/test_add.py create mode 100644 tests/cli/test_clean.py create mode 100644 tests/cli/test_freeze.py create mode 100644 tests/cli/test_init.py create mode 100644 tests/cli/test_install.py create mode 100644 tests/cli/test_lock.py create mode 100644 tests/cli/test_remove.py create mode 100644 tests/cli/test_sync.py create mode 100644 tests/cli/test_upgrade.py create mode 100644 tests/conftest.py create mode 100644 tests/integration/__init__.py diff --git a/tests/actions/__init__.py b/tests/actions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/actions/test_add.py b/tests/actions/test_add.py new file mode 100644 index 0000000..e7cd883 --- /dev/null +++ b/tests/actions/test_add.py @@ -0,0 +1,20 @@ +# -*- coding=utf-8 -*- +import passa.actions.init +import passa.actions.add +import passa.cli.options +import passa.models.projects + + +def test_add_one(project_directory): + project = passa.cli.options.Project(project_directory.strpath) + retcode = passa.actions.add.add_packages(["pytz"], project=project) + assert not retcode + assert 'pytz' in project.lockfile.default + + +def test_add_one_with_deps(project_directory): + project = passa.cli.options.Project(project_directory.strpath) + retcode = passa.actions.add.add_packages(["requests"], project=project) + assert not retcode + assert 'requests' in project.lockfile.default + assert 'idna' in project.lockfile.default diff --git a/tests/actions/test_clean.py b/tests/actions/test_clean.py new file mode 100644 index 0000000..3062ef5 --- /dev/null +++ b/tests/actions/test_clean.py @@ -0,0 +1,18 @@ +# -*- coding=utf-8 -*- +from passa.actions.add import add_packages +from passa.cli.options import Project +import vistir + + +def test_clean_subset(project_directory): + with vistir.contextmanagers.cd(project_directory.strpath): + project = Project(project_directory.strpath) + retcode = add_packages(["requests"], project=project) + assert not retcode + packages = ["requests", "chardet", "certifi", "idna"] + assert all(pkg in project.lockfile.default for pkg in packages) + import passa.actions.clean + pass + # clean_retcode = passa.actions.clean.clean(project=project) + # assert not clean_retcode + # assert project.lockfile.default._data == {} diff --git a/tests/actions/test_freeze.py b/tests/actions/test_freeze.py new file mode 100644 index 0000000..53d34f9 --- /dev/null +++ b/tests/actions/test_freeze.py @@ -0,0 +1,21 @@ +# -*- coding=utf-8 -*- +import passa.actions.add +import passa.actions.freeze +import passa.cli.options +import passa.models.projects + + +def test_freeze(project_directory): + project = passa.cli.options.Project(project_directory.strpath) + retcode = passa.actions.add.add_packages(["requests"], project=project) + assert not retcode + packages = ["requests", "chardet", "certifi", "idna"] + assert all(pkg in project.lockfile.default for pkg in packages) + freeze_file = project_directory.join("requirements.txt") + freeze_retcode = passa.actions.freeze.freeze( + project=project, include_hashes=False, target=freeze_file.strpath + ) + assert not freeze_retcode + lines = [line.strip() for line in freeze_file.readlines() if line.strip() != ''] + for pkg in packages: + assert any(line.startswith(pkg) for line in lines) diff --git a/tests/actions/test_init.py b/tests/actions/test_init.py new file mode 100644 index 0000000..2f12530 --- /dev/null +++ b/tests/actions/test_init.py @@ -0,0 +1,19 @@ +# -*- coding=utf-8 -*- + +import pytest + +import passa.actions.init +import passa.cli.options + + +def test_init(tmpdir): + init_retcode = passa.actions.init.init_project(root=tmpdir.strpath) + assert init_retcode == 0 + project = passa.cli.options.Project(tmpdir.strpath) + assert project.pipfile.packages._data == {} + assert project.pipfile.dev_packages._data == {} + + +def test_init_exists(project_directory): + with pytest.raises(RuntimeError, match=r'.* is already a Pipfile project'): + passa.actions.init.init_project(root=project_directory.strpath) diff --git a/tests/actions/test_install.py b/tests/actions/test_install.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/actions/test_lock.py b/tests/actions/test_lock.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/actions/test_remove.py b/tests/actions/test_remove.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/actions/test_sync.py b/tests/actions/test_sync.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/actions/test_upgrade.py b/tests/actions/test_upgrade.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/__init__.py b/tests/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/test_add.py b/tests/cli/test_add.py new file mode 100644 index 0000000..a6bcbda --- /dev/null +++ b/tests/cli/test_add.py @@ -0,0 +1 @@ +# -*- coding=utf-8 -*- diff --git a/tests/cli/test_clean.py b/tests/cli/test_clean.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/test_freeze.py b/tests/cli/test_freeze.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/test_init.py b/tests/cli/test_init.py new file mode 100644 index 0000000..a6bcbda --- /dev/null +++ b/tests/cli/test_init.py @@ -0,0 +1 @@ +# -*- coding=utf-8 -*- diff --git a/tests/cli/test_install.py b/tests/cli/test_install.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/test_lock.py b/tests/cli/test_lock.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/test_remove.py b/tests/cli/test_remove.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/test_sync.py b/tests/cli/test_sync.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/test_upgrade.py b/tests/cli/test_upgrade.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..577c942 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,21 @@ +# -*- coding=utf-8 -*- +import pytest + + +DEFAULT_PIPFILE_CONTENTS = """ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[packages] + +[dev-packages] +""".strip() + + +@pytest.fixture(scope="function") +def project_directory(tmpdir_factory): + project_dir = tmpdir_factory.mktemp("passa-project") + project_dir.join("Pipfile").write(DEFAULT_PIPFILE_CONTENTS) + return project_dir diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000..e69de29 From 4a52ed842617303976226c49c7ac668d9f9612ea Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 18 Sep 2018 05:23:23 -0400 Subject: [PATCH 08/32] Clean up options and add virtualenv module Signed-off-by: Dan Ryan --- src/passa/actions/add.py | 2 +- src/passa/actions/clean.py | 4 +- src/passa/actions/remove.py | 2 +- src/passa/cli/add.py | 7 +- src/passa/cli/clean.py | 9 +- src/passa/cli/options.py | 77 +++++++++-- src/passa/cli/remove.py | 6 +- src/passa/cli/sync.py | 4 +- src/passa/cli/upgrade.py | 4 +- src/passa/models/projects.py | 1 + src/passa/models/synchronizers.py | 85 ++++++++---- src/passa/models/virtualenv.py | 212 ++++++++++++++++++++++++++++++ 12 files changed, 364 insertions(+), 49 deletions(-) create mode 100644 src/passa/models/virtualenv.py diff --git a/src/passa/actions/add.py b/src/passa/actions/add.py index 6338466..3efc4dc 100644 --- a/src/passa/actions/add.py +++ b/src/passa/actions/add.py @@ -48,7 +48,7 @@ def add_packages(packages=[], editables=[], project=None, dev=False, sync=False, syncer = Synchronizer( project, default=default, develop=develop, - clean_unneeded=clean, + clean_unneeded=clean ) success = sync(syncer) if not success: diff --git a/src/passa/actions/clean.py b/src/passa/actions/clean.py index 3570e4d..8c6ae17 100644 --- a/src/passa/actions/clean.py +++ b/src/passa/actions/clean.py @@ -3,11 +3,11 @@ from __future__ import absolute_import, print_function, unicode_literals -def clean(project, dev=False): +def clean(project, default=True, dev=False, sync=True): from passa.models.synchronizers import Cleaner from passa.operations.sync import clean - cleaner = Cleaner(project, default=True, develop=dev) + cleaner = Cleaner(project, default=default, develop=dev, sync=sync) success = clean(cleaner) if not success: diff --git a/src/passa/actions/remove.py b/src/passa/actions/remove.py index 158f5e6..17ba1c7 100644 --- a/src/passa/actions/remove.py +++ b/src/passa/actions/remove.py @@ -3,7 +3,7 @@ from __future__ import absolute_import, print_function, unicode_literals -def remove(project=None, only="default", packages=[], clean=True): +def remove(project=None, only="default", packages=[], clean=True, sync=False): from passa.models.lockers import PinReuseLocker from passa.operations.lock import lock diff --git a/src/passa/cli/add.py b/src/passa/cli/add.py index d5596cd..2635b98 100644 --- a/src/passa/cli/add.py +++ b/src/passa/cli/add.py @@ -4,14 +4,14 @@ from ..actions.add import add_packages from ._base import BaseCommand -from .options import package_group +from .options import package_group, clean_group class Command(BaseCommand): name = "add" description = "Add packages to project." - arguments = [package_group] + arguments = [package_group, clean_group] def run(self, options): if not options.editables and not options.packages: @@ -20,7 +20,8 @@ def run(self, options): packages=options.packages, editables=options.editables, project=options.project, - dev=options.dev + dev=options.dev, + clean=options.clean ) diff --git a/src/passa/cli/clean.py b/src/passa/cli/clean.py index e23d5ee..a74d814 100644 --- a/src/passa/cli/clean.py +++ b/src/passa/cli/clean.py @@ -4,17 +4,20 @@ from ..actions.clean import clean from ._base import BaseCommand -from .options import dev, no_default +from .options import dev, no_default, sync_group class Command(BaseCommand): name = "clean" description = "Uninstall unlisted packages from the environment." - arguments = [dev, no_default] + arguments = [dev, no_default, sync_group] def run(self, options): - return clean(project=options.project, default=options.default, dev=options.dev) + return clean( + project=options.project, default=options.default, dev=options.dev, + sync=options.sync + ) if __name__ == "__main__": diff --git a/src/passa/cli/options.py b/src/passa/cli/options.py index da89a3b..07782db 100644 --- a/src/passa/cli/options.py +++ b/src/passa/cli/options.py @@ -2,12 +2,15 @@ from __future__ import absolute_import import argparse +import inspect import os import sys +import six import tomlkit.exceptions import passa.models.projects +import passa.models.virtualenv import vistir @@ -20,23 +23,67 @@ def __init__(self, root, *args, **kwargs): pipfile = root.joinpath("Pipfile") if not pipfile.is_file(): raise argparse.ArgumentError( - "{0!r} is not a Pipfile project".format(root), + project, "{0!r} is not a Pipfile project".format(root.as_posix()), ) + self.venv = self.get_venv(root) try: - super(Project, self).__init__(root.as_posix(), *args, **kwargs) + super(Project, self).__init__(root.as_posix(), env_prefix=self.venv.venv_dir, + *args, **kwargs) except tomlkit.exceptions.ParseError as e: raise argparse.ArgumentError( - "failed to parse Pipfile: {0!r}".format(str(e)), + project, "failed to parse Pipfile: {0!r}".format(str(e)), ) + def get_venv(self, root): + if 'VIRTUAL_ENV' in os.environ: + return passa.models.virtualenv.VirtualEnv(os.environ['VIRTUAL_ENV']) + return passa.models.virtualenv.VirtualEnv.from_project_path(root) + def __name__(self): return "Project Root" +class OptionMeta(type): + + @property + def action_map(self): + action_map = getattr(self, '_action_map', None) + if not action_map: + self.action_map = { + name.strip("_").replace("Action", ""): obj + for name, obj in inspect.getmembers(argparse) + if name.startswith('_') and name.endswith('Action') + } + return self._action_map + + @action_map.setter + def action_map(self, action_map): + self._action_map = action_map + + +@six.add_metaclass(OptionMeta) class Option(object): def __init__(self, *args, **kwargs): - self.args = args - self.kwargs = kwargs + self.args = list(args) + self.kwargs = kwargs.copy() + if "dest" not in kwargs and not args[0].startswith("-"): + dest = list(args).pop(0) + else: + dest = kwargs.pop("dest", args[0].lstrip("-").replace("-", "_")) + action = kwargs.pop("action", None) + if not action: + if 'const' in kwargs: + action = 'store_const' + else: + action = 'store' + self.action = self.get_option(action, args, dest, **kwargs) + + @classmethod + def get_option(cls, action, option_strings, dest, *args, **kwargs): + if action: + action = action.title().replace("_", "") + return cls.action_map[action](list(option_strings), dest, *args, **kwargs) + return def add_to_parser(self, parser): parser.add_argument(*self.args, **self.kwargs) @@ -46,7 +93,7 @@ def add_to_group(self, group): class ArgumentGroup(object): - def __init__(self, name, parser=None, is_mutually_exclusive=False, required=None, options=[]): + def __init__(self, name, parser=None, is_mutually_exclusive=False, required=False, options=[]): self.name = name self.options = options self.parser = parser @@ -65,6 +112,9 @@ def add_to_parser(self, parser): self.argument_group = group self.parser = parser + def add_to_group(self, group): + self.add_to_parser(group) + project = Option( "--project", metavar="project", default=os.getcwd(), type=Project, @@ -77,7 +127,7 @@ def add_to_parser(self, parser): ) python_version = Option( - "--py-version", "--python-version", "--requires-python", metavar="python-version", + "--py-version", "--python-version", "--requires-python", metavar="python_version", dest="python_version", default=PYTHON_VERSION, type=str, help="required minor python version for the project" ) @@ -102,6 +152,11 @@ def add_to_parser(self, parser): help="do not synchronize the environment", ) +sync = Option( + "--sync", dest="sync", action="store_true", help="synchronize the environment", + default=False +) + target = Option( "-t", "--target", default=None, help="file to export into (default is to print to stdout)" @@ -132,6 +187,10 @@ def add_to_parser(self, parser): help="do not remove packages not specified in Pipfile.lock", ) +clean = Option( + "--clean", dest="clean", action="store_true", default=False, + help="remove packages not specified in Pipfile.lock", +) dev_only = Option( "--dev", dest="only", action="store_const", const="dev", help="only try to modify [dev-packages]", @@ -149,5 +208,7 @@ def add_to_parser(self, parser): include_hashes_group = ArgumentGroup("include_hashes", is_mutually_exclusive=True, options=[include_hashes, no_include_hashes]) dev_group = ArgumentGroup("dev", is_mutually_exclusive="True", options=[dev_only, default_only]) -package_group = ArgumentGroup("packages", options=[packages, editable, dev, no_sync]) new_project_group = ArgumentGroup("new-project", options=[new_project, python_version]) +clean_group = ArgumentGroup("clean", is_mutually_exclusive=True, options=[clean, no_clean]) +sync_group = ArgumentGroup("sync", is_mutually_exclusive=True, options=[sync, no_sync]) +package_group = ArgumentGroup("packages", options=[packages, editable, dev, sync_group]) diff --git a/src/passa/cli/remove.py b/src/passa/cli/remove.py index 538acbf..041c195 100644 --- a/src/passa/cli/remove.py +++ b/src/passa/cli/remove.py @@ -4,18 +4,18 @@ from ..actions.remove import remove from ._base import BaseCommand -from .options import dev_group, no_clean, packages +from .options import dev_group, clean_group, sync_group, packages class Command(BaseCommand): name = "remove" description = "Remove packages from project." - arguments = [dev_group, no_clean, packages] + arguments = [dev_group, clean_group, sync_group, packages] def run(self, options): return remove(project=options.project, only=options.only, - packages=options.packages, clean=options.clean) + packages=options.packages, clean=options.clean, sync=options.sync) if __name__ == "__main__": diff --git a/src/passa/cli/sync.py b/src/passa/cli/sync.py index a09b784..9a31fe1 100644 --- a/src/passa/cli/sync.py +++ b/src/passa/cli/sync.py @@ -4,14 +4,14 @@ from ..actions.sync import sync from ._base import BaseCommand -from .options import dev, no_clean +from .options import dev, clean_group class Command(BaseCommand): name = "sync" description = "Install Pipfile.lock into the environment." - arguments = [dev, no_clean] + arguments = [dev, clean_group] def run(self, options): return sync(project=options.project, dev=options.dev, clean=options.clean) diff --git a/src/passa/cli/upgrade.py b/src/passa/cli/upgrade.py index cf7f502..c7696c2 100644 --- a/src/passa/cli/upgrade.py +++ b/src/passa/cli/upgrade.py @@ -3,14 +3,14 @@ from ..actions.upgrade import upgrade from ._base import BaseCommand -from .options import no_clean, no_sync, packages, strategy +from .options import clean_group, sync_group, packages, strategy class Command(BaseCommand): name = "upgrade" description = "Upgrade packages in project." - arguments = [packages, strategy, no_clean, no_sync] + arguments = [packages, strategy, clean_group, sync_group] def run(self, options): return upgrade(project=options.project, strategy=options.strategy, diff --git a/src/passa/models/projects.py b/src/passa/models/projects.py index f6e037d..7ff6f31 100644 --- a/src/passa/models/projects.py +++ b/src/passa/models/projects.py @@ -84,6 +84,7 @@ def dumps(self): class Project(object): root = attr.ib() + env_prefix = attr.ib(default=None) _p = attr.ib(init=False) _l = attr.ib(init=False) diff --git a/src/passa/models/synchronizers.py b/src/passa/models/synchronizers.py index bad4905..8fc3e0a 100644 --- a/src/passa/models/synchronizers.py +++ b/src/passa/models/synchronizers.py @@ -14,15 +14,19 @@ import packaging.version import requirementslib +from .virtualenv import VirtualEnv + from ..internals._pip import uninstall, EditableInstaller, WheelInstaller -def _is_installation_local(name): +def _is_installation_local(name, venv=None): """Check whether the distribution is in the current Python installation. This is used to distinguish packages seen by a virtual environment. A venv may be able to see global packages, but we don't want to mess with them. """ + if venv: + return venv.is_installed(name) loc = os.path.normcase(pkg_resources.working_set.by_key[name].location) pre = os.path.normcase(sys.prefix) return os.path.commonprefix([loc, pre]) == pre @@ -38,12 +42,14 @@ def _is_up_to_date(distro, version): ]) -def _group_installed_names(packages): +def _group_installed_names(packages, venv=None): """Group locally installed packages based on given specifications. `packages` is a name-package mapping that are used as baseline to determine how the installed package should be grouped. + `venv` is the virtual environment object of the virtualenv being installed into. + Returns a 3-tuple of disjoint sets, all containing names of installed packages: @@ -54,8 +60,13 @@ def _group_installed_names(packages): """ groupcoll = GroupCollection(set(), set(), set(), set()) - for distro in pkg_resources.working_set: - name = distro.key + if venv: + working_set = venv.get_working_set() + else: + working_set = pkg_resources.working_set + + for dist in working_set: + name = dist.key try: package = packages[name] except KeyError: @@ -66,7 +77,7 @@ def _group_installed_names(packages): if not r.is_named: # Always mark non-named. I think pip does something similar? groupcoll.outdated.add(name) - elif not _is_up_to_date(distro, r.get_version()): + elif not _is_up_to_date(dist, r.get_version()): groupcoll.outdated.add(name) else: groupcoll.uptodate.add(name) @@ -75,11 +86,14 @@ def _group_installed_names(packages): @contextlib.contextmanager -def _remove_package(name): - if name is None or not _is_installation_local(name): +def _remove_package(name, venv=None): + if name is None or not _is_installation_local(name, venv=venv): yield None return - with uninstall(name, auto_confirm=True, verbose=False) as uninstaller: + _uninstall = uninstall + if venv: + _uninstall = venv.uninstall + with _uninstall(name, auto_confirm=True, verbose=False) as uninstaller: yield uninstaller @@ -88,19 +102,22 @@ def _get_packages(lockfile, default, develop): # Extras don't matter because they only affect dependencies, and we # don't install dependencies anyway! packages = {} - if default: - packages.update(lockfile.default._data) if develop: packages.update(lockfile.develop._data) + if default: + packages.update(lockfile.default._data) return packages -def _build_paths(): +def _build_paths(venv=None): """Prepare paths for distlib.wheel.Wheel to install into. """ - paths = sysconfig.get_paths() + if venv: + paths = venv.paths + else: + paths = sysconfig.get_paths() return { - "prefix": sys.prefix, + "prefix": sys.prefix if not venv else venv.venv_dir.as_posix(), "data": paths["data"], "scripts": paths["scripts"], "headers": paths["include"], @@ -112,12 +129,12 @@ def _build_paths(): PROTECTED_FROM_CLEAN = {"setuptools", "pip", "wheel"} -def _clean(names): +def _clean(names, venv=None): cleaned = set() for name in names: if name in PROTECTED_FROM_CLEAN: continue - with _remove_package(name) as uninst: + with _remove_package(name, venv=venv) as uninst: if uninst: cleaned.add(name) return cleaned @@ -126,18 +143,34 @@ def _clean(names): class Synchronizer(object): """Helper class to install packages from a project's lock file. """ - def __init__(self, project, default, develop, clean_unneeded): + def __init__(self, project, default, develop, clean_unneeded, venv=None): self._root = project.root # Only for repr. self.packages = _get_packages(project.lockfile, default, develop) self.sources = project.lockfile.meta.sources._data - self.paths = _build_paths() self.clean_unneeded = clean_unneeded + if not venv: + self._venv = getattr(project, "venv", None) + else: + self._venv = venv + self.paths = _build_paths(venv=self.venv) + + @property + def venv(self): + if self._venv: + return self._venv + return self.project.venv def __repr__(self): return "<{0} @ {1!r}>".format(type(self).__name__, self._root) def sync(self): - groupcoll = _group_installed_names(self.packages) + if not self.venv: + return self._sync() + with self.venv.activated(): + return self._sync() + + def _sync(self): + groupcoll = _group_installed_names(self.packages, venv=self.venv) installed = set() updated = set() @@ -146,7 +179,7 @@ def sync(self): # TODO: Show a prompt to confirm cleaning. We will need to implement a # reporter pattern for this as well. if self.clean_unneeded: - names = _clean(groupcoll.unneeded) + names = _clean(groupcoll.unneeded, venv=self.venv) cleaned.update(names) # TODO: Specify installation order? (pypa/pipenv#2274) @@ -161,7 +194,7 @@ def sync(self): continue r.markers = None if r.editable: - installer = EditableInstaller(r) + installer = EditableInstaller(r, venv=self.venv) else: installer = WheelInstaller(r, self.sources, self.paths) try: @@ -181,7 +214,7 @@ def sync(self): else: name_to_remove = None try: - with _remove_package(name_to_remove): + with _remove_package(name_to_remove, venv=self.venv): installer.install() except Exception as e: if os.environ.get("PASSA_NO_SUPPRESS_EXCEPTIONS"): @@ -201,14 +234,18 @@ def sync(self): class Cleaner(object): """Helper class to clean packages not in a project's lock file. """ - def __init__(self, project, default, develop): + def __init__(self, project, default, develop, sync=True): self._root = project.root # Only for repr. self.packages = _get_packages(project.lockfile, default, develop) + self.sync = sync + self.project = project def __repr__(self): return "<{0} @ {1!r}>".format(type(self).__name__, self._root) def clean(self): - groupcoll = _group_installed_names(self.packages) - cleaned = _clean(groupcoll.unneeded) + groupcoll = _group_installed_names(self.packages, venv=self.project.venv) + cleaned = set() + if self.sync: + cleaned = _clean(groupcoll.unneeded, venv=self.project.venv) return cleaned diff --git a/src/passa/models/virtualenv.py b/src/passa/models/virtualenv.py new file mode 100644 index 0000000..78484c2 --- /dev/null +++ b/src/passa/models/virtualenv.py @@ -0,0 +1,212 @@ +# -*- coding=utf-8 -*- + +import base64 +import contextlib +import distlib.scripts +import hashlib +import importlib +import json +import os +import re +import sys +import sysconfig + +from cached_property import cached_property + +import vistir + +from ..internals._pip import RequirementUninstaller + + +class VirtualEnv(object): + def __init__(self, venv_dir): + self.venv_dir = vistir.compat.Path(venv_dir) + + @classmethod + def from_project_path(cls, path): + path = vistir.compat.Path(path) + if path.name == 'Pipfile': + pipfile_path = path + path = path.parent + else: + pipfile_path = path / 'Pipfile' + pipfile_location = cls.normalize_path(pipfile_path) + venv_path = path / '.venv' + if venv_path.exists(): + if not venv_path.is_dir(): + possible_path = vistir.compat.Path(venv_path.read_text().strip()) + if possible_path.exists(): + return cls(possible_path.as_posix()) + else: + if venv_path.joinpath('lib').exists(): + return cls(venv_path.as_posix()) + sanitized = re.sub(r'[ $`!*@"\\\r\n\t]', "_", path.name)[0:42] + hash_ = hashlib.sha256(pipfile_location.encode()).digest()[:6] + encoded_hash = base64.urlsafe_b64encode(hash_).decode() + hash_fragment = encoded_hash[:8] + venv_name = "{0}-{1}".format(sanitized, hash_fragment) + return cls(cls.get_workon_home().joinpath(venv_name).as_posix()) + + @classmethod + def normalize_path(cls, path): + if not path: + return + if not path.is_absolute(): + try: + path = path.resolve() + except OSError: + path = path.absolute() + path = vistir.path.unicode_path("{0}".format(path)) + if os.name != "nt": + return path + + drive, tail = os.path.splitdrive(path) + # Only match (lower cased) local drives (e.g. 'c:'), not UNC mounts. + if drive.islower() and len(drive) == 2 and drive[1] == ":": + path = "{}{}".format(drive.upper(), tail) + + return vistir.path.unicode_path(path) + + @classmethod + def get_workon_home(cls): + workon_home = os.environ.get("WORKON_HOME") + if not workon_home: + if os.name == "nt": + workon_home = "~/.virtualenvs" + else: + workon_home = os.path.join( + os.environ.get("XDG_DATA_HOME", "~/.local/share"), "virtualenvs" + ) + return vistir.compat.Path(os.path.expandvars(workon_home)).expanduser() + + @cached_property + def script_basedir(self): + script_dir = os.path.basename(sysconfig.get_paths()["scripts"]) + return script_dir + + @property + def python(self): + return self.venv_dir.joinpath(self.script_basedir).joinpath("python").as_posix() + + @cached_property + def sys_path(self): + c = vistir.misc.run([self.python, "-c", "import json,sys; print(json.dumps(sys.path))"], + return_object=True, nospin=True) + assert c.returncode == 0, "failed loading virtualenv path" + path = json.loads(c.out.strip()) + return path + + @cached_property + def paths(self): + paths = {} + with vistir.contextmanagers.temp_environ(), vistir.contextmanagers.temp_path(): + os.environ["PYTHONUSERBASE"] = vistir.compat.fs_str(self.venv_dir.as_posix()) + os.environ["PYTHONIOENCODING"] = vistir.compat.fs_str("utf-8") + os.environ["PYTHONDONTWRITEBYTECODE"] = vistir.compat.fs_str("1") + importlib.reload(sysconfig) + scheme, _, _ = sysconfig._get_default_scheme().partition('_') + scheme = "{0}_user".format(scheme) + paths = sysconfig.get_paths(scheme=scheme) + return paths + + @property + def scripts_dir(self): + return self.paths["scripts"] + + @cached_property + def passa_entry(self): + import pkg_resources + return pkg_resources.working_set.by_key['passa'].location + + def get_distributions(self): + import pkg_resources + importlib.reload(pkg_resources) + return pkg_resources.find_distributions(self.paths["purelib"], only=True) + + def get_working_set(self): + working_set = None + import pkg_resources + passa_entry = self.passa_entry + with self.activated(): + working_set = pkg_resources.WorkingSet(self.sys_path + [passa_entry,]) + return working_set + + @classmethod + def filter_sources(cls, requirement, sources): + if not sources or not requirement.index: + return sources + filtered_sources = [ + source for source in sources + if source.get("name") == requirement.index + ] + return filtered_sources or sources + + def install(self, req, editable=False, sources=[]): + with self.activated(): + import passa.internals._pip_shims + importlib.reload(passa.internals._pip_shms) + ireq = req.as_ireq() + if editable: + with vistir.contextmanagers.cd(ireq.setup_py_dir): + c = self.run([self.python, "setup.py", "develop", "--no-deps"], cwd=ireq.setup_py_dir) + return c.returncode + importlib.reload(distlib.scripts) + sources = self.filter_sources(req, sources) + hashes = req.hashes + wheel = passa.internals._pip_shims.build_wheel(ireq, sources, hashes) + wheel.install(self.paths, distlib.scripts.ScriptMaker(None, None)) + + @contextlib.contextmanager + def activated(self): + original_path = sys.path + original_prefix = sys.prefix + original_user_base = os.environ.get("PYTHONUSERBASE", None) + original_venv = os.environ.get("VIRTUAL_ENV", None) + passa_path = vistir.compat.Path(__file__).absolute().parent.parent.as_posix() + with vistir.contextmanagers.temp_environ(), vistir.contextmanagers.temp_path(): + os.environ["PYTHONUSERBASE"] = vistir.compat.fs_str(self.venv_dir.as_posix()) + os.environ["PYTHONIOENCODING"] = vistir.compat.fs_str("utf-8") + os.environ["PYTHONDONTWRITEBYTECODE"] = vistir.compat.fs_str("1") + os.environ["VIRTUAL_ENV"] = vistir.compat.fs_str(self.venv_dir.as_posix()) + sys.path = self.sys_path + sys.prefix = self.venv_dir + sys.path.append(passa_path) + activate_this = os.path.join(self.scripts_dir, "activate_this.py") + with open(activate_this, "r") as f: + code = compile(f.read(), activate_this, "exec") + exec(code, dict(__file__=activate_this)) + try: + yield + finally: + print("Deactivating virtualenv...") + del os.environ["VIRTUAL_ENV"] + del os.environ["PYTHONUSERBASE"] + if original_user_base: + os.environ["PYTHONUSERBASE"] = original_user_base + if original_venv: + os.environ["VIRTUAL_ENV"] = original_venv + sys.path = original_path + sys.prefix = original_prefix + + def run(self, cmd, cwd=os.curdir): + c = None + with self.activated(): + script = vistir.cmdparse.Script.parse(cmd) + c = vistir.misc.run(script._parts, return_object=True, nospin=True, cwd=cwd) + return c + + def is_installed(self, pkgname): + return any(d for d in self.get_distributions() if d.project_name == pkgname) + + def uninstall(self, pkgname, *args, **kwargs): + with self.activated(): + from pip_shims.shims import InstallRequirement + module_name = InstallRequirement.__module__ + del InstallRequirement + if module_name not in sys.modules: + pip_req_install = importlib.import_module(module_name) + else: + pip_req_install = sys.modules[module_name] + importlib.reload(sys.modules[module_name]) + ireq = pip_req_install.InstallRequirement.from_line(pkgname) + return RequirementUninstaller(ireq, *args, **kwargs) From ad79981b26516596cdb15ef2f2e8cefd2f7035ef Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 18 Sep 2018 05:23:46 -0400 Subject: [PATCH 09/32] Add tests (clean is still failing) Signed-off-by: Dan Ryan --- tests/actions/test_clean.py | 30 ++++++++++++++--------------- tests/conftest.py | 38 ++++++++++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 16 deletions(-) diff --git a/tests/actions/test_clean.py b/tests/actions/test_clean.py index 3062ef5..95b0b14 100644 --- a/tests/actions/test_clean.py +++ b/tests/actions/test_clean.py @@ -1,18 +1,18 @@ # -*- coding=utf-8 -*- -from passa.actions.add import add_packages -from passa.cli.options import Project -import vistir -def test_clean_subset(project_directory): - with vistir.contextmanagers.cd(project_directory.strpath): - project = Project(project_directory.strpath) - retcode = add_packages(["requests"], project=project) - assert not retcode - packages = ["requests", "chardet", "certifi", "idna"] - assert all(pkg in project.lockfile.default for pkg in packages) - import passa.actions.clean - pass - # clean_retcode = passa.actions.clean.clean(project=project) - # assert not clean_retcode - # assert project.lockfile.default._data == {} +def test_clean_subset(project): + from passa.actions.add import add_packages + from passa.actions.clean import clean + retcode = add_packages(["requests"], project=project) + assert not retcode + packages = ["requests", "chardet", "certifi", "idna"] + c = project.venv.run("pip install pytz") + assert c.returncode == 0 + c = project.venv.run("python -c 'import pytz'") + assert c.returncode == 0 + clean_retcode = clean(project=project) + assert not clean_retcode + c = project.venv.run("python -c 'import pytz'") + assert c.returncode != 0 + assert all(pkg in project.lockfile.default for pkg in packages) diff --git a/tests/conftest.py b/tests/conftest.py index 577c942..e1326ca 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,10 @@ # -*- coding=utf-8 -*- +import os import pytest +import passa.cli.options +import passa.models.virtualenv +import sys +import vistir DEFAULT_PIPFILE_CONTENTS = """ @@ -18,4 +23,35 @@ def project_directory(tmpdir_factory): project_dir = tmpdir_factory.mktemp("passa-project") project_dir.join("Pipfile").write(DEFAULT_PIPFILE_CONTENTS) - return project_dir + with vistir.contextmanagers.cd(project_dir.strpath): + yield project_dir + + +@pytest.fixture(scope="function") +def virtualenv(tmpdir_factory): + venv_dir = tmpdir_factory.mktemp("passa-testenv") + print("Creating virtualenv {0!r}".format(venv_dir.strpath)) + c = vistir.misc.run([sys.executable, "-m", "virtualenv", venv_dir.strpath], + return_object=True, block=True, nospin=True) + if c.returncode == 0: + print("Virtualenv created...") + return venv_dir + raise RuntimeError("Failed creating virtualenv for testing...{0!r}".format(c.err.strip())) + + +class _Project(passa.cli.options.Project): + def __init__(self, root, venv=None): + self.path = os.path.abspath(root) + self.venv = venv + super(_Project, self).__init__(self.path) + + +@pytest.fixture +def tmpvenv(virtualenv): + return passa.models.virtualenv.VirtualEnv(virtualenv) + + +@pytest.fixture(scope="function") +def project(project_directory, tmpvenv): + with tmpvenv.activated(): + yield _Project(project_directory, tmpvenv) From ab37932079b16a0e455c2c151e11683515a60310 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 18 Sep 2018 18:32:56 -0400 Subject: [PATCH 10/32] Wheel installation into virtualenv works Signed-off-by: Dan Ryan --- src/passa/models/virtualenv.py | 69 +++++++++++++++++++++++++++++++--- 1 file changed, 64 insertions(+), 5 deletions(-) diff --git a/src/passa/models/virtualenv.py b/src/passa/models/virtualenv.py index 78484c2..54d5682 100644 --- a/src/passa/models/virtualenv.py +++ b/src/passa/models/virtualenv.py @@ -8,6 +8,7 @@ import json import os import re +import six import sys import sysconfig @@ -96,6 +97,19 @@ def sys_path(self): path = json.loads(c.out.strip()) return path + @cached_property + def system_paths(self): + paths = {} + importlib.reload(sysconfig) + paths = sysconfig.get_paths() + return paths + + @cached_property + def sys_prefix(self): + c = self.run_py(["-c", "'import sys; print(sys.prefix)'"]) + sys_prefix = vistir.misc.to_text(c.out).strip() + return sys_prefix + @cached_property def paths(self): paths = {} @@ -141,19 +155,45 @@ def filter_sources(cls, requirement, sources): ] return filtered_sources or sources + @cached_property + def python_version(self): + with self.activated(): + importlib.reload(sysconfig) + py_version = sysconfig.get_python_version() + return py_version + + def get_setup_install_args(self, pkgname, setup_py, develop=False): + headers = vistir.compat.Path(self.sys_prefix) / "include" / "site" + headers = headers / "python{0}".format(self.python_version) / pkgname + install_arg = "install" if not develop else "develop" + return [ + self.python, "-u", "-c", SETUPTOOLS_SHIM % setup_py, install_arg, + "--single-version-externally-managed", "root={0}".format(), + "--install-headers={0}".format(headers.as_posix()), + "--install-purelib={0}".format(self.paths["purelib"]), + "--install-platlib={0}".format(self.paths["platlib"]), + "--install-scripts={0}".format(self.scripts_dir), + "--install-data={0}".format(self.paths["data"]), + ] + def install(self, req, editable=False, sources=[]): with self.activated(): - import passa.internals._pip_shims - importlib.reload(passa.internals._pip_shms) + import passa.internals._pip + install_options = ["--prefix={0}".format(self.venv_dir),] + importlib.reload(passa.internals._pip) ireq = req.as_ireq() if editable: - with vistir.contextmanagers.cd(ireq.setup_py_dir): - c = self.run([self.python, "setup.py", "develop", "--no-deps"], cwd=ireq.setup_py_dir) + with vistir.contextmanagers.cd(ireq.setup_py_dir, ireq.setup_py): + c = self.run( + install_options + self.get_setup_install_args( + req.name, develop=editable + ), cwd=ireq.setup_py_dir + ) return c.returncode importlib.reload(distlib.scripts) sources = self.filter_sources(req, sources) hashes = req.hashes - wheel = passa.internals._pip_shims.build_wheel(ireq, sources, hashes) + wheel = passa.internals._pip.build_wheel(ireq, sources, hashes) wheel.install(self.paths, distlib.scripts.ScriptMaker(None, None)) @contextlib.contextmanager @@ -195,6 +235,16 @@ def run(self, cmd, cwd=os.curdir): c = vistir.misc.run(script._parts, return_object=True, nospin=True, cwd=cwd) return c + def run_py(self, cmd, cwd=os.curdir): + c = None + if isinstance(cmd, six.string_types): + script = vistir.cmdparse.Script.parse("{0} {1}".format(self.python, cmd)) + else: + script = vistir.cmdparse.Script.parse([self.python,] + list(cmd)) + with self.activated(): + c = vistir.misc.run(script._parts, return_object=True, nospin=True, cwd=cwd) + return c + def is_installed(self, pkgname): return any(d for d in self.get_distributions() if d.project_name == pkgname) @@ -210,3 +260,12 @@ def uninstall(self, pkgname, *args, **kwargs): importlib.reload(sys.modules[module_name]) ireq = pip_req_install.InstallRequirement.from_line(pkgname) return RequirementUninstaller(ireq, *args, **kwargs) + + +SETUPTOOLS_SHIM = ( + "import setuptools, tokenize;__file__=%r;" + "f=getattr(tokenize, 'open', open)(__file__);" + "code=f.read().replace('\\r\\n', '\\n');" + "f.close();" + "exec(compile(code, __file__, 'exec'))" +) From 2c7b23513a7a90622f56aab391128efb05554938 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 18 Sep 2018 18:33:15 -0400 Subject: [PATCH 11/32] Update setup.cfg Signed-off-by: Dan Ryan --- setup.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.cfg b/setup.cfg index 98ea198..4daf9ee 100644 --- a/setup.cfg +++ b/setup.cfg @@ -45,6 +45,7 @@ install_requires = resolvelib>=0.2.1,!=1.0.0.dev0 requirementslib>=1.1.1 six + virtualenv vistir[spinner]>=0.1.4 [options.extras_require] @@ -52,6 +53,7 @@ pack = invoke parver tests = + cached-property pytest-xdist pytest-timeout pytest-cov From 9239d7bbbcd3afe37aed5c84b07f3a5385ec44a2 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 19 Sep 2018 01:19:33 -0400 Subject: [PATCH 12/32] Implement working versions of clean and install inside virtualenvs Signed-off-by: Dan Ryan --- src/passa/models/synchronizers.py | 2 +- src/passa/models/virtualenv.py | 91 ++++++++++++++++++++++++------- 2 files changed, 71 insertions(+), 22 deletions(-) diff --git a/src/passa/models/synchronizers.py b/src/passa/models/synchronizers.py index 8fc3e0a..33fcfc3 100644 --- a/src/passa/models/synchronizers.py +++ b/src/passa/models/synchronizers.py @@ -135,7 +135,7 @@ def _clean(names, venv=None): if name in PROTECTED_FROM_CLEAN: continue with _remove_package(name, venv=venv) as uninst: - if uninst: + if uninst.paths: cleaned.add(name) return cleaned diff --git a/src/passa/models/virtualenv.py b/src/passa/models/virtualenv.py index 54d5682..c179e3c 100644 --- a/src/passa/models/virtualenv.py +++ b/src/passa/models/virtualenv.py @@ -12,12 +12,11 @@ import sys import sysconfig +import passa.internals._pip from cached_property import cached_property import vistir -from ..internals._pip import RequirementUninstaller - class VirtualEnv(object): def __init__(self, venv_dir): @@ -100,7 +99,7 @@ def sys_path(self): @cached_property def system_paths(self): paths = {} - importlib.reload(sysconfig) + six.moves.reload_module(sysconfig) paths = sysconfig.get_paths() return paths @@ -117,7 +116,7 @@ def paths(self): os.environ["PYTHONUSERBASE"] = vistir.compat.fs_str(self.venv_dir.as_posix()) os.environ["PYTHONIOENCODING"] = vistir.compat.fs_str("utf-8") os.environ["PYTHONDONTWRITEBYTECODE"] = vistir.compat.fs_str("1") - importlib.reload(sysconfig) + six.moves.reload_module(sysconfig) scheme, _, _ = sysconfig._get_default_scheme().partition('_') scheme = "{0}_user".format(scheme) paths = sysconfig.get_paths(scheme=scheme) @@ -134,15 +133,15 @@ def passa_entry(self): def get_distributions(self): import pkg_resources - importlib.reload(pkg_resources) + six.moves.reload_module(pkg_resources) return pkg_resources.find_distributions(self.paths["purelib"], only=True) def get_working_set(self): working_set = None import pkg_resources passa_entry = self.passa_entry - with self.activated(): - working_set = pkg_resources.WorkingSet(self.sys_path + [passa_entry,]) + monkeypatch = self.monkeypatch_dist + working_set = pkg_resources.WorkingSet(self.sys_path + [passa_entry, monkeypatch]) return working_set @classmethod @@ -158,10 +157,26 @@ def filter_sources(cls, requirement, sources): @cached_property def python_version(self): with self.activated(): - importlib.reload(sysconfig) + six.moves.reload_module(sysconfig) py_version = sysconfig.get_python_version() return py_version + @classmethod + def safe_import(cls, name): + module = None + if name not in sys.modules: + module = importlib.import_module(name) + else: + module = sys.modules[name] + six.moves.reload_module(module) + return module + + @cached_property + def monkeypatch_dist(self): + pkg_resources = self.safe_import("pkg_resources") + monkey_patch = pkg_resources.get_distribution('recursive-monkey-patch').location + return monkey_patch + def get_setup_install_args(self, pkgname, setup_py, develop=False): headers = vistir.compat.Path(self.sys_prefix) / "include" / "site" headers = headers / "python{0}".format(self.python_version) / pkgname @@ -178,9 +193,8 @@ def get_setup_install_args(self, pkgname, setup_py, develop=False): def install(self, req, editable=False, sources=[]): with self.activated(): - import passa.internals._pip install_options = ["--prefix={0}".format(self.venv_dir),] - importlib.reload(passa.internals._pip) + six.moves.reload_module(passa.internals._pip) ireq = req.as_ireq() if editable: with vistir.contextmanagers.cd(ireq.setup_py_dir, ireq.setup_py): @@ -190,7 +204,7 @@ def install(self, req, editable=False, sources=[]): ), cwd=ireq.setup_py_dir ) return c.returncode - importlib.reload(distlib.scripts) + six.moves.reload_module(distlib.scripts) sources = self.filter_sources(req, sources) hashes = req.hashes wheel = passa.internals._pip.build_wheel(ireq, sources, hashes) @@ -203,6 +217,7 @@ def activated(self): original_user_base = os.environ.get("PYTHONUSERBASE", None) original_venv = os.environ.get("VIRTUAL_ENV", None) passa_path = vistir.compat.Path(__file__).absolute().parent.parent.as_posix() + monkeypatch_dist = self.monkeypatch_dist with vistir.contextmanagers.temp_environ(), vistir.contextmanagers.temp_path(): os.environ["PYTHONUSERBASE"] = vistir.compat.fs_str(self.venv_dir.as_posix()) os.environ["PYTHONIOENCODING"] = vistir.compat.fs_str("utf-8") @@ -210,11 +225,15 @@ def activated(self): os.environ["VIRTUAL_ENV"] = vistir.compat.fs_str(self.venv_dir.as_posix()) sys.path = self.sys_path sys.prefix = self.venv_dir - sys.path.append(passa_path) + import site + # sys.path.append(passa_path) activate_this = os.path.join(self.scripts_dir, "activate_this.py") with open(activate_this, "r") as f: code = compile(f.read(), activate_this, "exec") exec(code, dict(__file__=activate_this)) + site.addsitedir(passa_path) + site.addsitedir(monkeypatch_dist) + pkg_resources = self.safe_import("pkg_resources") try: yield finally: @@ -227,6 +246,7 @@ def activated(self): os.environ["VIRTUAL_ENV"] = original_venv sys.path = original_path sys.prefix = original_prefix + six.moves.reload_module(pkg_resources) def run(self, cmd, cwd=os.curdir): c = None @@ -248,18 +268,42 @@ def run_py(self, cmd, cwd=os.curdir): def is_installed(self, pkgname): return any(d for d in self.get_distributions() if d.project_name == pkgname) + def get_monkeypatched_pathset(self): + import recursive_monkey_patch + from pip_shims.shims import req_install + req_uninstall_name = "{0}.req_uninstall".format(req_install.__package__) + if req_uninstall_name not in sys.modules: + req_uninstall = importlib.import_module(req_uninstall_name) + else: + req_uninstall = sys.modules[req_uninstall_name] + six.moves.reload_module(req_uninstall) + recursive_monkey_patch.monkey_patch(PatchedUninstaller, req_uninstall.UninstallPathSet) + return req_uninstall.UninstallPathSet + + @contextlib.contextmanager def uninstall(self, pkgname, *args, **kwargs): + auto_confirm = kwargs.pop("auto_confirm", True) + verbose = kwargs.pop("verbose", False) with self.activated(): - from pip_shims.shims import InstallRequirement - module_name = InstallRequirement.__module__ - del InstallRequirement - if module_name not in sys.modules: - pip_req_install = importlib.import_module(module_name) + pathset_base = self.get_monkeypatched_pathset() + dist = next( + iter(filter(lambda d: d.project_name == pkgname, self.get_working_set())), + None + ) + pathset = pathset_base.from_dist(dist) + print(pathset.paths) + if pathset is not None: + pathset.remove(auto_confirm=auto_confirm, verbose=True) + try: + yield pathset + except Exception as e: + if pathset is not None: + pathset.rollback() else: - pip_req_install = sys.modules[module_name] - importlib.reload(sys.modules[module_name]) - ireq = pip_req_install.InstallRequirement.from_line(pkgname) - return RequirementUninstaller(ireq, *args, **kwargs) + if pathset is not None: + pathset.commit() + if pathset is None: + return SETUPTOOLS_SHIM = ( @@ -269,3 +313,8 @@ def uninstall(self, pkgname, *args, **kwargs): "f.close();" "exec(compile(code, __file__, 'exec'))" ) + + +class PatchedUninstaller(object): + def _permitted(self, path): + return True From 766d73e409b94574e83fed0f2e220f1761779a4b Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 20 Sep 2018 01:15:03 -0400 Subject: [PATCH 13/32] Fix cleaning working set and messaging Signed-off-by: Dan Ryan --- src/passa/actions/clean.py | 3 +- src/passa/actions/init.py | 2 +- src/passa/models/synchronizers.py | 11 +++- src/passa/models/virtualenv.py | 103 ++++++++++++++---------------- src/passa/operations/sync.py | 4 +- 5 files changed, 62 insertions(+), 61 deletions(-) diff --git a/src/passa/actions/clean.py b/src/passa/actions/clean.py index 8c6ae17..9006f22 100644 --- a/src/passa/actions/clean.py +++ b/src/passa/actions/clean.py @@ -13,4 +13,5 @@ def clean(project, default=True, dev=False, sync=True): if not success: return 1 - print("Cleaned project at", project.root) + if sync: + print("Cleaned project at", project.root) diff --git a/src/passa/actions/init.py b/src/passa/actions/init.py index 1d9f592..bbab009 100644 --- a/src/passa/actions/init.py +++ b/src/passa/actions/init.py @@ -42,7 +42,7 @@ def init_project(root=None, python_version=None): index_urls = [parsed.index_url] + parsed.extra_index_urls sources = get_sources(index_urls, parsed.trusted_hosts) data = { - "sources": sources, + "source": sources, "packages": {}, "dev-packages": {}, } diff --git a/src/passa/models/synchronizers.py b/src/passa/models/synchronizers.py index 33fcfc3..9c8aa9c 100644 --- a/src/passa/models/synchronizers.py +++ b/src/passa/models/synchronizers.py @@ -234,7 +234,7 @@ def _sync(self): class Cleaner(object): """Helper class to clean packages not in a project's lock file. """ - def __init__(self, project, default, develop, sync=True): + def __init__(self, project, default, develop, sync=True, verbose=False): self._root = project.root # Only for repr. self.packages = _get_packages(project.lockfile, default, develop) self.sync = sync @@ -243,9 +243,18 @@ def __init__(self, project, default, develop, sync=True): def __repr__(self): return "<{0} @ {1!r}>".format(type(self).__name__, self._root) + def print(self, packages): + if not self.sync: + message = "Would clean: {0}" + else: + message = "Cleaned: {0}" + print(message.format(", ".join(sorted(set(packages))))) + def clean(self): groupcoll = _group_installed_names(self.packages, venv=self.project.venv) cleaned = set() if self.sync: cleaned = _clean(groupcoll.unneeded, venv=self.project.venv) + else: + return groupcoll.unneeded return cleaned diff --git a/src/passa/models/virtualenv.py b/src/passa/models/virtualenv.py index c179e3c..a4f3954 100644 --- a/src/passa/models/virtualenv.py +++ b/src/passa/models/virtualenv.py @@ -6,6 +6,7 @@ import hashlib import importlib import json +import posixpath import os import re import six @@ -20,6 +21,7 @@ class VirtualEnv(object): def __init__(self, venv_dir): + self.recursive_monkey_patch = self.safe_import("recursive_monkey_patch") self.venv_dir = vistir.compat.Path(venv_dir) @classmethod @@ -79,6 +81,26 @@ def get_workon_home(cls): ) return vistir.compat.Path(os.path.expandvars(workon_home)).expanduser() + @classmethod + def filter_sources(cls, requirement, sources): + if not sources or not requirement.index: + return sources + filtered_sources = [ + source for source in sources + if source.get("name") == requirement.index + ] + return filtered_sources or sources + + @classmethod + def safe_import(cls, name): + module = None + if name not in sys.modules: + module = importlib.import_module(name) + else: + module = sys.modules[name] + six.moves.reload_module(module) + return module + @cached_property def script_basedir(self): script_dir = os.path.basename(sysconfig.get_paths()["scripts"]) @@ -93,13 +115,16 @@ def sys_path(self): c = vistir.misc.run([self.python, "-c", "import json,sys; print(json.dumps(sys.path))"], return_object=True, nospin=True) assert c.returncode == 0, "failed loading virtualenv path" - path = json.loads(c.out.strip()) + path = [ + path for path in json.loads(c.out.strip()) + if posixpath.normpath(path).startswith(posixpath.normpath(str(self.venv_dir))) + ] return path @cached_property def system_paths(self): paths = {} - six.moves.reload_module(sysconfig) + sysconfig = self.safe_import("sysconfig") paths = sysconfig.get_paths() return paths @@ -116,7 +141,7 @@ def paths(self): os.environ["PYTHONUSERBASE"] = vistir.compat.fs_str(self.venv_dir.as_posix()) os.environ["PYTHONIOENCODING"] = vistir.compat.fs_str("utf-8") os.environ["PYTHONDONTWRITEBYTECODE"] = vistir.compat.fs_str("1") - six.moves.reload_module(sysconfig) + sysconfig = self.safe_import("sysconfig") scheme, _, _ = sysconfig._get_default_scheme().partition('_') scheme = "{0}_user".format(scheme) paths = sysconfig.get_paths(scheme=scheme) @@ -128,55 +153,27 @@ def scripts_dir(self): @cached_property def passa_entry(self): - import pkg_resources + pkg_resources = self.safe_import("pkg_resources") return pkg_resources.working_set.by_key['passa'].location def get_distributions(self): - import pkg_resources - six.moves.reload_module(pkg_resources) + pkg_resources = self.safe_import("pkg_resources") return pkg_resources.find_distributions(self.paths["purelib"], only=True) def get_working_set(self): working_set = None import pkg_resources passa_entry = self.passa_entry - monkeypatch = self.monkeypatch_dist - working_set = pkg_resources.WorkingSet(self.sys_path + [passa_entry, monkeypatch]) + working_set = pkg_resources.WorkingSet(self.sys_path + [passa_entry]) return working_set - @classmethod - def filter_sources(cls, requirement, sources): - if not sources or not requirement.index: - return sources - filtered_sources = [ - source for source in sources - if source.get("name") == requirement.index - ] - return filtered_sources or sources - @cached_property def python_version(self): with self.activated(): - six.moves.reload_module(sysconfig) + sysconfig = self.safe_import("sysconfig") py_version = sysconfig.get_python_version() return py_version - @classmethod - def safe_import(cls, name): - module = None - if name not in sys.modules: - module = importlib.import_module(name) - else: - module = sys.modules[name] - six.moves.reload_module(module) - return module - - @cached_property - def monkeypatch_dist(self): - pkg_resources = self.safe_import("pkg_resources") - monkey_patch = pkg_resources.get_distribution('recursive-monkey-patch').location - return monkey_patch - def get_setup_install_args(self, pkgname, setup_py, develop=False): headers = vistir.compat.Path(self.sys_prefix) / "include" / "site" headers = headers / "python{0}".format(self.python_version) / pkgname @@ -194,7 +191,7 @@ def get_setup_install_args(self, pkgname, setup_py, develop=False): def install(self, req, editable=False, sources=[]): with self.activated(): install_options = ["--prefix={0}".format(self.venv_dir),] - six.moves.reload_module(passa.internals._pip) + passa_pip = self.safe_import("passa.internals._pip") ireq = req.as_ireq() if editable: with vistir.contextmanagers.cd(ireq.setup_py_dir, ireq.setup_py): @@ -204,11 +201,11 @@ def install(self, req, editable=False, sources=[]): ), cwd=ireq.setup_py_dir ) return c.returncode - six.moves.reload_module(distlib.scripts) + distlib_scripts = self.safe_import("distlib.scripts") sources = self.filter_sources(req, sources) hashes = req.hashes - wheel = passa.internals._pip.build_wheel(ireq, sources, hashes) - wheel.install(self.paths, distlib.scripts.ScriptMaker(None, None)) + wheel = passa_pip.build_wheel(ireq, sources, hashes) + wheel.install(self.paths, distlib_scripts.ScriptMaker(None, None)) @contextlib.contextmanager def activated(self): @@ -217,22 +214,20 @@ def activated(self): original_user_base = os.environ.get("PYTHONUSERBASE", None) original_venv = os.environ.get("VIRTUAL_ENV", None) passa_path = vistir.compat.Path(__file__).absolute().parent.parent.as_posix() - monkeypatch_dist = self.monkeypatch_dist with vistir.contextmanagers.temp_environ(), vistir.contextmanagers.temp_path(): - os.environ["PYTHONUSERBASE"] = vistir.compat.fs_str(self.venv_dir.as_posix()) os.environ["PYTHONIOENCODING"] = vistir.compat.fs_str("utf-8") os.environ["PYTHONDONTWRITEBYTECODE"] = vistir.compat.fs_str("1") - os.environ["VIRTUAL_ENV"] = vistir.compat.fs_str(self.venv_dir.as_posix()) - sys.path = self.sys_path - sys.prefix = self.venv_dir - import site - # sys.path.append(passa_path) activate_this = os.path.join(self.scripts_dir, "activate_this.py") with open(activate_this, "r") as f: code = compile(f.read(), activate_this, "exec") exec(code, dict(__file__=activate_this)) + os.environ["PYTHONUSERBASE"] = vistir.compat.fs_str(self.venv_dir.as_posix()) + os.environ["VIRTUAL_ENV"] = vistir.compat.fs_str(self.venv_dir.as_posix()) + sys.path = self.sys_path + sys.prefix = self.venv_dir + site = self.safe_import("site") site.addsitedir(passa_path) - site.addsitedir(monkeypatch_dist) + sys.modules["recursive_monkey_patch"] = self.recursive_monkey_patch pkg_resources = self.safe_import("pkg_resources") try: yield @@ -269,15 +264,12 @@ def is_installed(self, pkgname): return any(d for d in self.get_distributions() if d.project_name == pkgname) def get_monkeypatched_pathset(self): - import recursive_monkey_patch from pip_shims.shims import req_install req_uninstall_name = "{0}.req_uninstall".format(req_install.__package__) - if req_uninstall_name not in sys.modules: - req_uninstall = importlib.import_module(req_uninstall_name) - else: - req_uninstall = sys.modules[req_uninstall_name] - six.moves.reload_module(req_uninstall) - recursive_monkey_patch.monkey_patch(PatchedUninstaller, req_uninstall.UninstallPathSet) + req_uninstall = self.safe_import(req_uninstall_name) + self.recursive_monkey_patch.monkey_patch( + PatchedUninstaller, req_uninstall.UninstallPathSet + ) return req_uninstall.UninstallPathSet @contextlib.contextmanager @@ -291,9 +283,8 @@ def uninstall(self, pkgname, *args, **kwargs): None ) pathset = pathset_base.from_dist(dist) - print(pathset.paths) if pathset is not None: - pathset.remove(auto_confirm=auto_confirm, verbose=True) + pathset.remove(auto_confirm=auto_confirm, verbose=verbose) try: yield pathset except Exception as e: diff --git a/src/passa/operations/sync.py b/src/passa/operations/sync.py index 3014e8d..45502a4 100644 --- a/src/passa/operations/sync.py +++ b/src/passa/operations/sync.py @@ -16,8 +16,8 @@ def sync(syncer): def clean(cleaner): - print("Cleaning") + print("Cleaning...") cleaned = cleaner.clean() if cleaned: - print("Uninstalled: {}".format(", ".join(sorted(cleaned)))) + cleaner.print(cleaned) return True From bf16fde9da48ac381197501fe9a343d5106b59a7 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 21 Sep 2018 00:56:15 -0400 Subject: [PATCH 14/32] Split into separate libraries Signed-off-by: Dan Ryan --- docs/requirements.txt | 2 + setup.cfg | 4 + src/passa/actions/add.py | 4 +- src/passa/actions/clean.py | 4 +- src/passa/actions/init.py | 15 +- src/passa/actions/install.py | 4 +- src/passa/actions/remove.py | 4 +- src/passa/actions/sync.py | 4 +- src/passa/actions/upgrade.py | 4 +- src/passa/cli/options.py | 10 +- src/passa/internals/_pip.py | 397 ---------------------------- src/passa/internals/_pip_shims.py | 61 ----- src/passa/internals/candidates.py | 7 +- src/passa/internals/dependencies.py | 22 +- src/passa/internals/utils.py | 15 -- src/passa/models/caches.py | 3 +- src/passa/models/providers.py | 4 +- src/passa/models/synchronizers.py | 260 ------------------ src/passa/models/virtualenv.py | 311 ---------------------- src/passa/operations/sync.py | 23 -- 20 files changed, 39 insertions(+), 1119 deletions(-) create mode 100644 docs/requirements.txt delete mode 100644 src/passa/internals/_pip.py delete mode 100644 src/passa/internals/_pip_shims.py delete mode 100644 src/passa/models/synchronizers.py delete mode 100644 src/passa/models/virtualenv.py delete mode 100644 src/passa/operations/sync.py diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..8213302 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,2 @@ +sphinx +sphinx_rtd_theme diff --git a/setup.cfg b/setup.cfg index 4daf9ee..8d9c3b7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,9 @@ setup_requires = setuptools>=36.2.2 install_requires = appdirs distlib + installer packaging + packagebuilder pip-shims>=0.1.2 plette[validation]>=0.2.2 requests @@ -52,6 +54,8 @@ install_requires = pack = invoke parver +virtualenv = + mork tests = cached-property pytest-xdist diff --git a/src/passa/actions/add.py b/src/passa/actions/add.py index 3efc4dc..6cfb3e2 100644 --- a/src/passa/actions/add.py +++ b/src/passa/actions/add.py @@ -39,8 +39,8 @@ def add_packages(packages=[], editables=[], project=None, dev=False, sync=False, if not sync: return - from passa.models.synchronizers import Synchronizer - from passa.operations.sync import sync + from installer.synchronizer import Synchronizer + from installer.operations import sync lockfile_diff = project.difference_lockfile(prev_lockfile) default = any(lockfile_diff.default) diff --git a/src/passa/actions/clean.py b/src/passa/actions/clean.py index 9006f22..d4b8193 100644 --- a/src/passa/actions/clean.py +++ b/src/passa/actions/clean.py @@ -4,8 +4,8 @@ def clean(project, default=True, dev=False, sync=True): - from passa.models.synchronizers import Cleaner - from passa.operations.sync import clean + from installer.synchronizer import Cleaner + from installer.operations import clean cleaner = Cleaner(project, default=default, develop=dev, sync=sync) diff --git a/src/passa/actions/init.py b/src/passa/actions/init.py index bbab009..baf8445 100644 --- a/src/passa/actions/init.py +++ b/src/passa/actions/init.py @@ -4,14 +4,13 @@ import io import os -from pip_shims import Command as PipCommand, cmdoptions -import plette + import six -import vistir +import packagebuilder -class PipCmd(PipCommand): - name = "PipCmd" +import plette +import vistir def get_sources(urls, trusted_hosts): @@ -36,11 +35,7 @@ def init_project(root=None, python_version=None): raise RuntimeError("{0!r} is already a Pipfile project".format(root)) if not os.path.exists(root): vistir.path.mkdir_p(root, mode=0o755) - pip_command = PipCmd() - cmdoptions.make_option_group(cmdoptions.index_group, pip_command.parser) - parsed, _ = pip_command.parser.parse_args([]) - index_urls = [parsed.index_url] + parsed.extra_index_urls - sources = get_sources(index_urls, parsed.trusted_hosts) + sources = packagebuilder.get_sources() data = { "source": sources, "packages": {}, diff --git a/src/passa/actions/install.py b/src/passa/actions/install.py index 1728dae..c920906 100644 --- a/src/passa/actions/install.py +++ b/src/passa/actions/install.py @@ -17,8 +17,8 @@ def install(project=None, check=True, dev=False, clean=True): project._l.write() print("Written to project at", project.root) - from passa.models.synchronizers import Synchronizer - from passa.operations.sync import sync + from installer.synchronizer import Synchronizer + from installer.operations import sync syncer = Synchronizer( project, default=True, develop=dev, diff --git a/src/passa/actions/remove.py b/src/passa/actions/remove.py index 17ba1c7..2fcba75 100644 --- a/src/passa/actions/remove.py +++ b/src/passa/actions/remove.py @@ -27,8 +27,8 @@ def remove(project=None, only="default", packages=[], clean=True, sync=False): if not clean: return - from passa.models.synchronizers import Cleaner - from passa.operations.sync import clean + from installer.synchronizer import Cleaner + from installer.operations import clean cleaner = Cleaner(project, default=True, develop=True) success = clean(cleaner) diff --git a/src/passa/actions/sync.py b/src/passa/actions/sync.py index 23e36ee..e785627 100644 --- a/src/passa/actions/sync.py +++ b/src/passa/actions/sync.py @@ -4,8 +4,8 @@ def sync(project=None, dev=False, clean=True): - from passa.models.synchronizers import Synchronizer - from passa.operations.sync import sync + from installer.synchronizers import Synchronizer + from installer.operations import sync project = project syncer = Synchronizer( diff --git a/src/passa/actions/upgrade.py b/src/passa/actions/upgrade.py index fb3ad7f..7fd3c1e 100644 --- a/src/passa/actions/upgrade.py +++ b/src/passa/actions/upgrade.py @@ -34,8 +34,8 @@ def upgrade(project=None, strategy="only-if-needed", sync=True, packages=[]): if not sync: return - from passa.operations.sync import sync - from passa.models.synchronizers import Synchronizer + from installer.operations import sync + from installer.synchronizer import Synchronizer lockfile_diff = project.difference_lockfile(prev_lockfile) default = bool(any(lockfile_diff.default)) diff --git a/src/passa/cli/options.py b/src/passa/cli/options.py index 07782db..d2a6378 100644 --- a/src/passa/cli/options.py +++ b/src/passa/cli/options.py @@ -10,7 +10,6 @@ import tomlkit.exceptions import passa.models.projects -import passa.models.virtualenv import vistir @@ -25,20 +24,13 @@ def __init__(self, root, *args, **kwargs): raise argparse.ArgumentError( project, "{0!r} is not a Pipfile project".format(root.as_posix()), ) - self.venv = self.get_venv(root) try: - super(Project, self).__init__(root.as_posix(), env_prefix=self.venv.venv_dir, - *args, **kwargs) + super(Project, self).__init__(root.as_posix(), *args, **kwargs) except tomlkit.exceptions.ParseError as e: raise argparse.ArgumentError( project, "failed to parse Pipfile: {0!r}".format(str(e)), ) - def get_venv(self, root): - if 'VIRTUAL_ENV' in os.environ: - return passa.models.virtualenv.VirtualEnv(os.environ['VIRTUAL_ENV']) - return passa.models.virtualenv.VirtualEnv.from_project_path(root) - def __name__(self): return "Project Root" diff --git a/src/passa/internals/_pip.py b/src/passa/internals/_pip.py deleted file mode 100644 index 2aa143a..0000000 --- a/src/passa/internals/_pip.py +++ /dev/null @@ -1,397 +0,0 @@ -# -*- coding=utf-8 -*- - -from __future__ import absolute_import, unicode_literals - -import contextlib -import io -import itertools -import distutils.log -import os - -import distlib.database -import distlib.scripts -import distlib.wheel -import packaging.utils -import pip_shims -import setuptools.dist -import six -import vistir - -from ..models.caches import CACHE_DIR -from ._pip_shims import VCS_SUPPORT, build_wheel as _build_wheel, unpack_url -from .utils import filter_sources - - -@vistir.path.ensure_mkdir_p(mode=0o775) -def _get_src_dir(): - src = os.environ.get("PIP_SRC") - if src: - return src - virtual_env = os.environ.get("VIRTUAL_ENV") - if virtual_env: - return os.path.join(virtual_env, "src") - return os.path.join(os.getcwd(), "src") # Match pip's behavior. - - -def _prepare_wheel_building_kwargs(ireq): - download_dir = os.path.join(CACHE_DIR, "pkgs") - vistir.mkdir_p(download_dir) - - wheel_download_dir = os.path.join(CACHE_DIR, "wheels") - vistir.mkdir_p(wheel_download_dir) - - if ireq.source_dir is not None: - src_dir = ireq.source_dir - elif ireq.editable: - src_dir = _get_src_dir() - else: - src_dir = vistir.path.create_tracked_tempdir(prefix='passa-src') - - # This logic matches pip's behavior, although I don't fully understand the - # intention. I guess the idea is to build editables in-place, otherwise out - # of the source tree? - if ireq.editable: - build_dir = src_dir - else: - build_dir = vistir.path.create_tracked_tempdir(prefix="passa-build") - - return { - "build_dir": build_dir, - "src_dir": src_dir, - "download_dir": download_dir, - "wheel_download_dir": wheel_download_dir, - } - - -def _get_pip_index_urls(sources): - index_urls = [] - trusted_hosts = [] - for source in sources: - url = source.get("url") - if not url: - continue - index_urls.append(url) - if source.get("verify_ssl", True): - continue - host = six.moves.urllib.parse.urlparse(source["url"]).hostname - trusted_hosts.append(host) - return index_urls, trusted_hosts - - -class _PipCommand(pip_shims.Command): - name = "PipCommand" - - -def _get_pip_session(trusted_hosts): - cmd = _PipCommand() - options, _ = cmd.parser.parse_args([]) - options.cache_dir = CACHE_DIR - options.trusted_hosts = trusted_hosts - session = cmd._build_session(options) - return session - - -def _get_finder(sources): - index_urls, trusted_hosts = _get_pip_index_urls(sources) - session = _get_pip_session(trusted_hosts) - finder = pip_shims.PackageFinder( - find_links=[], - index_urls=index_urls, - trusted_hosts=trusted_hosts, - allow_all_prereleases=True, - session=session, - ) - return finder - - -def _get_wheel_cache(): - format_control = pip_shims.FormatControl(set(), set()) - wheel_cache = pip_shims.WheelCache(CACHE_DIR, format_control) - return wheel_cache - - -def _convert_hashes(values): - """Convert Pipfile.lock hash lines into InstallRequirement option format. - - The option format uses a str-list mapping. Keys are hash algorithms, and - the list contains all values of that algorithm. - """ - hashes = {} - if not values: - return hashes - for value in values: - try: - name, value = value.split(":", 1) - except ValueError: - name = "sha256" - if name not in hashes: - hashes[name] = [] - hashes[name].append(value) - return hashes - - -class WheelBuildError(RuntimeError): - pass - - -def build_wheel(ireq, sources, hashes=None): - """Build a wheel file for the InstallRequirement object. - - An artifact is downloaded (or read from cache). If the artifact is not a - wheel, build one out of it. The dynamically built wheel is ephemeral; do - not depend on its existence after the returned wheel goes out of scope. - - If `hashes` is truthy, it is assumed to be a list of hashes (as formatted - in Pipfile.lock) to be checked against the download. - - Returns a `distlib.wheel.Wheel` instance. Raises a `WheelBuildError` (a - `RuntimeError` subclass) if the wheel cannot be built. - """ - kwargs = _prepare_wheel_building_kwargs(ireq) - finder = _get_finder(sources) - - # Not for upgrade, hash not required. Hashes are not required here even - # when we provide them, because pip skips local wheel cache if we set it - # to True. Hashes are checked later if we need to download the file. - ireq.populate_link(finder, False, False) - - # Ensure ireq.source_dir is set. - # This is intentionally set to build_dir, not src_dir. Comments from pip: - # [...] if filesystem packages are not marked editable in a req, a non - # deterministic error occurs when the script attempts to unpack the - # build directory. - # Also see comments in `_prepare_wheel_building_kwargs()` -- If the ireq - # is editable, build_dir is actually src_dir, making the build in-place. - ireq.ensure_has_source_dir(kwargs["build_dir"]) - - # Ensure the source is fetched. For wheels, it is enough to just download - # because we'll use them directly. For an sdist, we need to unpack so we - # can build it. - if not ireq.editable or not pip_shims.is_file_url(ireq.link): - if ireq.is_wheel: - only_download = True - download_dir = kwargs["wheel_download_dir"] - else: - only_download = False - download_dir = kwargs["download_dir"] - ireq.options["hashes"] = _convert_hashes(hashes) - unpack_url( - ireq.link, ireq.source_dir, download_dir, - only_download=only_download, session=finder.session, - hashes=ireq.hashes(False), progress_bar="off", - ) - - if ireq.is_wheel: - # If this is a wheel, use the downloaded thing. - output_dir = kwargs["wheel_download_dir"] - wheel_path = os.path.join(output_dir, ireq.link.filename) - else: - # Othereise we need to build an ephemeral wheel. - wheel_path = _build_wheel( - ireq, vistir.path.create_tracked_tempdir(prefix="ephem"), - finder, _get_wheel_cache(), kwargs, - ) - if wheel_path is None or not os.path.exists(wheel_path): - raise WheelBuildError - return distlib.wheel.Wheel(wheel_path) - - -def _obtrain_ref(vcs_obj, src_dir, name, rev=None): - target_dir = os.path.join(src_dir, name) - target_rev = vcs_obj.make_rev_options(rev) - if not os.path.exists(target_dir): - vcs_obj.obtain(target_dir) - if (not vcs_obj.is_commit_id_equal(target_dir, rev) and - not vcs_obj.is_commit_id_equal(target_dir, target_rev)): - vcs_obj.update(target_dir, target_rev) - return vcs_obj.get_revision(target_dir) - - -def get_vcs_ref(requirement): - backend = VCS_SUPPORT.get_backend(requirement.vcs) - vcs = backend(url=requirement.req.vcs_uri) - src = _get_src_dir() - name = requirement.normalized_name - ref = _obtrain_ref(vcs, src, name, rev=requirement.req.ref) - return ref - - -def find_installation_candidates(ireq, sources): - finder = _get_finder(sources) - return finder.find_all_candidates(ireq.name) - - -class RequirementUninstaller(object): - """A context manager to remove a package for the inner block. - - This uses `UninstallPathSet` to control the workflow. If the inner block - exits correctly, the uninstallation is committed, otherwise rolled back. - """ - def __init__(self, ireq, auto_confirm, verbose): - self.ireq = ireq - self.pathset = None - self.auto_confirm = auto_confirm - self.verbose = verbose - - def __enter__(self): - self.pathset = self.ireq.uninstall( - auto_confirm=self.auto_confirm, - verbose=self.verbose, - ) - return self.pathset - - def __exit__(self, exc_type, exc_value, traceback): - if self.pathset is None: - return - if exc_type is None: - self.pathset.commit() - else: - self.pathset.rollback() - - -def uninstall(name, **kwargs): - ireq = pip_shims.InstallRequirement.from_line(name) - return RequirementUninstaller(ireq, **kwargs) - - -@contextlib.contextmanager -def _suppress_distutils_logs(): - """Hack to hide noise generated by `setup.py develop`. - - There isn't a good way to suppress them now, so let's monky-patch. - See https://bugs.python.org/issue25392. - """ - f = distutils.log.Log._log - - def _log(log, level, msg, args): - if level >= distutils.log.ERROR: - f(log, level, msg, args) - - distutils.log.Log._log = _log - yield - distutils.log.Log._log = f - - -class NoopInstaller(object): - """An installer. - - This class is not designed to be instantiated by itself, but used as a - common interface for subclassing. - - An installer has two methods, `prepare()` and `install()`. Neither takes - arguments, and should be called in that order to prepare an installation - operation, and to actually install things. - """ - def prepare(self): - pass - - def install(self): - pass - - -class EditableInstaller(NoopInstaller): - """Installer to handle editable. - """ - def __init__(self, requirement): - ireq = requirement.as_ireq() - self.working_directory = ireq.setup_py_dir - self.setup_py = ireq.setup_py - - def install(self): - with vistir.cd(self.working_directory), _suppress_distutils_logs(): - # Access from Setuptools to ensure things are patched correctly. - setuptools.dist.distutils.core.run_setup( - self.setup_py, ["develop", "--no-deps"], - ) - - -class WheelInstaller(NoopInstaller): - """Installer by building a wheel. - - The wheel is built during `prepare()`, and installed in `install()`. - """ - def __init__(self, requirement, sources, paths): - self.ireq = requirement.as_ireq() - self.sources = filter_sources(requirement, sources) - self.hashes = requirement.hashes or None - self.paths = paths - self.wheel = None - - def prepare(self): - self.wheel = build_wheel(self.ireq, self.sources, self.hashes) - - def install(self): - self.wheel.install(self.paths, distlib.scripts.ScriptMaker(None, None)) - - -def _iter_egg_info_directories(root, name): - name = packaging.utils.canonicalize_name(name) - for parent, dirnames, filenames in os.walk(root): - matched_indexes = [] - for i, dirname in enumerate(dirnames): - if not dirname.lower().endswith("egg-info"): - continue - egg_info_name = packaging.utils.canonicalize_name(dirname[:-9]) - if egg_info_name != name: - continue - matched_indexes.append(i) - yield os.path.join(parent, dirname) - - # Modify dirnames in-place to NOT look into egg-info directories. - # This is a documented behavior in stdlib. - for i in reversed(matched_indexes): - del dirnames[i] - - -def _read_pkg_info(directory): - path = os.path.join(directory, "PKG-INFO") - try: - with io.open(path, encoding="utf-8", errors="replace") as f: - return f.read() - except (IOError, OSError): - return None - - -def _find_egg_info(ireq): - """Find this package's .egg-info directory. - - Due to how sdists are designed, the .egg-info directory cannot be reliably - found without running setup.py to aggregate all configurations. This - function instead uses some heuristics to locate the egg-info directory - that most likely represents this package. - - The best .egg-info directory's path is returned as a string. None is - returned if no matches can be found. - """ - root = ireq.setup_py_dir - - directory_iterator = _iter_egg_info_directories(root, ireq.name) - try: - top_egg_info = next(directory_iterator) - except StopIteration: # No egg-info found. Wat. - return None - directory_iterator = itertools.chain([top_egg_info], directory_iterator) - - # Read the sdist's PKG-INFO to determine which egg_info is best. - pkg_info = _read_pkg_info(root) - - # PKG-INFO not readable. Just return whatever comes first, I guess. - if pkg_info is None: - return top_egg_info - - # Walk the sdist to find the egg-info with matching PKG-INFO. - for directory in directory_iterator: - egg_pkg_info = _read_pkg_info(directory) - if egg_pkg_info == pkg_info: - return directory - - # Nothing matches...? Use the first one we found, I guess. - return top_egg_info - - -def read_sdist_metadata(ireq): - egg_info_dir = _find_egg_info(ireq) - if not egg_info_dir: - return None - distribution = distlib.database.EggInfoDistribution(egg_info_dir) - return distribution.metadata diff --git a/src/passa/internals/_pip_shims.py b/src/passa/internals/_pip_shims.py deleted file mode 100644 index b2c7b6e..0000000 --- a/src/passa/internals/_pip_shims.py +++ /dev/null @@ -1,61 +0,0 @@ -# -*- coding=utf-8 -*- - -"""Shims to make the pip interface more consistent accross versions. - -There are currently two members: - -* VCS_SUPPORT is an instance of VcsSupport. -* build_wheel abstracts the process to build a wheel out of a bunch parameters. -* unpack_url wraps the actual function in pip to accept modern parameters. -""" - -from __future__ import absolute_import, unicode_literals - -import pip_shims - - -def _build_wheel_pre10(ireq, output_dir, finder, wheel_cache, kwargs): - kwargs.update({"wheel_cache": wheel_cache, "session": finder.session}) - reqset = pip_shims.RequirementSet(**kwargs) - builder = pip_shims.WheelBuilder(reqset, finder) - return builder._build_one(ireq, output_dir) - - -def _build_wheel_modern(ireq, output_dir, finder, wheel_cache, kwargs): - """Build a wheel. - - * ireq: The InstallRequirement object to build - * output_dir: The directory to build the wheel in. - * finder: pip's internal Finder object to find the source out of ireq. - * kwargs: Various keyword arguments from `_prepare_wheel_building_kwargs`. - """ - kwargs.update({"progress_bar": "off", "build_isolation": False}) - with pip_shims.RequirementTracker() as req_tracker: - if req_tracker: - kwargs["req_tracker"] = req_tracker - preparer = pip_shims.RequirementPreparer(**kwargs) - builder = pip_shims.WheelBuilder(finder, preparer, wheel_cache) - return builder._build_one(ireq, output_dir) - - -def _unpack_url_pre10(*args, **kwargs): - """Shim for unpack_url in various pip versions. - - pip before 10.0 does not accept `progress_bar` here. Simply drop it. - """ - kwargs.pop("progress_bar", None) - return pip_shims.unpack_url(*args, **kwargs) - - -PIP_VERSION = pip_shims.utils._parse(pip_shims.pip_version) -VERSION_10 = pip_shims.utils._parse("10") - - -VCS_SUPPORT = pip_shims.VcsSupport() - -build_wheel = _build_wheel_modern -unpack_url = pip_shims.unpack_url - -if PIP_VERSION < VERSION_10: - build_wheel = _build_wheel_pre10 - unpack_url = _unpack_url_pre10 diff --git a/src/passa/internals/candidates.py b/src/passa/internals/candidates.py index 67b0904..9076206 100644 --- a/src/passa/internals/candidates.py +++ b/src/passa/internals/candidates.py @@ -6,7 +6,8 @@ import packaging.version import requirementslib -from ._pip import find_installation_candidates, get_vcs_ref +from packagebuilder import find_installation_candidates +from ..models.caches import CACHE_DIR def _filter_matching_python_requirement(candidates, required_python): @@ -57,11 +58,11 @@ def find_candidates(requirement, sources, requires_python, allow_prereleases): if not requirement.is_named: candidate = _copy_requirement(requirement) if candidate.is_vcs: - candidate.req.ref = get_vcs_ref(candidate) + candidate.req.ref = requirement.get_commit_hash() return [candidate] ireq = requirement.as_ireq() - icans = find_installation_candidates(ireq, sources) + icans = find_installation_candidates(ireq, sources, cache_dir=CACHE_DIR) if requires_python: matching_icans = list(_filter_matching_python_requirement( diff --git a/src/passa/internals/dependencies.py b/src/passa/internals/dependencies.py index 410a5e6..e8c6d31 100644 --- a/src/passa/internals/dependencies.py +++ b/src/passa/internals/dependencies.py @@ -6,6 +6,7 @@ import os import sys +import packagebuilder import packaging.specifiers import packaging.utils import packaging.version @@ -13,8 +14,7 @@ import requirementslib import six -from ..models.caches import DependencyCache, RequiresPythonCache -from ._pip import WheelBuildError, build_wheel, read_sdist_metadata +from ..models.caches import DependencyCache, RequiresPythonCache, CACHE_DIR from .markers import contains_extra, get_contained_extras, get_without_extra from .utils import get_pinned_version, is_pinned @@ -224,20 +224,10 @@ def _get_dependencies_from_pip(ireq, sources): 2. Read metadata out of the egg-info directory if it is present. """ extras = ireq.extras or () - try: - wheel = build_wheel(ireq, sources) - except WheelBuildError: - # XXX: This depends on a side effect of `build_wheel`. This block is - # reached when it fails to build an sdist, where the sdist would have - # been downloaded, extracted into `ireq.source_dir`, and partially - # built (hopefully containing .egg-info). - metadata = read_sdist_metadata(ireq) - if not metadata: - raise - else: - metadata = wheel.metadata - requirements = _read_requirements(metadata, extras) - requires_python = _read_requires_python(metadata) + builder = packagebuilder.BuiltDist(ireq, sources=sources, cache_dir=CACHE_DIR) + dist = builder.build() + requirements = _read_requirements(dist.metadata, extras) + requires_python = _read_requires_python(dist.metadata) return requirements, requires_python diff --git a/src/passa/internals/utils.py b/src/passa/internals/utils.py index 8f8e6fd..0dd75d1 100644 --- a/src/passa/internals/utils.py +++ b/src/passa/internals/utils.py @@ -82,21 +82,6 @@ def is_pinned(ireq): return True -def filter_sources(requirement, sources): - """Returns a filtered list of sources for this requirement. - - This considers the index specified by the requirement, and returns only - matching source entries if there is at least one. - """ - if not sources or not requirement.index: - return sources - filtered_sources = [ - source for source in sources - if source.get("name") == requirement.index - ] - return filtered_sources or sources - - def get_allow_prereleases(requirement, global_setting): # TODO: Implement per-package prereleases flag. (pypa/pipenv#1696) return global_setting diff --git a/src/passa/models/caches.py b/src/passa/models/caches.py index c6d29b5..bda8cc2 100644 --- a/src/passa/models/caches.py +++ b/src/passa/models/caches.py @@ -13,12 +13,13 @@ import requests import vistir -from ..internals._pip_shims import VCS_SUPPORT from ..internals.utils import get_pinned_version CACHE_DIR = os.environ.get("PASSA_CACHE_DIR", appdirs.user_cache_dir("passa")) +VCS_SUPPORT = pip_shims.VcsSupport() + class HashCache(pip_shims.SafeFileCache): """Caches hashes of PyPI artifacts so we do not need to re-download them. diff --git a/src/passa/models/providers.py b/src/passa/models/providers.py index 36b2f2e..e62a031 100644 --- a/src/passa/models/providers.py +++ b/src/passa/models/providers.py @@ -6,10 +6,12 @@ import resolvelib +from packagebuilder import filter_sources + from ..internals.candidates import find_candidates from ..internals.dependencies import get_dependencies from ..internals.utils import ( - filter_sources, get_allow_prereleases, identify_requirment, strip_extras, + get_allow_prereleases, identify_requirment, strip_extras, ) diff --git a/src/passa/models/synchronizers.py b/src/passa/models/synchronizers.py deleted file mode 100644 index 9c8aa9c..0000000 --- a/src/passa/models/synchronizers.py +++ /dev/null @@ -1,260 +0,0 @@ -# -*- coding=utf-8 -*- - -from __future__ import absolute_import, unicode_literals - -import collections -import contextlib -import os -import sys -import sysconfig - -import pkg_resources - -import packaging.markers -import packaging.version -import requirementslib - -from .virtualenv import VirtualEnv - -from ..internals._pip import uninstall, EditableInstaller, WheelInstaller - - -def _is_installation_local(name, venv=None): - """Check whether the distribution is in the current Python installation. - - This is used to distinguish packages seen by a virtual environment. A venv - may be able to see global packages, but we don't want to mess with them. - """ - if venv: - return venv.is_installed(name) - loc = os.path.normcase(pkg_resources.working_set.by_key[name].location) - pre = os.path.normcase(sys.prefix) - return os.path.commonprefix([loc, pre]) == pre - - -def _is_up_to_date(distro, version): - # This is done in strings to avoid type mismatches caused by vendering. - return str(version) == str(packaging.version.parse(distro.version)) - - -GroupCollection = collections.namedtuple("GroupCollection", [ - "uptodate", "outdated", "noremove", "unneeded", -]) - - -def _group_installed_names(packages, venv=None): - """Group locally installed packages based on given specifications. - - `packages` is a name-package mapping that are used as baseline to - determine how the installed package should be grouped. - - `venv` is the virtual environment object of the virtualenv being installed into. - - Returns a 3-tuple of disjoint sets, all containing names of installed - packages: - - * `uptodate`: These match the specifications. - * `outdated`: These installations are specified, but don't match the - specifications in `packages`. - * `unneeded`: These are installed, but not specified in `packages`. - """ - groupcoll = GroupCollection(set(), set(), set(), set()) - - if venv: - working_set = venv.get_working_set() - else: - working_set = pkg_resources.working_set - - for dist in working_set: - name = dist.key - try: - package = packages[name] - except KeyError: - groupcoll.unneeded.add(name) - continue - - r = requirementslib.Requirement.from_pipfile(name, package) - if not r.is_named: - # Always mark non-named. I think pip does something similar? - groupcoll.outdated.add(name) - elif not _is_up_to_date(dist, r.get_version()): - groupcoll.outdated.add(name) - else: - groupcoll.uptodate.add(name) - - return groupcoll - - -@contextlib.contextmanager -def _remove_package(name, venv=None): - if name is None or not _is_installation_local(name, venv=venv): - yield None - return - _uninstall = uninstall - if venv: - _uninstall = venv.uninstall - with _uninstall(name, auto_confirm=True, verbose=False) as uninstaller: - yield uninstaller - - -def _get_packages(lockfile, default, develop): - # Don't need to worry about duplicates because only extras can differ. - # Extras don't matter because they only affect dependencies, and we - # don't install dependencies anyway! - packages = {} - if develop: - packages.update(lockfile.develop._data) - if default: - packages.update(lockfile.default._data) - return packages - - -def _build_paths(venv=None): - """Prepare paths for distlib.wheel.Wheel to install into. - """ - if venv: - paths = venv.paths - else: - paths = sysconfig.get_paths() - return { - "prefix": sys.prefix if not venv else venv.venv_dir.as_posix(), - "data": paths["data"], - "scripts": paths["scripts"], - "headers": paths["include"], - "purelib": paths["purelib"], - "platlib": paths["platlib"], - } - - -PROTECTED_FROM_CLEAN = {"setuptools", "pip", "wheel"} - - -def _clean(names, venv=None): - cleaned = set() - for name in names: - if name in PROTECTED_FROM_CLEAN: - continue - with _remove_package(name, venv=venv) as uninst: - if uninst.paths: - cleaned.add(name) - return cleaned - - -class Synchronizer(object): - """Helper class to install packages from a project's lock file. - """ - def __init__(self, project, default, develop, clean_unneeded, venv=None): - self._root = project.root # Only for repr. - self.packages = _get_packages(project.lockfile, default, develop) - self.sources = project.lockfile.meta.sources._data - self.clean_unneeded = clean_unneeded - if not venv: - self._venv = getattr(project, "venv", None) - else: - self._venv = venv - self.paths = _build_paths(venv=self.venv) - - @property - def venv(self): - if self._venv: - return self._venv - return self.project.venv - - def __repr__(self): - return "<{0} @ {1!r}>".format(type(self).__name__, self._root) - - def sync(self): - if not self.venv: - return self._sync() - with self.venv.activated(): - return self._sync() - - def _sync(self): - groupcoll = _group_installed_names(self.packages, venv=self.venv) - - installed = set() - updated = set() - cleaned = set() - - # TODO: Show a prompt to confirm cleaning. We will need to implement a - # reporter pattern for this as well. - if self.clean_unneeded: - names = _clean(groupcoll.unneeded, venv=self.venv) - cleaned.update(names) - - # TODO: Specify installation order? (pypa/pipenv#2274) - installers = [] - for name, package in self.packages.items(): - r = requirementslib.Requirement.from_pipfile(name, package) - name = r.normalized_name - if name in groupcoll.uptodate: - continue - markers = r.markers - if markers and not packaging.markers.Marker(markers).evaluate(): - continue - r.markers = None - if r.editable: - installer = EditableInstaller(r, venv=self.venv) - else: - installer = WheelInstaller(r, self.sources, self.paths) - try: - installer.prepare() - except Exception as e: - if os.environ.get("PASSA_NO_SUPPRESS_EXCEPTIONS"): - raise - print("failed to prepare {0!r}: {1}".format( - r.as_line(include_hashes=False), e, - )) - else: - installers.append((name, installer)) - - for name, installer in installers: - if name in groupcoll.outdated: - name_to_remove = name - else: - name_to_remove = None - try: - with _remove_package(name_to_remove, venv=self.venv): - installer.install() - except Exception as e: - if os.environ.get("PASSA_NO_SUPPRESS_EXCEPTIONS"): - raise - print("failed to install {0!r}: {1}".format( - r.as_line(include_hashes=False), e, - )) - continue - if name in groupcoll.outdated or name in groupcoll.noremove: - updated.add(name) - else: - installed.add(name) - - return installed, updated, cleaned - - -class Cleaner(object): - """Helper class to clean packages not in a project's lock file. - """ - def __init__(self, project, default, develop, sync=True, verbose=False): - self._root = project.root # Only for repr. - self.packages = _get_packages(project.lockfile, default, develop) - self.sync = sync - self.project = project - - def __repr__(self): - return "<{0} @ {1!r}>".format(type(self).__name__, self._root) - - def print(self, packages): - if not self.sync: - message = "Would clean: {0}" - else: - message = "Cleaned: {0}" - print(message.format(", ".join(sorted(set(packages))))) - - def clean(self): - groupcoll = _group_installed_names(self.packages, venv=self.project.venv) - cleaned = set() - if self.sync: - cleaned = _clean(groupcoll.unneeded, venv=self.project.venv) - else: - return groupcoll.unneeded - return cleaned diff --git a/src/passa/models/virtualenv.py b/src/passa/models/virtualenv.py deleted file mode 100644 index a4f3954..0000000 --- a/src/passa/models/virtualenv.py +++ /dev/null @@ -1,311 +0,0 @@ -# -*- coding=utf-8 -*- - -import base64 -import contextlib -import distlib.scripts -import hashlib -import importlib -import json -import posixpath -import os -import re -import six -import sys -import sysconfig - -import passa.internals._pip -from cached_property import cached_property - -import vistir - - -class VirtualEnv(object): - def __init__(self, venv_dir): - self.recursive_monkey_patch = self.safe_import("recursive_monkey_patch") - self.venv_dir = vistir.compat.Path(venv_dir) - - @classmethod - def from_project_path(cls, path): - path = vistir.compat.Path(path) - if path.name == 'Pipfile': - pipfile_path = path - path = path.parent - else: - pipfile_path = path / 'Pipfile' - pipfile_location = cls.normalize_path(pipfile_path) - venv_path = path / '.venv' - if venv_path.exists(): - if not venv_path.is_dir(): - possible_path = vistir.compat.Path(venv_path.read_text().strip()) - if possible_path.exists(): - return cls(possible_path.as_posix()) - else: - if venv_path.joinpath('lib').exists(): - return cls(venv_path.as_posix()) - sanitized = re.sub(r'[ $`!*@"\\\r\n\t]', "_", path.name)[0:42] - hash_ = hashlib.sha256(pipfile_location.encode()).digest()[:6] - encoded_hash = base64.urlsafe_b64encode(hash_).decode() - hash_fragment = encoded_hash[:8] - venv_name = "{0}-{1}".format(sanitized, hash_fragment) - return cls(cls.get_workon_home().joinpath(venv_name).as_posix()) - - @classmethod - def normalize_path(cls, path): - if not path: - return - if not path.is_absolute(): - try: - path = path.resolve() - except OSError: - path = path.absolute() - path = vistir.path.unicode_path("{0}".format(path)) - if os.name != "nt": - return path - - drive, tail = os.path.splitdrive(path) - # Only match (lower cased) local drives (e.g. 'c:'), not UNC mounts. - if drive.islower() and len(drive) == 2 and drive[1] == ":": - path = "{}{}".format(drive.upper(), tail) - - return vistir.path.unicode_path(path) - - @classmethod - def get_workon_home(cls): - workon_home = os.environ.get("WORKON_HOME") - if not workon_home: - if os.name == "nt": - workon_home = "~/.virtualenvs" - else: - workon_home = os.path.join( - os.environ.get("XDG_DATA_HOME", "~/.local/share"), "virtualenvs" - ) - return vistir.compat.Path(os.path.expandvars(workon_home)).expanduser() - - @classmethod - def filter_sources(cls, requirement, sources): - if not sources or not requirement.index: - return sources - filtered_sources = [ - source for source in sources - if source.get("name") == requirement.index - ] - return filtered_sources or sources - - @classmethod - def safe_import(cls, name): - module = None - if name not in sys.modules: - module = importlib.import_module(name) - else: - module = sys.modules[name] - six.moves.reload_module(module) - return module - - @cached_property - def script_basedir(self): - script_dir = os.path.basename(sysconfig.get_paths()["scripts"]) - return script_dir - - @property - def python(self): - return self.venv_dir.joinpath(self.script_basedir).joinpath("python").as_posix() - - @cached_property - def sys_path(self): - c = vistir.misc.run([self.python, "-c", "import json,sys; print(json.dumps(sys.path))"], - return_object=True, nospin=True) - assert c.returncode == 0, "failed loading virtualenv path" - path = [ - path for path in json.loads(c.out.strip()) - if posixpath.normpath(path).startswith(posixpath.normpath(str(self.venv_dir))) - ] - return path - - @cached_property - def system_paths(self): - paths = {} - sysconfig = self.safe_import("sysconfig") - paths = sysconfig.get_paths() - return paths - - @cached_property - def sys_prefix(self): - c = self.run_py(["-c", "'import sys; print(sys.prefix)'"]) - sys_prefix = vistir.misc.to_text(c.out).strip() - return sys_prefix - - @cached_property - def paths(self): - paths = {} - with vistir.contextmanagers.temp_environ(), vistir.contextmanagers.temp_path(): - os.environ["PYTHONUSERBASE"] = vistir.compat.fs_str(self.venv_dir.as_posix()) - os.environ["PYTHONIOENCODING"] = vistir.compat.fs_str("utf-8") - os.environ["PYTHONDONTWRITEBYTECODE"] = vistir.compat.fs_str("1") - sysconfig = self.safe_import("sysconfig") - scheme, _, _ = sysconfig._get_default_scheme().partition('_') - scheme = "{0}_user".format(scheme) - paths = sysconfig.get_paths(scheme=scheme) - return paths - - @property - def scripts_dir(self): - return self.paths["scripts"] - - @cached_property - def passa_entry(self): - pkg_resources = self.safe_import("pkg_resources") - return pkg_resources.working_set.by_key['passa'].location - - def get_distributions(self): - pkg_resources = self.safe_import("pkg_resources") - return pkg_resources.find_distributions(self.paths["purelib"], only=True) - - def get_working_set(self): - working_set = None - import pkg_resources - passa_entry = self.passa_entry - working_set = pkg_resources.WorkingSet(self.sys_path + [passa_entry]) - return working_set - - @cached_property - def python_version(self): - with self.activated(): - sysconfig = self.safe_import("sysconfig") - py_version = sysconfig.get_python_version() - return py_version - - def get_setup_install_args(self, pkgname, setup_py, develop=False): - headers = vistir.compat.Path(self.sys_prefix) / "include" / "site" - headers = headers / "python{0}".format(self.python_version) / pkgname - install_arg = "install" if not develop else "develop" - return [ - self.python, "-u", "-c", SETUPTOOLS_SHIM % setup_py, install_arg, - "--single-version-externally-managed", "root={0}".format(), - "--install-headers={0}".format(headers.as_posix()), - "--install-purelib={0}".format(self.paths["purelib"]), - "--install-platlib={0}".format(self.paths["platlib"]), - "--install-scripts={0}".format(self.scripts_dir), - "--install-data={0}".format(self.paths["data"]), - ] - - def install(self, req, editable=False, sources=[]): - with self.activated(): - install_options = ["--prefix={0}".format(self.venv_dir),] - passa_pip = self.safe_import("passa.internals._pip") - ireq = req.as_ireq() - if editable: - with vistir.contextmanagers.cd(ireq.setup_py_dir, ireq.setup_py): - c = self.run( - install_options + self.get_setup_install_args( - req.name, develop=editable - ), cwd=ireq.setup_py_dir - ) - return c.returncode - distlib_scripts = self.safe_import("distlib.scripts") - sources = self.filter_sources(req, sources) - hashes = req.hashes - wheel = passa_pip.build_wheel(ireq, sources, hashes) - wheel.install(self.paths, distlib_scripts.ScriptMaker(None, None)) - - @contextlib.contextmanager - def activated(self): - original_path = sys.path - original_prefix = sys.prefix - original_user_base = os.environ.get("PYTHONUSERBASE", None) - original_venv = os.environ.get("VIRTUAL_ENV", None) - passa_path = vistir.compat.Path(__file__).absolute().parent.parent.as_posix() - with vistir.contextmanagers.temp_environ(), vistir.contextmanagers.temp_path(): - os.environ["PYTHONIOENCODING"] = vistir.compat.fs_str("utf-8") - os.environ["PYTHONDONTWRITEBYTECODE"] = vistir.compat.fs_str("1") - activate_this = os.path.join(self.scripts_dir, "activate_this.py") - with open(activate_this, "r") as f: - code = compile(f.read(), activate_this, "exec") - exec(code, dict(__file__=activate_this)) - os.environ["PYTHONUSERBASE"] = vistir.compat.fs_str(self.venv_dir.as_posix()) - os.environ["VIRTUAL_ENV"] = vistir.compat.fs_str(self.venv_dir.as_posix()) - sys.path = self.sys_path - sys.prefix = self.venv_dir - site = self.safe_import("site") - site.addsitedir(passa_path) - sys.modules["recursive_monkey_patch"] = self.recursive_monkey_patch - pkg_resources = self.safe_import("pkg_resources") - try: - yield - finally: - print("Deactivating virtualenv...") - del os.environ["VIRTUAL_ENV"] - del os.environ["PYTHONUSERBASE"] - if original_user_base: - os.environ["PYTHONUSERBASE"] = original_user_base - if original_venv: - os.environ["VIRTUAL_ENV"] = original_venv - sys.path = original_path - sys.prefix = original_prefix - six.moves.reload_module(pkg_resources) - - def run(self, cmd, cwd=os.curdir): - c = None - with self.activated(): - script = vistir.cmdparse.Script.parse(cmd) - c = vistir.misc.run(script._parts, return_object=True, nospin=True, cwd=cwd) - return c - - def run_py(self, cmd, cwd=os.curdir): - c = None - if isinstance(cmd, six.string_types): - script = vistir.cmdparse.Script.parse("{0} {1}".format(self.python, cmd)) - else: - script = vistir.cmdparse.Script.parse([self.python,] + list(cmd)) - with self.activated(): - c = vistir.misc.run(script._parts, return_object=True, nospin=True, cwd=cwd) - return c - - def is_installed(self, pkgname): - return any(d for d in self.get_distributions() if d.project_name == pkgname) - - def get_monkeypatched_pathset(self): - from pip_shims.shims import req_install - req_uninstall_name = "{0}.req_uninstall".format(req_install.__package__) - req_uninstall = self.safe_import(req_uninstall_name) - self.recursive_monkey_patch.monkey_patch( - PatchedUninstaller, req_uninstall.UninstallPathSet - ) - return req_uninstall.UninstallPathSet - - @contextlib.contextmanager - def uninstall(self, pkgname, *args, **kwargs): - auto_confirm = kwargs.pop("auto_confirm", True) - verbose = kwargs.pop("verbose", False) - with self.activated(): - pathset_base = self.get_monkeypatched_pathset() - dist = next( - iter(filter(lambda d: d.project_name == pkgname, self.get_working_set())), - None - ) - pathset = pathset_base.from_dist(dist) - if pathset is not None: - pathset.remove(auto_confirm=auto_confirm, verbose=verbose) - try: - yield pathset - except Exception as e: - if pathset is not None: - pathset.rollback() - else: - if pathset is not None: - pathset.commit() - if pathset is None: - return - - -SETUPTOOLS_SHIM = ( - "import setuptools, tokenize;__file__=%r;" - "f=getattr(tokenize, 'open', open)(__file__);" - "code=f.read().replace('\\r\\n', '\\n');" - "f.close();" - "exec(compile(code, __file__, 'exec'))" -) - - -class PatchedUninstaller(object): - def _permitted(self, path): - return True diff --git a/src/passa/operations/sync.py b/src/passa/operations/sync.py deleted file mode 100644 index 45502a4..0000000 --- a/src/passa/operations/sync.py +++ /dev/null @@ -1,23 +0,0 @@ -# -*- coding=utf-8 -*- - -from __future__ import absolute_import, print_function, unicode_literals - - -def sync(syncer): - print("Starting synchronization") - installed, updated, cleaned = syncer.sync() - if cleaned: - print("Uninstalled: {}".format(", ".join(sorted(cleaned)))) - if installed: - print("Installed: {}".format(", ".join(sorted(installed)))) - if updated: - print("Updated: {}".format(", ".join(sorted(updated)))) - return True - - -def clean(cleaner): - print("Cleaning...") - cleaned = cleaner.clean() - if cleaned: - cleaner.print(cleaned) - return True From 432c6a5e038bb204fd2e216670f08500b7b8666c Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 21 Sep 2018 01:13:15 -0400 Subject: [PATCH 15/32] Fix package task Signed-off-by: Dan Ryan --- tasks/package.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tasks/package.py b/tasks/package.py index f229e3a..5ae379b 100644 --- a/tasks/package.py +++ b/tasks/package.py @@ -5,7 +5,8 @@ import distlib.scripts import distlib.wheel import invoke -import passa.internals._pip +import packagebuilder +import passa.models.caches import plette import requirementslib @@ -75,8 +76,8 @@ def pack(ctx, remove_lib=True): package.pop('editable', None) # Don't install things as editable. package.pop('markers', None) # Always install everything. r = requirementslib.Requirement.from_pipfile(name, package) - wheel = passa.internals._pip.build_wheel( - r.as_ireq(), sources, r.hashes or None, + wheel = packagebuilder._pip.build_wheel( + r.as_ireq(), sources, r.hashes or None, cache_dir=passa.models.caches.CACHE_DIR ) wheel.install(paths, maker, lib_only=True) From 27ca19d1be25dad870071e622b12b62a98a66234 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 22 Sep 2018 19:00:36 -0400 Subject: [PATCH 16/32] Update lockfile and setup.cfg Signed-off-by: Dan Ryan --- Pipfile | 2 +- Pipfile.lock | 188 ++++++++++++++++++++++++++++++++++----------------- setup.cfg | 3 +- 3 files changed, 128 insertions(+), 65 deletions(-) diff --git a/Pipfile b/Pipfile index 25ed7ee..51464ab 100644 --- a/Pipfile +++ b/Pipfile @@ -1,5 +1,5 @@ [packages] -passa = { editable = true, path = '.' } +passa = { editable = true, path = '.', extras = ['virtualenv'] } [dev-packages] black = '*' diff --git a/Pipfile.lock b/Pipfile.lock index c83e057..662eeb7 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "ff353680e286436d85c9b1b00ccb7ee257cb2922a1eec0becf9713817605d92b" + "sha256": "994d50f9fd0acc91cf216cf0ffc1233ebbfd0a411b57320f44a0f4487943e546" }, "pipfile-spec": 6, "requires": {}, @@ -19,7 +19,7 @@ "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92", "sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '3.6'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '3.6'", "version": "==1.4.3" }, "attrs": { @@ -27,9 +27,15 @@ "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '3.6'", "version": "==18.2.0" }, + "backports-functools-lru-cache": { + "hashes": [ + "sha256:f0b0e4eba956de51238e17573b7087e852dfe9854afd2e9c873f73fc0ca0a6dd" + ], + "markers": "python_version <= '2.7' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.5" + }, "backports-shutil-get-terminal-size": { "hashes": [ "sha256:0975ba55054c15e346944b38956a4c9cbee9009391e41b86c68990effb8c1f64" @@ -44,6 +50,14 @@ "markers": "python_version < '3.3' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.0.post1" }, + "cached-property": { + "hashes": [ + "sha256:3a026f1a54135677e7da5ce819b0c690f156f37976f3e30c5430740725203d7f", + "sha256:9217a59f14a5682da7c4b8829deadbfc194ac22e9908ccf7c8820234e80a1504" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.5.1" + }, "cerberus": { "hashes": [ "sha256:f5c2e048fb15ecb3c088d192164316093fcfa602a74b3386eefb2983aa7e800a" @@ -56,7 +70,6 @@ "sha256:376690d6f16d32f9d1fe8932551d80b23e9d393a8578c5633a2ed39a64861638", "sha256:456048c7e371c089d0a77a5212fb37a2c2dce1e24146e3b7e0261736aaeaa22a" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2018.8.24" }, "chardet": { @@ -64,14 +77,13 @@ "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==3.0.4" }, "distlib": { "hashes": [ "sha256:cd502c66fc27c535bab62dc4f482e403e2369c2c05281a79cc2d4e2f42a87f20" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.2.7" }, "enum34": { @@ -89,7 +101,7 @@ "sha256:3bb3de3582cb27071cfb514f00ed784dc444b7f96dc21e140de65fe00585c95e", "sha256:41d5b64e70507d0c3ca742d68010a76060eea8a3d863e9b5130ab11a4a91aa0e" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.0.1" }, "idna": { @@ -97,7 +109,6 @@ "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.7" }, "importlib": { @@ -107,6 +118,14 @@ "markers": "python_version < '2.7' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.0.4" }, + "installer": { + "hashes": [ + "sha256:4ad3f14753d506d9ce646ac10320a2b82395208109f2356ed9e7a234976d3b4b", + "sha256:b849d2d876ef71162cfa6fc92204120a5cd215cad373b83fddcab401490d5fbc" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.1.0" + }, "modutil": { "hashes": [ "sha256:2c85c1666649e92e56de17c00e1e831313602d9b55e8661d39c01e39003b45f7", @@ -115,17 +134,34 @@ "markers": "python_version >= '2.6' and python_version >= '3.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.0.0" }, + "mork": { + "hashes": [ + "sha256:988dd3d768369e9c88bdade3d9b90ed3bfc91cb5eb229a0da4db9b119b0a5d7a", + "sha256:9cf945de7f69ed7ace8062509c5613c9484bdc2a5ac14caed6bba71f31f459c5" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.1.0" + }, + "packagebuilder": { + "hashes": [ + "sha256:1e85c4e0e994322996b93cd6685c12834d30f3558889154f8e3de8fb1f3fd1e7", + "sha256:dc525d06ecd102db23ab421b879d7d27021d784ff933e33e8c411a53af5c9dbe" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.1.0" + }, "packaging": { "hashes": [ "sha256:e9215d2d2535d3ae866c3d6efc77d5b24a0192cce0ff20e42896cc0664f889c0", "sha256:f019b770dd64e585a99714f1fd5e01c7a8f11b45635aa953fd41c689a657375b" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==17.1" }, "passa": { "editable": true, - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "extras": [ + "virtualenv" + ], "path": "." }, "pathlib2": { @@ -133,7 +169,7 @@ "sha256:8eb170f8d0d61825e09a95b38be068299ddeda82f35e96c3301a8a5e7604cb83", "sha256:d1aa2a11ba7b8f7b21ab852b1fb5afb277e1bb99d5dfc663380b5015c0d80c5a" ], - "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.6' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.3.2" }, "pip-shims": { @@ -157,18 +193,24 @@ }, "pyparsing": { "hashes": [ - "sha256:0832bcf47acd283788593e7a0f542407bd9550a55a8a8435214a1960e04bcb04", - "sha256:fee43f17a9c4087e7ed1605bd6df994c6173c1e977d7ade7b651292fab2bd010" + "sha256:905d8090c335314568b5faee0025b1829f27bb974604a5762a6cdef3a7dfc3b7", + "sha256:f493ee323be1e94929416b3585eefcc04943115cecbaaa35a8c86d1a2368af19" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.2.0" + "version": "==2.2.1" + }, + "recursive-monkey-patch": { + "hashes": [ + "sha256:546739fea5be2ea9f98b5ec44fafeb697b5cf9fdcda64a03422582ab03ee24c4", + "sha256:98922554e77f2e2c85a4f5d873a0f52efdc1b553f32444bd6c788b2ff583bf3e" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.4.0" }, "requests": { "hashes": [ "sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1", "sha256:ec22d826a36ed72a7358ff3fe56cbd4ba69dd7a6718ffd450ff0e9df7a47ce6a" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.19.1" }, "requirementslib": { @@ -176,7 +218,7 @@ "sha256:90151d8963f814e17190e067b60e92fb35fd1bc46c99f8dba3d7b0d93a3dd958", "sha256:c3aeaa4e0b80843ba65a68878293e07ea52a8d0706dbba86b02dad6cd20ef2dd" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.6" }, "resolvelib": { @@ -201,7 +243,7 @@ "sha256:c9009c527929f6e25604aec39b0a43c3f831d2947d89d6caaab22f057b7055c8", "sha256:f5c71e29b4e2af7ccdc03a020c626ede51da471173b4a6ad1e904f2b2e04b4bd" ], - "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version < '3.6' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.9.0" }, "six": { @@ -209,7 +251,6 @@ "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" ], - "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.11.0" }, "toml": { @@ -233,7 +274,7 @@ "sha256:57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4", "sha256:a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a" ], - "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "(python_version >= '2.7' and python_version < '2.8') or (python_version >= '3.4' and python_version < '3.5') and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5'", "version": "==3.6.6" }, "urllib3": { @@ -241,9 +282,16 @@ "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf", "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5" ], - "markers": "python_version < '4' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.23" }, + "virtualenv": { + "hashes": [ + "sha256:2ce32cd126117ce2c539f0134eb89de91a8413a29baac49cbab3eb50e2026669", + "sha256:ca07b4c0b54e14a91af9f34d0919790b016923d157afda5efdde55c96718f752" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==16.0.0" + }, "vistir": { "extras": [ "spinner" @@ -260,7 +308,6 @@ "sha256:0a2e54558a0628f2145d2fc822137e322412115173e8a2ddbe1c9024338ae83c", "sha256:80044e51ec5bbf6c894ba0bc48d26a8c20a9ba629f4ca19ea26ecfcf87685f5f" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.31.1" }, "yaspin": { @@ -278,7 +325,6 @@ "sha256:674bb3bab080f598371f4443c5008cbfeb1a5e622dd312395d2d82af2c54c456", "sha256:b63b1f4dc77c074d386752ec4a8a7517600f6c0db8cd42980cae17ab7b3275d7" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.7.11" }, "apipkg": { @@ -294,7 +340,7 @@ "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92", "sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '3.6'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '3.6'", "version": "==1.4.3" }, "argparse": { @@ -302,7 +348,6 @@ "sha256:62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4", "sha256:c31647edb69fd3d465a847ea3157d37bed1f95f19760b11a47aa91c04b666314" ], - "markers": "python_version == '2.6'", "version": "==1.4.0" }, "arpeggio": { @@ -310,7 +355,6 @@ "sha256:a5258b84f76661d558492fa87e42db634df143685a0e51802d59cae7daad8732", "sha256:dc5c0541e7cc2c6033dc0338133436abfac53655624784736e9bc8bd35e56583" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.9.0" }, "atomicwrites": { @@ -326,7 +370,6 @@ "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '3.6'", "version": "==18.2.0" }, "babel": { @@ -334,9 +377,15 @@ "sha256:6778d85147d5d85345c14a26aada5e478ab04e39b078b0745ee6870c2b5cf669", "sha256:8cba50f48c529ca3fa18cf81fa9403be176d374ac4d60738b839122dfaaa3d23" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.6.0" }, + "backports-functools-lru-cache": { + "hashes": [ + "sha256:f0b0e4eba956de51238e17573b7087e852dfe9854afd2e9c873f73fc0ca0a6dd" + ], + "markers": "python_version <= '2.7' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.5" + }, "backports-shutil-get-terminal-size": { "hashes": [ "sha256:0975ba55054c15e346944b38956a4c9cbee9009391e41b86c68990effb8c1f64" @@ -359,6 +408,14 @@ "markers": "python_version >= '3.6'", "version": "==18.6b4" }, + "cached-property": { + "hashes": [ + "sha256:3a026f1a54135677e7da5ce819b0c690f156f37976f3e30c5430740725203d7f", + "sha256:9217a59f14a5682da7c4b8829deadbfc194ac22e9908ccf7c8820234e80a1504" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.5.1" + }, "cerberus": { "hashes": [ "sha256:f5c2e048fb15ecb3c088d192164316093fcfa602a74b3386eefb2983aa7e800a" @@ -371,7 +428,6 @@ "sha256:376690d6f16d32f9d1fe8932551d80b23e9d393a8578c5633a2ed39a64861638", "sha256:456048c7e371c089d0a77a5212fb37a2c2dce1e24146e3b7e0261736aaeaa22a" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2018.8.24" }, "chardet": { @@ -379,7 +435,6 @@ "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==3.0.4" }, "click": { @@ -394,7 +449,7 @@ "sha256:463f8483208e921368c9f306094eb6f725c6ca42b0f97e313cb5d5512459feda", "sha256:48eb22f4f8461b1df5734a074b57042430fb06e1d61bd1e11b078c0fe6d7a1f1" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and sys_platform == 'win32'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and sys_platform == 'win32' or sys_platform == 'win32'", "version": "==0.3.9" }, "coverage": { @@ -438,7 +493,7 @@ "hashes": [ "sha256:cd502c66fc27c535bab62dc4f482e403e2369c2c05281a79cc2d4e2f42a87f20" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.2.7" }, "docutils": { @@ -447,7 +502,6 @@ "sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274", "sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.14" }, "enum34": { @@ -473,7 +527,7 @@ "sha256:3bb3de3582cb27071cfb514f00ed784dc444b7f96dc21e140de65fe00585c95e", "sha256:41d5b64e70507d0c3ca742d68010a76060eea8a3d863e9b5130ab11a4a91aa0e" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.0.1" }, "funcsigs": { @@ -489,7 +543,6 @@ "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.7" }, "imagesize": { @@ -497,7 +550,6 @@ "sha256:3f349de3eb99145973fefb7dbe38554414e5c30abd0c8e4b970a7c9d09f3a1d8", "sha256:f3832918bc3c66617f92e35f5d70729187676313caa60c187eb0f28b8fe5e3b5" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.0" }, "importlib": { @@ -514,6 +566,14 @@ ], "version": "==17.5.0" }, + "installer": { + "hashes": [ + "sha256:4ad3f14753d506d9ce646ac10320a2b82395208109f2356ed9e7a234976d3b4b", + "sha256:b849d2d876ef71162cfa6fc92204120a5cd215cad373b83fddcab401490d5fbc" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.1.0" + }, "invoke": { "hashes": [ "sha256:4f4de934b15c2276caa4fbc5a3b8a61c0eb0b234f2be1780d2b793321995c2d6", @@ -559,12 +619,19 @@ "markers": "python_version < '3.0' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1" }, + "packagebuilder": { + "hashes": [ + "sha256:1e85c4e0e994322996b93cd6685c12834d30f3558889154f8e3de8fb1f3fd1e7", + "sha256:dc525d06ecd102db23ab421b879d7d27021d784ff933e33e8c411a53af5c9dbe" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.1.0" + }, "packaging": { "hashes": [ "sha256:e9215d2d2535d3ae866c3d6efc77d5b24a0192cce0ff20e42896cc0664f889c0", "sha256:f019b770dd64e585a99714f1fd5e01c7a8f11b45635aa953fd41c689a657375b" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==17.1" }, "parver": { @@ -572,7 +639,6 @@ "sha256:ac4afff688d19d5e1876bb68d4bccc1a1b6a5cc8bd6a646939a14d366695ba15", "sha256:f025fba8f88a9c776971df6d62b6cf7f37d1108f84c163bda91e157d7d527075" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.1.1" }, "passa": { @@ -580,7 +646,6 @@ "extras": [ "tests" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "path": "." }, "pathlib2": { @@ -588,7 +653,7 @@ "sha256:8eb170f8d0d61825e09a95b38be068299ddeda82f35e96c3301a8a5e7604cb83", "sha256:d1aa2a11ba7b8f7b21ab852b1fb5afb277e1bb99d5dfc663380b5015c0d80c5a" ], - "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.6' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.3.2" }, "pip-shims": { @@ -638,24 +703,22 @@ "sha256:78f3f434bcc5d6ee09020f92ba487f95ba50f1e3ef83ae96b9d5ffa1bab25c5d", "sha256:dbae1046def0efb574852fab9e90209b23f556367b5a320c0bcb871c77c3e8cc" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.2.0" }, "pyparsing": { "hashes": [ - "sha256:0832bcf47acd283788593e7a0f542407bd9550a55a8a8435214a1960e04bcb04", - "sha256:fee43f17a9c4087e7ed1605bd6df994c6173c1e977d7ade7b651292fab2bd010" + "sha256:905d8090c335314568b5faee0025b1829f27bb974604a5762a6cdef3a7dfc3b7", + "sha256:f493ee323be1e94929416b3585eefcc04943115cecbaaa35a8c86d1a2368af19" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.2.0" + "version": "==2.2.1" }, "pytest": { "hashes": [ - "sha256:453cbbbe5ce6db38717d282b758b917de84802af4288910c12442984bde7b823", - "sha256:a8a07f84e680482eb51e244370aaf2caa6301ef265f37c2bdefb3dd3b663f99d" + "sha256:0a72d8a9f559c006ba153e0c9b4838efd7b656cf1f993747ba7128770d6eb12c", + "sha256:95529588ff4e85114a0b0ad8e9cf0131ca47d46b28230e25366c5aba66b1d854" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==3.8.0" + "version": "==3.8.1" }, "pytest-cov": { "hashes": [ @@ -694,7 +757,6 @@ "sha256:a061aa0a9e06881eb8b3b2b43f05b9439d6583c206d0a6c340ff72a7b6669053", "sha256:ffb9ef1de172603304d9d2819af6f5ece76f2e85ec10692a524dd876e72bf277" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2018.5" }, "requests": { @@ -702,7 +764,6 @@ "sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1", "sha256:ec22d826a36ed72a7358ff3fe56cbd4ba69dd7a6718ffd450ff0e9df7a47ce6a" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.19.1" }, "requests-toolbelt": { @@ -717,7 +778,7 @@ "sha256:90151d8963f814e17190e067b60e92fb35fd1bc46c99f8dba3d7b0d93a3dd958", "sha256:c3aeaa4e0b80843ba65a68878293e07ea52a8d0706dbba86b02dad6cd20ef2dd" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.6" }, "resolvelib": { @@ -742,7 +803,7 @@ "sha256:c9009c527929f6e25604aec39b0a43c3f831d2947d89d6caaab22f057b7055c8", "sha256:f5c71e29b4e2af7ccdc03a020c626ede51da471173b4a6ad1e904f2b2e04b4bd" ], - "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version < '3.6' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.9.0" }, "six": { @@ -750,7 +811,6 @@ "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" ], - "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.11.0" }, "snowballstemmer": { @@ -758,16 +818,14 @@ "sha256:919f26a68b2c17a7634da993d91339e288964f93c274f1343e3bbbe2096e1128", "sha256:9f3bcd3c401c3e862ec0ebe6d2c069ebc012ce142cce209c098ccb5b09136e89" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.2.1" }, "sphinx": { "hashes": [ - "sha256:95acd6648902333647a0e0564abdb28a74b0a76d2333148aa35e5ed1f56d3c4b", - "sha256:c091dbdd5cc5aac6eb95d591a819fd18bccec90ffb048ec465b165a48b839b45" + "sha256:652eb8c566f18823a022bb4b6dbc868d366df332a11a0226b5bc3a798a479f17", + "sha256:d222626d8356de702431e813a05c68a35967e3d66c6cd1c2c89539bb179a7464" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.8.0" + "version": "==1.8.1" }, "sphinx-rtd-theme": { "hashes": [ @@ -781,7 +839,6 @@ "sha256:68ca7ff70785cbe1e7bccc71a48b5b6d965d79ca50629606c7861a21b206d9dd", "sha256:9de47f375baf1ea07cdb3436ff39d7a9c76042c10a769c52353ec46e4e8fc3b9" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.0" }, "toml": { @@ -811,7 +868,6 @@ "sha256:18f1818ce951aeb9ea162ae1098b43f583f7d057b34d706f66939353d1208889", "sha256:df02c0650160986bac0218bb07952245fc6960d23654648b5d5526ad5a4128c9" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1'", "version": "==4.26.0" }, "twine": { @@ -827,7 +883,7 @@ "sha256:57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4", "sha256:a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a" ], - "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "(python_version >= '2.7' and python_version < '2.8') or (python_version >= '3.4' and python_version < '3.5') and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5'", "version": "==3.6.6" }, "urllib3": { @@ -835,9 +891,16 @@ "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf", "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5" ], - "markers": "python_version < '4' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.23" }, + "virtualenv": { + "hashes": [ + "sha256:2ce32cd126117ce2c539f0134eb89de91a8413a29baac49cbab3eb50e2026669", + "sha256:ca07b4c0b54e14a91af9f34d0919790b016923d157afda5efdde55c96718f752" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==16.0.0" + }, "vistir": { "extras": [ "spinner" @@ -854,7 +917,6 @@ "sha256:0a2e54558a0628f2145d2fc822137e322412115173e8a2ddbe1c9024338ae83c", "sha256:80044e51ec5bbf6c894ba0bc48d26a8c20a9ba629f4ca19ea26ecfcf87685f5f" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.31.1" }, "yaspin": { diff --git a/setup.cfg b/setup.cfg index 8d9c3b7..e6879b5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,6 +37,7 @@ python_requires = >=2.7,!=3.0,!=3.1,!=3.2,!=3.3 setup_requires = setuptools>=36.2.2 install_requires = appdirs + cached-property distlib installer packaging @@ -45,6 +46,7 @@ install_requires = plette[validation]>=0.2.2 requests resolvelib>=0.2.1,!=1.0.0.dev0 + backports.functools_lru_cache; python_version<='2.7' requirementslib>=1.1.1 six virtualenv @@ -57,7 +59,6 @@ pack = virtualenv = mork tests = - cached-property pytest-xdist pytest-timeout pytest-cov From 1c708f2b9753d04d62f54f13c4cf6fdbe5d6a6ed Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 22 Sep 2018 19:03:14 -0400 Subject: [PATCH 17/32] Add LRU cache for cacheable function calls Signed-off-by: Dan Ryan --- src/passa/internals/markers.py | 19 ++++++++++++++++--- src/passa/internals/specifiers.py | 13 +++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/passa/internals/markers.py b/src/passa/internals/markers.py index 95efab9..c2b7fdd 100644 --- a/src/passa/internals/markers.py +++ b/src/passa/internals/markers.py @@ -4,6 +4,17 @@ from packaging.markers import Marker +try: + from functools import lru_cache +except ImportError: + from backports.functools_lru_cache import lru_cache + + +def _ensure_marker(marker): + if not isinstance(marker, Marker): + return Marker(str(marker)) + return marker + def _strip_extra(elements): """Remove the "extra == ..." operands from the list. @@ -49,7 +60,7 @@ def get_without_extra(marker): # meet the demands of a pep... -d if not marker: return None - marker = Marker(str(marker)) + marker = _ensure_marker(marker) elements = marker._markers _strip_extra(elements) if elements: @@ -68,6 +79,7 @@ def _markers_collect_extras(markers, collection): _markers_collect_extras(el, collection) +@lru_cache(maxsize=128) def get_contained_extras(marker): """Collect "extra == ..." operands from a marker. @@ -75,8 +87,8 @@ def get_contained_extras(marker): """ if not marker: return set() - marker = Marker(str(marker)) extras = set() + marker = _ensure_marker(marker) _markers_collect_extras(marker._markers, extras) return extras @@ -92,10 +104,11 @@ def _markers_contains_extra(markers): return False +@lru_cache(maxsize=128) def contains_extra(marker): """Check whehter a marker contains an "extra == ..." operand. """ if not marker: return False - marker = Marker(str(marker)) + marker = _ensure_marker(marker) return _markers_contains_extra(marker._markers) diff --git a/src/passa/internals/specifiers.py b/src/passa/internals/specifiers.py index 75afb6a..bf06ce5 100644 --- a/src/passa/internals/specifiers.py +++ b/src/passa/internals/specifiers.py @@ -9,10 +9,18 @@ from vistir.misc import dedup +try: + from functools import lru_cache +except ImportError: + from backports.functools_lru_cache import lru_cache + + +@lru_cache(maxsize=128) def _tuplize_version(version): return tuple(int(x) for x in version.split(".")) +@lru_cache(maxsize=128) def _format_version(version): return ".".join(str(i) for i in version) @@ -21,6 +29,7 @@ def _format_version(version): REPLACE_RANGES = {">": ">=", "<=": "<"} +@lru_cache(maxsize=128) def _format_pyspec(specifier): if isinstance(specifier, str): if not any(op in specifier for op in Specifier._operators.keys()): @@ -42,6 +51,7 @@ def _format_pyspec(specifier): return specifier +@lru_cache(maxsize=128) def _get_specs(specset): if isinstance(specset, Specifier): specset = str(specset) @@ -53,6 +63,7 @@ def _get_specs(specset): ] +@lru_cache(maxsize=128) def _group_by_op(specs): specs = [_get_specs(x) for x in list(specs)] flattened = [(op, version) for spec in specs for op, version in spec] @@ -61,6 +72,7 @@ def _group_by_op(specs): return grouping +@lru_cache(maxsize=128) def cleanup_pyspecs(specs, joiner="or"): specs = {_format_pyspec(spec) for spec in specs} # for != operator we want to group by version @@ -113,6 +125,7 @@ def cleanup_pyspecs(specs, joiner="or"): return results +@lru_cache(maxsize=128) def pyspec_from_markers(marker): if marker._markers[0][0] != 'python_version': return From b41621314b4f502db95f95ef6269fba3670855a5 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 22 Sep 2018 19:06:40 -0400 Subject: [PATCH 18/32] Drop markers from editable requirements - Also clean up python specifiers Signed-off-by: Dan Ryan --- src/passa/internals/specifiers.py | 23 ++++++++++++++--------- src/passa/models/lockers.py | 3 +++ src/passa/models/metadata.py | 5 +++-- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/passa/internals/specifiers.py b/src/passa/internals/specifiers.py index bf06ce5..fd9aec8 100644 --- a/src/passa/internals/specifiers.py +++ b/src/passa/internals/specifiers.py @@ -5,7 +5,9 @@ import itertools import operator -from packaging.specifiers import SpecifierSet, Specifier +from packaging.markers import Marker +from packaging.specifiers import Specifier, SpecifierSet + from vistir.misc import dedup @@ -17,7 +19,7 @@ @lru_cache(maxsize=128) def _tuplize_version(version): - return tuple(int(x) for x in version.split(".")) + return tuple(int(x) for x in filter(lambda i: i != "*", version.split("."))) @lru_cache(maxsize=128) @@ -35,8 +37,9 @@ def _format_pyspec(specifier): if not any(op in specifier for op in Specifier._operators.keys()): specifier = "=={0}".format(specifier) specifier = Specifier(specifier) - if specifier.operator == "==" and specifier.version.endswith(".*"): - specifier = Specifier("=={0}".format(specifier.version[:-2])) + version = specifier.version.replace(".*", "") + if ".*" in specifier.version: + specifier = Specifier("{0}{1}".format(specifier.operator, version)) try: op = REPLACE_RANGES[specifier.operator] except KeyError: @@ -53,14 +56,16 @@ def _format_pyspec(specifier): @lru_cache(maxsize=128) def _get_specs(specset): + if specset is None: + return if isinstance(specset, Specifier): specset = str(specset) if isinstance(specset, str): specset = SpecifierSet(specset.replace(".*", "")) - return [ - (spec._spec[0], _tuplize_version(spec._spec[1])) - for spec in getattr(specset, "_specs", []) - ] + result = [] + for spec in set(specset): + result.append((spec.operator, _tuplize_version(spec.version))) + return result @lru_cache(maxsize=128) @@ -78,7 +83,7 @@ def cleanup_pyspecs(specs, joiner="or"): # for != operator we want to group by version # if all are consecutive, join as a list results = set() - for op, versions in _group_by_op(specs): + for op, versions in _group_by_op(tuple(specs)): versions = [version[1] for version in versions] versions = sorted(dedup(versions)) # if we are doing an or operation, we need to use the min for >= diff --git a/src/passa/models/lockers.py b/src/passa/models/lockers.py index c25ca60..53f1cab 100644 --- a/src/passa/models/lockers.py +++ b/src/passa/models/lockers.py @@ -71,6 +71,9 @@ def _collect_derived_entries(state, traces, identifiers): extras[name].extend(requirement.extras) except KeyError: extras[name] = list(requirement.extras) + if requirement.editable and requirement.markers: + requirement.markers = set() + requirement.req.req.markers = set() entries[name] = next(iter(requirement.as_pipfile().values())) for name, ext in extras.items(): entries[name]["extras"] = ext diff --git a/src/passa/models/metadata.py b/src/passa/models/metadata.py index a949f1e..45fb223 100644 --- a/src/passa/models/metadata.py +++ b/src/passa/models/metadata.py @@ -91,12 +91,13 @@ def _build_metasets(dependencies, pythons, key, trace, all_metasets): return all_parent_metasets.append((parent, parent_metasets)) - metaset_iters = [] + metasets = set() for parent, parent_metasets in all_parent_metasets: r = dependencies[parent][key] python = pythons[key] + markers = None if r.editable else get_without_extra(r.markers) metaset = ( - get_without_extra(r.markers), + markers, packaging.specifiers.SpecifierSet(python), ) metaset_iters.append( From cbddbeed0d528a0a7dfea1c00b2dbc039f58993f Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 22 Sep 2018 19:09:00 -0400 Subject: [PATCH 19/32] Add PySpec object to handle comparison and consolidation - Also add magic methods to MetaSet object - Super fast and efficient now due to set comparisons - Old method had sometimes thousands of the same pyspecs - This consolidates as it builds Signed-off-by: Dan Ryan --- src/passa/internals/specifiers.py | 163 +++++++++++++++++++++++++++++- src/passa/models/metadata.py | 56 ++++++---- 2 files changed, 197 insertions(+), 22 deletions(-) diff --git a/src/passa/internals/specifiers.py b/src/passa/internals/specifiers.py index fd9aec8..ae2025b 100644 --- a/src/passa/internals/specifiers.py +++ b/src/passa/internals/specifiers.py @@ -2,9 +2,11 @@ from __future__ import absolute_import, unicode_literals +import collections import itertools import operator +from cached_property import cached_property from packaging.markers import Marker from packaging.specifiers import Specifier, SpecifierSet @@ -44,7 +46,6 @@ def _format_pyspec(specifier): op = REPLACE_RANGES[specifier.operator] except KeyError: return specifier - version = specifier.version.replace(".*", "") curr_tuple = _tuplize_version(version) try: next_tuple = (curr_tuple[0], curr_tuple[1] + 1) @@ -152,3 +153,163 @@ def pyspec_from_markers(marker): if specset: return specset return None + + +class PySpecs(collections.Set): + def __init__(self, specs=None): + if not specs: + specs = SpecifierSet() + self.specifierset = specs + self.previous_specifierset = None + self.cleaned_tuples = set() + self.markers = None + self.clean() + + def __key(self): + return tuple(sorted(self.specifierset, key=operator.attrgetter("_spec"))) + + def __contains__(self, other): + # The current specifierset fully has every value in the supplied specifierset + if not other.as_set - self.as_set: + return True + return False + + def __eq__(self, other): + return self.__key() == other.__key() + + def __hash__(self): + return hash(self.__key()) + + def __repr__(self): + return u"PySpecs({0!r})".format(str(self.specifierset)) + + def __len__(self): + return len(self.cleaned_tuples) + + def __iter__(self): + for version in self.as_string_set: + yield version + return + + def clean(self): + if len(set(self.specifierset)) == 1: + spec = next(iter(spec for spec in self.specifierset), None) + if spec: + self.cleaned_tuples.add((spec.operator, spec.version)) + else: + self.cleaned_tuples = cleanup_pyspecs(self.specifierset) + self.specifierset = self.as_specset + + def add(self, other): + new_pyspec = PySpecs(self.specifierset) + new_pyspec.specifierset &= other.specifierset + return new_pyspec + + @cached_property + @lru_cache(maxsize=128) + def as_specset(self): + specs = set() + for spec in self.cleaned_tuples: + op, value = spec + if op in ('in', 'not in'): + new_op = '!=' if op == 'not in' else '==' + for val in value.split(","): + specs.add(Specifier("{0}{1}".format(new_op, val))) + else: + specs.add(Specifier("{0}{1}".format(op, value))) + specifierset = SpecifierSet() + specifierset._specs = frozenset(specs) + return specifierset + + @cached_property + @lru_cache(maxsize=128) + def as_set(self): + return set(self.specifierset) + + @cached_property + @lru_cache(maxsize=128) + def as_string_set(self): + returnval = set() + if len(self.cleaned_tuples) == 1: + val = next(iter(spec for spec in self.cleaned_tuples), None) + if val: + returnval.add("python_version {0[0]} '{0[1]}'".format(val)) + return returnval + return set( + "python_version {0[0]} '{0[1]}'".format(s) + for s in sorted(self.cleaned_tuples) + ) + + @cached_property + @lru_cache(maxsize=128) + def marker_set(self): + markerset = {Marker(spec) for spec in self.as_string_set} + return markerset + + @cached_property + @lru_cache(maxsize=128) + def marker_string(self): + marker_string = " and ".join(sorted(str(m) for m in self.as_string_set)) + if not marker_string: + return "" + return marker_string + + @cached_property + @lru_cache(maxsize=128) + def as_markers(self): + print("generating marker using string: %s" % self.marker_string) + if not self.marker_string: + return "" + marker = Marker(self.marker_string) + return marker + + @lru_cache(maxsize=128) + def __str__(self): + string_repr = u"{0}".format(str(self.marker_string)) + print("converting to string: %s" % string_repr) + return string_repr + + def __bool__(self): + return bool(self.specifierset) + + def __nonzero__(self): # Python 2. + return self.__bool__() + + @lru_cache(maxsize=128) + def __or__(self, specset): + if not isinstance(specset, PySpecs): + specset = PySpecs(specset) + if str(self) == str(specset): + return self + combined_set = self.as_set | specset.as_set + new_specset = SpecifierSet() + new_specset._specs = frozenset(combined_set) + new_pyspec = PySpecs(new_specset) + return new_pyspec + + @classmethod + @lru_cache(maxsize=128) + def from_marker(cls, marker): + if marker._markers[0][0] != 'python_version': + return + op = marker._markers[0][1].value + version = marker._markers[0][2].value + specset = set() + if op == "in": + specset.update( + Specifier("=={0}".format(v.strip())) + for v in version.split(",") + ) + elif op == "not in": + specset.update( + Specifier("!={0}".format(v.strip())) + for v in version.split(",") + ) + else: + specset.add(Specifier("".join([op, version]))) + if specset: + specifierset = SpecifierSet() + specifierset._specs = frozenset(specset) + newset = cls(specifierset) + return newset + return None diff --git a/src/passa/models/metadata.py b/src/passa/models/metadata.py index 45fb223..e626678 100644 --- a/src/passa/models/metadata.py +++ b/src/passa/models/metadata.py @@ -11,7 +11,7 @@ import vistir.misc from ..internals.markers import get_without_extra -from ..internals.specifiers import cleanup_pyspecs, pyspec_from_markers +from ..internals.specifiers import PySpecs def dedup_markers(s): @@ -28,24 +28,39 @@ class MetaSet(object): """ def __init__(self): self.markerset = frozenset() - self.pyspecset = packaging.specifiers.SpecifierSet() + self.pyspecset = PySpecs() def __repr__(self): return "MetaSet(markerset={0!r}, pyspecset={1!r})".format( ",".join(sorted(self.markerset)), str(self.pyspecset), ) + def __key(self): + return (tuple(self.markerset), hash(self.pyspecset)) + + def __hash__(self): + return hash(self.__key()) + + def __eq__(self, other): + return self.__key() == other.__key() + + def __len__(self): + return len(self.markerset) + len(self.pyspecset) + + def __iter__(self): + return itertools.chain(self.markerset, self.pyspecset) + def __str__(self): - pyspecs = set() + pyspecs = PySpecs() markerset = set() for m in self.markerset: - marker_specs = pyspec_from_markers(packaging.markers.Marker(m)) + marker_specs = PySpecs.from_marker(packaging.markers.Marker(m)) if marker_specs: pyspecs.add(marker_specs) else: markerset.add(m) if pyspecs: - self.pyspecset._specs &= pyspecs + self.pyspecset.add(pyspecs) self.markerset = frozenset(markerset) return " and ".join(dedup_markers(itertools.chain( # Make sure to always use the same quotes so we can dedup properly. @@ -54,8 +69,7 @@ def __str__(self): for ms in (str(m).replace('"', "'") for m in self.markerset) ), ( - "python_version {0[0]} '{0[1]}'".format(spec) - for spec in cleanup_pyspecs(self.pyspecset) + "{0}".format(str(spec)) for spec in self.pyspecset ), ))) @@ -68,16 +82,17 @@ def __nonzero__(self): # Python 2. def __or__(self, pair): marker, specset = pair markerset = set(self.markerset) + specset = PySpecs(specset) if marker: - marker_specs = pyspec_from_markers(marker) + marker_specs = PySpecs.from_marker(marker) if not marker_specs: markerset.add(str(marker)) else: - specset._specs &= marker_specs + specset.add(marker_specs) metaset = MetaSet() metaset.markerset = frozenset(markerset) # TODO: Implement some logic to clean up dups like '3.0.*' and '3.0'. - metaset.pyspecset &= self.pyspecset & specset + metaset.pyspecset = self.pyspecset | specset return metaset @@ -100,11 +115,10 @@ def _build_metasets(dependencies, pythons, key, trace, all_metasets): markers, packaging.specifiers.SpecifierSet(python), ) - metaset_iters.append( - parent_metaset | metaset - for parent_metaset in parent_metasets - ) - return list(itertools.chain.from_iterable(metaset_iters)) + for parent_metaset in parent_metasets: + child_metaset = parent_metaset | metaset + metasets.add(child_metaset) + return list(metasets) def _calculate_metasets_mapping(dependencies, pythons, traces): @@ -118,7 +132,7 @@ def _calculate_metasets_mapping(dependencies, pythons, traces): metasets = _build_metasets( dependencies, pythons, key, trace, all_metasets, ) - if metasets is None: + if metasets is None or len(metasets) == 0: continue new_metasets[key] = metasets if not new_metasets: @@ -136,11 +150,11 @@ def _format_metasets(metasets): return None # This extra str(Marker()) call helps simplify the expression. - return str(packaging.markers.Marker(" or ".join( - "{0}".format(s) if " and " in s else s - for s in dedup_markers(str(metaset) for metaset in metasets - if metaset) - ))) + _metasets = (dedup_markers(str(metaset) for metaset in metasets if metaset)) + metaset_string = " or ".join(meta for meta in list(_metasets)) + if not metaset_string: + return metaset_string + return str(packaging.markers.Marker(metaset_string)) def set_metadata(candidates, traces, dependencies, pythons): From 2ccbada45a9f7032a83126230e766937b1a393be Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 23 Sep 2018 00:25:40 -0400 Subject: [PATCH 20/32] Cleanup Signed-off-by: Dan Ryan --- src/passa/internals/markers.py | 5 +---- src/passa/internals/specifiers.py | 4 +--- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/passa/internals/markers.py b/src/passa/internals/markers.py index c2b7fdd..3793e96 100644 --- a/src/passa/internals/markers.py +++ b/src/passa/internals/markers.py @@ -54,10 +54,7 @@ def get_without_extra(marker): This could return `None` if the `extra == ...` part is the only one in the input marker. """ - # TODO: Why is this very deep in the internals? Why is a better solution - # implementing it yourself when someone is already maintaining a codebase - # for this? It's literally a grammar implementation that is required to - # meet the demands of a pep... -d + if not marker: return None marker = _ensure_marker(marker) diff --git a/src/passa/internals/specifiers.py b/src/passa/internals/specifiers.py index ae2025b..cf4c23e 100644 --- a/src/passa/internals/specifiers.py +++ b/src/passa/internals/specifiers.py @@ -155,7 +155,7 @@ def pyspec_from_markers(marker): return None -class PySpecs(collections.Set): +class PySpecs(collections.abc.Set): def __init__(self, specs=None): if not specs: specs = SpecifierSet() @@ -257,7 +257,6 @@ def marker_string(self): @cached_property @lru_cache(maxsize=128) def as_markers(self): - print("generating marker using string: %s" % self.marker_string) if not self.marker_string: return "" marker = Marker(self.marker_string) @@ -266,7 +265,6 @@ def as_markers(self): @lru_cache(maxsize=128) def __str__(self): string_repr = u"{0}".format(str(self.marker_string)) - print("converting to string: %s" % string_repr) return string_repr def __bool__(self): From fc59b47de5250dc1f074a01054749c7d0c670d5f Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 23 Sep 2018 00:26:04 -0400 Subject: [PATCH 21/32] Test fixes Signed-off-by: Dan Ryan --- tests/actions/test_clean.py | 8 ++++---- tests/conftest.py | 12 ++++++++---- tox.ini | 2 +- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/tests/actions/test_clean.py b/tests/actions/test_clean.py index 95b0b14..c56d06d 100644 --- a/tests/actions/test_clean.py +++ b/tests/actions/test_clean.py @@ -1,17 +1,17 @@ # -*- coding=utf-8 -*- +import passa.actions.add +import passa.actions.clean def test_clean_subset(project): - from passa.actions.add import add_packages - from passa.actions.clean import clean - retcode = add_packages(["requests"], project=project) + retcode = passa.actions.add.add_packages(["requests"], project=project) assert not retcode packages = ["requests", "chardet", "certifi", "idna"] c = project.venv.run("pip install pytz") assert c.returncode == 0 c = project.venv.run("python -c 'import pytz'") assert c.returncode == 0 - clean_retcode = clean(project=project) + clean_retcode = passa.actions.clean.clean(project=project) assert not clean_retcode c = project.venv.run("python -c 'import pytz'") assert c.returncode != 0 diff --git a/tests/conftest.py b/tests/conftest.py index e1326ca..1aaf419 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,9 @@ # -*- coding=utf-8 -*- import os import pytest +import passa import passa.cli.options -import passa.models.virtualenv +import mork.virtualenv import sys import vistir @@ -41,17 +42,20 @@ def virtualenv(tmpdir_factory): class _Project(passa.cli.options.Project): def __init__(self, root, venv=None): - self.path = os.path.abspath(root) + self.path = root.strpath self.venv = venv super(_Project, self).__init__(self.path) @pytest.fixture def tmpvenv(virtualenv): - return passa.models.virtualenv.VirtualEnv(virtualenv) + return mork.virtualenv.VirtualEnv(virtualenv.strpath) @pytest.fixture(scope="function") def project(project_directory, tmpvenv): - with tmpvenv.activated(): + venv_working_set = tmpvenv.initial_working_set + passa_dist = venv_working_set.by_key["passa"] + resolved = tmpvenv.resolve_dist(passa_dist, venv_working_set) + with tmpvenv.activated(extra_dists=list(resolved)): yield _Project(project_directory, tmpvenv) diff --git a/tox.ini b/tox.ini index 2bc8e1d..52124e8 100644 --- a/tox.ini +++ b/tox.ini @@ -8,7 +8,7 @@ setenv = LC_ALL = en_US.UTF-8 deps = coverage - -e .[tests] + -e .[virtualenv,tests] commands = coverage run --parallel -m pytest --timeout 300 [] install_command = python -m pip install {opts} {packages} usedevelop = True From 9586ce0a2f65e192e45738d01016f71973dbca41 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 23 Sep 2018 00:57:25 -0400 Subject: [PATCH 22/32] Fix specifier dedup logic Signed-off-by: Dan Ryan --- Pipfile.lock | 19 ++++++++++--------- src/passa/internals/specifiers.py | 12 ++++++++---- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 662eeb7..8285ba1 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -120,11 +120,11 @@ }, "installer": { "hashes": [ - "sha256:4ad3f14753d506d9ce646ac10320a2b82395208109f2356ed9e7a234976d3b4b", - "sha256:b849d2d876ef71162cfa6fc92204120a5cd215cad373b83fddcab401490d5fbc" + "sha256:1ba23de573e9b95a8dcbd04fd026c40a64b77db0aadc48f28a844b4cb87479fe", + "sha256:f4f195c9b17ea7d2b631a758451485c6b080975349b4adebe45ef4bb022db069" ], "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.1.0" + "version": "==0.1.1" }, "modutil": { "hashes": [ @@ -136,11 +136,11 @@ }, "mork": { "hashes": [ - "sha256:988dd3d768369e9c88bdade3d9b90ed3bfc91cb5eb229a0da4db9b119b0a5d7a", - "sha256:9cf945de7f69ed7ace8062509c5613c9484bdc2a5ac14caed6bba71f31f459c5" + "sha256:2a85d76bf669fd27d2532fa6721fc76fb7815d7ae7de62e095353f8aa845818c", + "sha256:bca61f20754f9432a7eec1874103b8fb6461a747e7b9b7dddcbd2fd6f9ffde32" ], "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.1.0" + "version": "==0.1.1" }, "packagebuilder": { "hashes": [ @@ -348,6 +348,7 @@ "sha256:62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4", "sha256:c31647edb69fd3d465a847ea3157d37bed1f95f19760b11a47aa91c04b666314" ], + "markers": "python_version == '2.6'", "version": "==1.4.0" }, "arpeggio": { @@ -568,11 +569,11 @@ }, "installer": { "hashes": [ - "sha256:4ad3f14753d506d9ce646ac10320a2b82395208109f2356ed9e7a234976d3b4b", - "sha256:b849d2d876ef71162cfa6fc92204120a5cd215cad373b83fddcab401490d5fbc" + "sha256:1ba23de573e9b95a8dcbd04fd026c40a64b77db0aadc48f28a844b4cb87479fe", + "sha256:f4f195c9b17ea7d2b631a758451485c6b080975349b4adebe45ef4bb022db069" ], "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.1.0" + "version": "==0.1.1" }, "invoke": { "hashes": [ diff --git a/src/passa/internals/specifiers.py b/src/passa/internals/specifiers.py index cf4c23e..05863c7 100644 --- a/src/passa/internals/specifiers.py +++ b/src/passa/internals/specifiers.py @@ -249,10 +249,10 @@ def marker_set(self): @cached_property @lru_cache(maxsize=128) def marker_string(self): - marker_string = " and ".join(sorted(str(m) for m in self.as_string_set)) + marker_string = " and ".join(sorted(str(m) for m in self.marker_set)) if not marker_string: return "" - return marker_string + return str(Marker(marker_string)) @cached_property @lru_cache(maxsize=128) @@ -264,7 +264,7 @@ def as_markers(self): @lru_cache(maxsize=128) def __str__(self): - string_repr = u"{0}".format(str(self.marker_string)) + string_repr = "{0}".format(str(self.marker_string)) return string_repr def __bool__(self): @@ -299,9 +299,13 @@ def from_marker(cls, marker): for v in version.split(",") ) elif op == "not in": + versions = version.split(",") + bad_versions = ["3.0", "3.1", "3.2", "3.3"] + if len(versions) >= 2 and any(v in versions for v in bad_versions): + versions = bad_versions specset.update( Specifier("!={0}".format(v.strip())) - for v in version.split(",") + for v in bad_versions ) else: specset.add(Specifier("".join([op, version]))) From d167d00a4f1eb83190dc5974ad5f49f10f26fe90 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 23 Sep 2018 01:01:12 -0400 Subject: [PATCH 23/32] Fix import Signed-off-by: Dan Ryan --- src/passa/internals/specifiers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/passa/internals/specifiers.py b/src/passa/internals/specifiers.py index 05863c7..a7c413f 100644 --- a/src/passa/internals/specifiers.py +++ b/src/passa/internals/specifiers.py @@ -3,6 +3,7 @@ from __future__ import absolute_import, unicode_literals import collections +import collections.abc import itertools import operator From f3f6c268e7e97002b6963d2dee56dae8b3e3814b Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 23 Sep 2018 01:01:59 -0400 Subject: [PATCH 24/32] Fix tox Signed-off-by: Dan Ryan --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 52124e8..d909724 100644 --- a/tox.ini +++ b/tox.ini @@ -23,7 +23,7 @@ commands = [testenv:docs] deps = -r{toxinidir}/docs/requirements.txt - -e .[tests] + -e .[virtualenv,tests] commands = sphinx-build -d {envtmpdir}/doctrees -b html docs docs/build/html sphinx-build -d {envtmpdir}/doctrees -b man docs docs/build/man From 56523a8c7c75975307748c512781f80bbf9efa07 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 23 Sep 2018 13:27:04 -0400 Subject: [PATCH 25/32] Use working set from environment instead of real prefix Signed-off-by: Dan Ryan --- tests/conftest.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 1aaf419..aff13f7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,6 +4,7 @@ import passa import passa.cli.options import mork.virtualenv +import pkg_resources import sys import vistir @@ -20,6 +21,9 @@ """.strip() +BASE_WORKING_SET = pkg_resources.WorkingSet(entries=sys.path) + + @pytest.fixture(scope="function") def project_directory(tmpdir_factory): project_dir = tmpdir_factory.mktemp("passa-project") @@ -54,8 +58,7 @@ def tmpvenv(virtualenv): @pytest.fixture(scope="function") def project(project_directory, tmpvenv): - venv_working_set = tmpvenv.initial_working_set - passa_dist = venv_working_set.by_key["passa"] - resolved = tmpvenv.resolve_dist(passa_dist, venv_working_set) + passa_dist = BASE_WORKING_SET["passa"] + resolved = tmpvenv.resolve_dist(passa_dist, BASE_WORKING_SET) with tmpvenv.activated(extra_dists=list(resolved)): yield _Project(project_directory, tmpvenv) From 6b7a9af5234df551fc439d3f0fc3c5480393a449 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 23 Sep 2018 13:53:44 -0400 Subject: [PATCH 26/32] Fix virtualenv usage and Set inheritance Signed-off-by: Dan Ryan --- src/passa/internals/specifiers.py | 9 ++++++--- tests/conftest.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/passa/internals/specifiers.py b/src/passa/internals/specifiers.py index a7c413f..5a2bcd9 100644 --- a/src/passa/internals/specifiers.py +++ b/src/passa/internals/specifiers.py @@ -2,11 +2,11 @@ from __future__ import absolute_import, unicode_literals -import collections -import collections.abc import itertools import operator +import six + from cached_property import cached_property from packaging.markers import Marker from packaging.specifiers import Specifier, SpecifierSet @@ -14,6 +14,9 @@ from vistir.misc import dedup +six.add_move(six.MovedAttribute("Set", "collections", "collections.abc")) +from six.moves import Set + try: from functools import lru_cache except ImportError: @@ -156,7 +159,7 @@ def pyspec_from_markers(marker): return None -class PySpecs(collections.abc.Set): +class PySpecs(Set): def __init__(self, specs=None): if not specs: specs = SpecifierSet() diff --git a/tests/conftest.py b/tests/conftest.py index aff13f7..0b490de 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -58,7 +58,7 @@ def tmpvenv(virtualenv): @pytest.fixture(scope="function") def project(project_directory, tmpvenv): - passa_dist = BASE_WORKING_SET["passa"] + passa_dist = BASE_WORKING_SET.by_key["passa"] resolved = tmpvenv.resolve_dist(passa_dist, BASE_WORKING_SET) with tmpvenv.activated(extra_dists=list(resolved)): yield _Project(project_directory, tmpvenv) From 704c9547c6ffd39c475191654ec66daff3dd2538 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 23 Sep 2018 22:15:41 -0400 Subject: [PATCH 27/32] Add set operations and comparison methods to metasets Signed-off-by: Dan Ryan --- Pipfile.lock | 62 +++++++++---------------------- src/passa/internals/specifiers.py | 18 ++++++++- src/passa/models/metadata.py | 61 ++++++++++++++++++++++++------ 3 files changed, 83 insertions(+), 58 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 8285ba1..c3a800a 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -19,7 +19,6 @@ "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92", "sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '3.6'", "version": "==1.4.3" }, "attrs": { @@ -33,7 +32,6 @@ "hashes": [ "sha256:f0b0e4eba956de51238e17573b7087e852dfe9854afd2e9c873f73fc0ca0a6dd" ], - "markers": "python_version <= '2.7' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.5" }, "backports-shutil-get-terminal-size": { @@ -55,7 +53,6 @@ "sha256:3a026f1a54135677e7da5ce819b0c690f156f37976f3e30c5430740725203d7f", "sha256:9217a59f14a5682da7c4b8829deadbfc194ac22e9908ccf7c8820234e80a1504" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.5.1" }, "cerberus": { @@ -83,7 +80,6 @@ "hashes": [ "sha256:cd502c66fc27c535bab62dc4f482e403e2369c2c05281a79cc2d4e2f42a87f20" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.2.7" }, "enum34": { @@ -93,7 +89,7 @@ "sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79", "sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1" ], - "markers": "python_version >= '2.6' and python_version >= '2.7' and python_version < '2.8' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.6" }, "first": { @@ -101,7 +97,6 @@ "sha256:3bb3de3582cb27071cfb514f00ed784dc444b7f96dc21e140de65fe00585c95e", "sha256:41d5b64e70507d0c3ca742d68010a76060eea8a3d863e9b5130ab11a4a91aa0e" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.0.1" }, "idna": { @@ -131,16 +126,16 @@ "sha256:2c85c1666649e92e56de17c00e1e831313602d9b55e8661d39c01e39003b45f7", "sha256:cc3dad264e36ed359fdd67c4588959d2996bd0402ad9c9d974ca906821537218" ], - "markers": "python_version >= '2.6' and python_version >= '3.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.0.0" }, "mork": { "hashes": [ - "sha256:2a85d76bf669fd27d2532fa6721fc76fb7815d7ae7de62e095353f8aa845818c", - "sha256:bca61f20754f9432a7eec1874103b8fb6461a747e7b9b7dddcbd2fd6f9ffde32" + "sha256:2ac3eed13edb8d854b6be470f0126da38514e9fb50b2112e45fc263e846d1576", + "sha256:bd16240d34ae4833cffa44b2671306c24fe830456105b908dac34fd31d2017af" ], "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.1.1" + "version": "==0.1.2" }, "packagebuilder": { "hashes": [ @@ -169,7 +164,7 @@ "sha256:8eb170f8d0d61825e09a95b38be068299ddeda82f35e96c3301a8a5e7604cb83", "sha256:d1aa2a11ba7b8f7b21ab852b1fb5afb277e1bb99d5dfc663380b5015c0d80c5a" ], - "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.6' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.3.2" }, "pip-shims": { @@ -218,7 +213,6 @@ "sha256:90151d8963f814e17190e067b60e92fb35fd1bc46c99f8dba3d7b0d93a3dd958", "sha256:c3aeaa4e0b80843ba65a68878293e07ea52a8d0706dbba86b02dad6cd20ef2dd" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.6" }, "resolvelib": { @@ -226,7 +220,6 @@ "sha256:6c4c6690b0bdd78bcc002e1a5d1b6abbde58c694a6ea1838f165b20d2c943db7", "sha256:8734e53271ef98f38a2c99324d5e7905bc00c97dc3fc5bb7d83c82a979e71c04" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.2.2" }, "scandir": { @@ -243,7 +236,7 @@ "sha256:c9009c527929f6e25604aec39b0a43c3f831d2947d89d6caaab22f057b7055c8", "sha256:f5c71e29b4e2af7ccdc03a020c626ede51da471173b4a6ad1e904f2b2e04b4bd" ], - "markers": "python_version < '3.5' and python_version < '3.6' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.9.0" }, "six": { @@ -274,7 +267,7 @@ "sha256:57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4", "sha256:a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a" ], - "markers": "(python_version >= '2.7' and python_version < '2.8') or (python_version >= '3.4' and python_version < '3.5') and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5'", + "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==3.6.6" }, "urllib3": { @@ -289,7 +282,7 @@ "sha256:2ce32cd126117ce2c539f0134eb89de91a8413a29baac49cbab3eb50e2026669", "sha256:ca07b4c0b54e14a91af9f34d0919790b016923d157afda5efdde55c96718f752" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==16.0.0" }, "vistir": { @@ -332,7 +325,6 @@ "sha256:37228cda29411948b422fae072f57e31d3396d2ee1c9783775980ee9c9990af6", "sha256:58587dd4dc3daefad0487f6d9ae32b4542b185e1c36db6993290e7c41ca2b47c" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.5" }, "appdirs": { @@ -340,7 +332,6 @@ "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92", "sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '3.6'", "version": "==1.4.3" }, "argparse": { @@ -363,7 +354,6 @@ "sha256:0312ad34fcad8fac3704d441f7b317e50af620823353ec657a53e981f92920c0", "sha256:ec9ae8adaae229e4f8446952d204a3e4b5fdd2d099f9be3aaf556120135fb3ee" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.2.1" }, "attrs": { @@ -384,7 +374,6 @@ "hashes": [ "sha256:f0b0e4eba956de51238e17573b7087e852dfe9854afd2e9c873f73fc0ca0a6dd" ], - "markers": "python_version <= '2.7' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.5" }, "backports-shutil-get-terminal-size": { @@ -414,7 +403,6 @@ "sha256:3a026f1a54135677e7da5ce819b0c690f156f37976f3e30c5430740725203d7f", "sha256:9217a59f14a5682da7c4b8829deadbfc194ac22e9908ccf7c8820234e80a1504" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.5.1" }, "cerberus": { @@ -450,7 +438,7 @@ "sha256:463f8483208e921368c9f306094eb6f725c6ca42b0f97e313cb5d5512459feda", "sha256:48eb22f4f8461b1df5734a074b57042430fb06e1d61bd1e11b078c0fe6d7a1f1" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and sys_platform == 'win32' or sys_platform == 'win32'", + "markers": "sys_platform == 'win32'", "version": "==0.3.9" }, "coverage": { @@ -487,14 +475,12 @@ "sha256:e05cb4d9aad6233d67e0541caa7e511fa4047ed7750ec2510d466e806e0255d6", "sha256:f3f501f345f24383c0000395b26b726e46758b71393267aeae0bd36f8b3ade80" ], - "markers": "python_version < '4' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==4.5.1" }, "distlib": { "hashes": [ "sha256:cd502c66fc27c535bab62dc4f482e403e2369c2c05281a79cc2d4e2f42a87f20" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.2.7" }, "docutils": { @@ -512,7 +498,7 @@ "sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79", "sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1" ], - "markers": "python_version >= '2.6' and python_version >= '2.7' and python_version < '2.8' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.6" }, "execnet": { @@ -520,7 +506,6 @@ "sha256:a7a84d5fa07a089186a329528f127c9d73b9de57f1a1131b82bb5320ee651f6a", "sha256:fc155a6b553c66c838d1a22dba1dc9f5f505c43285a878c6f74a79c024750b83" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.5.0" }, "first": { @@ -528,7 +513,6 @@ "sha256:3bb3de3582cb27071cfb514f00ed784dc444b7f96dc21e140de65fe00585c95e", "sha256:41d5b64e70507d0c3ca742d68010a76060eea8a3d863e9b5130ab11a4a91aa0e" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.0.1" }, "funcsigs": { @@ -536,7 +520,7 @@ "sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca", "sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50" ], - "markers": "python_version < '3.0' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.0'", "version": "==1.0.2" }, "idna": { @@ -601,7 +585,7 @@ "sha256:2c85c1666649e92e56de17c00e1e831313602d9b55e8661d39c01e39003b45f7", "sha256:cc3dad264e36ed359fdd67c4588959d2996bd0402ad9c9d974ca906821537218" ], - "markers": "python_version >= '2.6' and python_version >= '3.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.0.0" }, "more-itertools": { @@ -610,14 +594,13 @@ "sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e", "sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==4.3.0" }, "ordereddict": { "hashes": [ "sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f" ], - "markers": "python_version < '3.0' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.0'", "version": "==1.1" }, "packagebuilder": { @@ -654,7 +637,7 @@ "sha256:8eb170f8d0d61825e09a95b38be068299ddeda82f35e96c3301a8a5e7604cb83", "sha256:d1aa2a11ba7b8f7b21ab852b1fb5afb277e1bb99d5dfc663380b5015c0d80c5a" ], - "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.6' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.3.2" }, "pip-shims": { @@ -688,7 +671,6 @@ "sha256:6e3836e39f4d36ae72840833db137f7b7d35105079aee6ec4a62d9f80d594dd1", "sha256:95eb8364a4708392bae89035f45341871286a333f749c3141c20573d2b3876e1" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.7.1" }, "py": { @@ -696,7 +678,6 @@ "sha256:06a30435d058473046be836d3fc4f27167fd84c45b99704f2fb5509ef61f9af1", "sha256:50402e9d1c9005d759426988a492e0edaadb7f4e68bcddfea586bc7432d009c6" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.6.0" }, "pygments": { @@ -718,7 +699,6 @@ "sha256:0a72d8a9f559c006ba153e0c9b4838efd7b656cf1f993747ba7128770d6eb12c", "sha256:95529588ff4e85114a0b0ad8e9cf0131ca47d46b28230e25366c5aba66b1d854" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==3.8.1" }, "pytest-cov": { @@ -726,7 +706,6 @@ "sha256:513c425e931a0344944f84ea47f3956be0e416d95acbd897a44970c8d926d5d7", "sha256:e360f048b7dae3f2f2a9a4d067b2dd6b6a015d384d1577c994a43f3f7cbad762" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.6.0" }, "pytest-forked": { @@ -734,7 +713,6 @@ "sha256:e4500cd0509ec4a26535f7d4112a8cc0f17d3a41c29ffd4eab479d2a55b30805", "sha256:f275cb48a73fc61a6710726348e1da6d68a978f0ec0c54ece5a5fae5977e5a08" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.2" }, "pytest-timeout": { @@ -742,7 +720,6 @@ "sha256:1117fc0536e1638862917efbdc0895e6b62fa61e6cf4f39bb655686af7af9627", "sha256:b050a05da96a9992e90e884bc19b4790678b40c25471d2b77015b388417e1fa8" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.3.2" }, "pytest-xdist": { @@ -750,7 +727,6 @@ "sha256:0875deac20f6d96597036bdf63970887a6f36d28289c2f6682faf652dfea687b", "sha256:28e25e79698b2662b648319d3971c0f9ae0e6500f88258ccb9b153c31110ba9b" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.23.0" }, "pytz": { @@ -779,7 +755,6 @@ "sha256:90151d8963f814e17190e067b60e92fb35fd1bc46c99f8dba3d7b0d93a3dd958", "sha256:c3aeaa4e0b80843ba65a68878293e07ea52a8d0706dbba86b02dad6cd20ef2dd" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.6" }, "resolvelib": { @@ -787,7 +762,6 @@ "sha256:6c4c6690b0bdd78bcc002e1a5d1b6abbde58c694a6ea1838f165b20d2c943db7", "sha256:8734e53271ef98f38a2c99324d5e7905bc00c97dc3fc5bb7d83c82a979e71c04" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.2.2" }, "scandir": { @@ -804,7 +778,7 @@ "sha256:c9009c527929f6e25604aec39b0a43c3f831d2947d89d6caaab22f057b7055c8", "sha256:f5c71e29b4e2af7ccdc03a020c626ede51da471173b4a6ad1e904f2b2e04b4bd" ], - "markers": "python_version < '3.5' and python_version < '3.6' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.9.0" }, "six": { @@ -884,7 +858,7 @@ "sha256:57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4", "sha256:a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a" ], - "markers": "(python_version >= '2.7' and python_version < '2.8') or (python_version >= '3.4' and python_version < '3.5') and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5'", + "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==3.6.6" }, "urllib3": { @@ -899,7 +873,7 @@ "sha256:2ce32cd126117ce2c539f0134eb89de91a8413a29baac49cbab3eb50e2026669", "sha256:ca07b4c0b54e14a91af9f34d0919790b016923d157afda5efdde55c96718f752" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==16.0.0" }, "vistir": { diff --git a/src/passa/internals/specifiers.py b/src/passa/internals/specifiers.py index 5a2bcd9..09e352e 100644 --- a/src/passa/internals/specifiers.py +++ b/src/passa/internals/specifiers.py @@ -15,7 +15,7 @@ six.add_move(six.MovedAttribute("Set", "collections", "collections.abc")) -from six.moves import Set +from six.moves import reduce, Set try: from functools import lru_cache @@ -159,6 +159,13 @@ def pyspec_from_markers(marker): return None +def gen_marker(mkr): + m = Marker("python_version == '1'") + m._markers.pop() + m._markers.append(mkr) + return m + + class PySpecs(Set): def __init__(self, specs=None): if not specs: @@ -292,7 +299,14 @@ def __or__(self, specset): @classmethod @lru_cache(maxsize=128) def from_marker(cls, marker): - if marker._markers[0][0] != 'python_version': + if len(marker._markers) > 1: + markers = [ + gen_marker(mkr) for mkr in marker._markers + if not isinstance(mkr, six.string_types) + ] + markers = [cls.from_marker(mkr) for mkr in markers] + return reduce(lambda x, y: x | y, [marker for marker in markers if marker]) + if marker._markers[0][0].value != 'python_version': return op = marker._markers[0][1].value version = marker._markers[0][2].value diff --git a/src/passa/models/metadata.py b/src/passa/models/metadata.py index e626678..03733b5 100644 --- a/src/passa/models/metadata.py +++ b/src/passa/models/metadata.py @@ -4,12 +4,15 @@ import copy import itertools +import operator import packaging.markers import packaging.specifiers import vistir import vistir.misc +from six.moves import reduce + from ..internals.markers import get_without_extra from ..internals.specifiers import PySpecs @@ -50,6 +53,18 @@ def __len__(self): def __iter__(self): return itertools.chain(self.markerset, self.pyspecset) + def __lt__(self, other): + return operator.lt(self.__key(), other.__key()) + + def __le__(self, other): + return operator.le(self.__key(), other.__key()) + + def __ge__(self, other): + return operator.ge(self.__key(), other.__key()) + + def __gt__(self, other): + return operator.gt(self.__key(), other.__key()) + def __str__(self): pyspecs = PySpecs() markerset = set() @@ -80,19 +95,38 @@ def __nonzero__(self): # Python 2. return self.__bool__() def __or__(self, pair): - marker, specset = pair + metaset = MetaSet() markerset = set(self.markerset) - specset = PySpecs(specset) + pyspecset = PySpecs() + pyspecset |= self.pyspecset + if isinstance(pair, MetaSet): + markerset |= set(pair.markerset) + metaset.markerset = frozenset(markerset) + pyspecset |= pair.pyspecset + pyspecset.clean() + metaset.pyspecset = pyspecset + return metaset + marker, specset = pair + pyspecset |= PySpecs(specset) if marker: - marker_specs = PySpecs.from_marker(marker) - if not marker_specs: - markerset.add(str(marker)) - else: - specset.add(marker_specs) - metaset = MetaSet() + real_markers = set() + pyspec_markers = [] + for mkr in marker._markers: + if isinstance(mkr, tuple): + marker_string = " ".join([elem.serialize() for elem in mkr]) + if mkr[0].value == "python_version": + pyspec_markers.append(marker_string) + else: + real_markers.add(marker_string) + if pyspec_markers: + pyspec_markers = " and ".join(pyspec_markers) + marker_specs = PySpecs.from_marker(packaging.markers.Marker(pyspec_markers)) + pyspecset |= marker_specs + if real_markers: + markerset |= real_markers + pyspecset.clean() + metaset.pyspecset = pyspecset metaset.markerset = frozenset(markerset) - # TODO: Implement some logic to clean up dups like '3.0.*' and '3.0'. - metaset.pyspecset = self.pyspecset | specset return metaset @@ -150,10 +184,13 @@ def _format_metasets(metasets): return None # This extra str(Marker()) call helps simplify the expression. - _metasets = (dedup_markers(str(metaset) for metaset in metasets if metaset)) - metaset_string = " or ".join(meta for meta in list(_metasets)) + combined_metaset = MetaSet() + metasets = [combined_metaset,] + metasets + combined_metaset = reduce(lambda x, y: x | y, metasets) + metaset_string = str(combined_metaset) if not metaset_string: return metaset_string + metaset_string = str(MetaSet() | (packaging.markers.Marker(metaset_string), None)) return str(packaging.markers.Marker(metaset_string)) From e50f898d5553c3420aec235652b78858fd24ba54 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 26 Sep 2018 01:01:29 -0400 Subject: [PATCH 28/32] Add abstract extraction methods for marker cleanup Signed-off-by: Dan Ryan --- src/passa/internals/markers.py | 169 ++++++++++++++++++++++++++++++--- 1 file changed, 154 insertions(+), 15 deletions(-) diff --git a/src/passa/internals/markers.py b/src/passa/internals/markers.py index 3793e96..08b4322 100644 --- a/src/passa/internals/markers.py +++ b/src/passa/internals/markers.py @@ -3,6 +3,12 @@ from __future__ import absolute_import, unicode_literals from packaging.markers import Marker +from .specifiers import PySpecs, gen_marker +import six +import distlib.markers + +six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) +from six.moves import Mapping, reduce try: from functools import lru_cache @@ -17,24 +23,36 @@ def _ensure_marker(marker): def _strip_extra(elements): - """Remove the "extra == ..." operands from the list. + """Remove the "extra == ..." operands from the list.""" + + return _strip_marker_elem("extra", elements) + + +def _strip_pyversion(elements): + return _strip_marker_elem("python_version", elements) + + +def _strip_marker_elem(elem_name, elements): + """Remove the supplied element from the marker. This is not a comprehensive implementation, but relies on an important - characteristic of metadata generation: The "extra == ..." operand is always + characteristic of metadata generation: The element's operand is always associated with an "and" operator. This means that we can simply remove the operand and the "and" operator associated with it. """ + extra_indexes = [] + preceding_operators = ["and"] if elem_name == "extra" else ["and", "or"] for i, element in enumerate(elements): if isinstance(element, list): - cancelled = _strip_extra(element) + cancelled = _strip_marker_elem(elem_name, element) if cancelled: extra_indexes.append(i) - elif isinstance(element, tuple) and element[0].value == "extra": + elif isinstance(element, tuple) and element[0].value == elem_name: extra_indexes.append(i) for i in reversed(extra_indexes): del elements[i] - if i > 0 and elements[i - 1] == "and": + if i > 0 and elements[i - 1] in preceding_operators: # Remove the "and" before it. del elements[i - 1] elif elements: @@ -45,6 +63,19 @@ def _strip_extra(elements): return (not elements) +def _get_stripped_marker(marker, strip_func): + """Build a new marker which is cleaned according to `strip_func`""" + + if not marker: + return None + marker = _ensure_marker(marker) + elements = marker._markers + strip_func(elements) + if elements: + return marker + return None + + def get_without_extra(marker): """Build a new marker without the `extra == ...` part. @@ -55,14 +86,17 @@ def get_without_extra(marker): input marker. """ - if not marker: - return None - marker = _ensure_marker(marker) - elements = marker._markers - _strip_extra(elements) - if elements: - return marker - return None + return _get_stripped_marker(marker, _strip_extra) + + +def get_without_pyversion(marker): + """Built a new marker without the `python_version` part. + + This could return `None` if the `python_version` section is the only section in the + marker. + """ + + return _get_stripped_marker(marker, _strip_pyversion) def _markers_collect_extras(markers, collection): @@ -76,6 +110,23 @@ def _markers_collect_extras(markers, collection): _markers_collect_extras(el, collection) +def _markers_collect_pyversions(markers, collection): + local_collection = [] + marker_format_str = "{0}" + for i, el in enumerate(reversed(markers)): + if (isinstance(el, tuple) and + el[0].value == "python_version"): + new_marker = str(gen_marker(el)) + local_collection.append(marker_format_str.format(new_marker)) + elif isinstance(el, six.string_types): + local_collection.append(el) + elif isinstance(el, list): + _markers_collect_pyversions(el, local_collection) + if local_collection: + local_collection = "{0}".format(" ".join(local_collection)) + collection.append(local_collection) + + @lru_cache(maxsize=128) def get_contained_extras(marker): """Collect "extra == ..." operands from a marker. @@ -90,13 +141,54 @@ def get_contained_extras(marker): return extras +def get_contained_pyversions(marker): + """Collect all `python_version` operands from a marker. + + Returns a set of :class:`~passa.internals.specifiers.PySpecs` instances. + """ + + collection = [] + if not marker: + return set() + marker = _ensure_marker(marker) + # Collect the (Variable, Op, Value) tuples and string joiners from the marker + _markers_collect_pyversions(marker._markers, collection) + marker_str = " ".join(collection) + if not marker_str: + return set() + # Use the distlib dictionary parser to create a dictionary 'trie' which is a bit + # easier to reason about + marker_dict = distlib.markers.parse_marker(marker_str)[0] + version_set = set() + pyversions = parse_marker_dict(marker_dict) + if isinstance(pyversions, set): + version_set.update(pyversions) + else: + version_set.add(pyversions) + # Each distinct element in the set was separated by an "and" operator in the marker + # So we will need to reduce them with an intersection here rather than a union + # in order to find the boundaries + versions = reduce(lambda x, y: x & y, version_set) + if not versions: + return PySpecs() + return versions + + def _markers_contains_extra(markers): # Optimization: the marker element is usually appended at the end. + return _markers_contains_key(markers, "extra") + + +def _markers_contains_pyversion(markers): + return _markers_contains_key(markers, "python_version") + + +def _markers_contains_key(markers, key): for element in reversed(markers): - if isinstance(element, tuple) and element[0].value == "extra": + if isinstance(element, tuple) and element[0].value == key: return True elif isinstance(element, list): - if _markers_contains_extra(element): + if _markers_contains_key(element, key): return True return False @@ -109,3 +201,50 @@ def contains_extra(marker): return False marker = _ensure_marker(marker) return _markers_contains_extra(marker._markers) + + +@lru_cache(maxsize=128) +def contains_pyversion(marker): + """Check whether a marker contains a python_version operand. + """ + + if not marker: + return False + marker = _ensure_marker(marker) + return _markers_contains_pyversion(marker._markers) + + +def parse_marker_dict(marker_dict): + op = marker_dict["op"] + lhs = marker_dict["lhs"] + rhs = marker_dict["rhs"] + # This is where the spec sets for each side land if we have an "or" operator + sides = set() + # And if we hit the end of the parse tree we use this format string to make a marker + format_string = "{lhs} {op} {rhs}" + # Essentially we will iterate over each side of the parsed marker if either one is + # A mapping instance (i.e. a dictionary) and recursively parse and reduce the specset + # Union the "and" specs, intersect the "or"s to find the most appropriate range + if any(issubclass(type(side), Mapping) for side in (lhs, rhs)): + for side in (lhs, rhs): + specs = PySpecs() + if issubclass(type(side), Mapping): + specs.add(parse_marker_dict(side)) + else: + # This is the easiest way to go from a string to a PySpec instance + specs.add(PySpecs.from_marker(Marker(side))) + sides.add(specs) + if op == "and": + # When we are "and"-ing things together, it probably makes the most sense + # to reduce them here into a single PySpec instance + if not sides: + sides = [lhs, rhs] + sides = reduce(lambda x, y: x | y, sides) + return PySpecs.from_marker(Marker(str(sides))) + # Actually when we "or" things as well we can also just turn them into a reduced + # set using this logic now + return reduce(lambda x, y: x & y, sides) + else: + # At the tip of the tree we are dealing with strings all around and they just need + # to be smashed together + return PySpecs.from_marker(Marker(format_string.format(**marker_dict))) From 9e51f06738ae4d7e95a3c49ba40462385450a09a Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 26 Sep 2018 01:12:02 -0400 Subject: [PATCH 29/32] Add intersection method for PySpecs and smarter unions and creation Signed-off-by: Dan Ryan --- src/passa/internals/specifiers.py | 89 +++++++++++++++++++++++-------- 1 file changed, 67 insertions(+), 22 deletions(-) diff --git a/src/passa/internals/specifiers.py b/src/passa/internals/specifiers.py index 09e352e..3f2ae8c 100644 --- a/src/passa/internals/specifiers.py +++ b/src/passa/internals/specifiers.py @@ -69,7 +69,15 @@ def _get_specs(specset): specset = SpecifierSet(specset.replace(".*", "")) result = [] for spec in set(specset): - result.append((spec.operator, _tuplize_version(spec.version))) + version = spec.version + op = spec.operator + if op in ("in", "not in"): + versions = version.split(",") + op = "==" if op == "in" else "!=" + for ver in versions: + result.append((op, _tuplize_version(ver.strip()))) + else: + result.append((spec.operator, _tuplize_version(spec.version))) return result @@ -77,7 +85,7 @@ def _get_specs(specset): def _group_by_op(specs): specs = [_get_specs(x) for x in list(specs)] flattened = [(op, version) for spec in specs for op, version in spec] - specs = sorted(flattened, key=operator.itemgetter(1)) + specs = sorted(flattened) grouping = itertools.groupby(specs, key=operator.itemgetter(0)) return grouping @@ -208,15 +216,26 @@ def clean(self): if spec: self.cleaned_tuples.add((spec.operator, spec.version)) else: - self.cleaned_tuples = cleanup_pyspecs(self.specifierset) + self.cleaned_tuples = cleanup_pyspecs(self.specifierset, joiner="and") self.specifierset = self.as_specset def add(self, other): - new_pyspec = PySpecs(self.specifierset) - new_pyspec.specifierset &= other.specifierset - return new_pyspec - - @cached_property + if not isinstance(other, self.__class__): + if isinstance(other, SpecifierSet): + other = PySpecs(other) + else: + raise TypeError("Cannot add type {0!r} to PySpecs".format(type(other))) + new_specifierset = SpecifierSet() + new_specifierset &= self.as_specset + try: + new_specifierset &= other.as_specset + except AttributeError: + pass + new_pyspec = PySpecs(new_specifierset) + self.specifierset = new_pyspec.specifierset + self.cleaned_tuples = new_pyspec.cleaned_tuples + + @property @lru_cache(maxsize=128) def as_specset(self): specs = set() @@ -232,12 +251,12 @@ def as_specset(self): specifierset._specs = frozenset(specs) return specifierset - @cached_property + @property @lru_cache(maxsize=128) def as_set(self): return set(self.specifierset) - @cached_property + @property @lru_cache(maxsize=128) def as_string_set(self): returnval = set() @@ -251,13 +270,13 @@ def as_string_set(self): for s in sorted(self.cleaned_tuples) ) - @cached_property + @property @lru_cache(maxsize=128) def marker_set(self): markerset = {Marker(spec) for spec in self.as_string_set} return markerset - @cached_property + @property @lru_cache(maxsize=128) def marker_string(self): marker_string = " and ".join(sorted(str(m) for m in self.marker_set)) @@ -265,7 +284,7 @@ def marker_string(self): return "" return str(Marker(marker_string)) - @cached_property + @property @lru_cache(maxsize=128) def as_markers(self): if not self.marker_string: @@ -284,6 +303,29 @@ def __bool__(self): def __nonzero__(self): # Python 2. return self.__bool__() + @lru_cache(maxsize=128) + def __and__(self, other): + if not isinstance(other, PySpecs): + specset = PySpecs(other) + if self == other: + return self + new_specset = SpecifierSet() + diff_specset = SpecifierSet() + intersection = set(self.as_specset) & set(other.as_specset) + diff_specset._specs = frozenset(set(self.as_specset) ^ set(other.as_specset)) + tuples = cleanup_pyspecs(diff_specset, joiner="or") + new_marker_str = " and ".join( + "python_version {0} '{1}'".format(op, val) + for op, val in tuples + ) + specset = set() + if new_marker_str: + marker = Marker(new_marker_str) + specset = set(PySpecs.from_marker(marker).as_specset) + specset = frozenset(specset | intersection) + new_specset._specs = specset + return PySpecs(new_specset) + @lru_cache(maxsize=128) def __or__(self, specset): if not isinstance(specset, PySpecs): @@ -297,15 +339,18 @@ def __or__(self, specset): return new_pyspec @classmethod - @lru_cache(maxsize=128) def from_marker(cls, marker): + if not marker: + return PySpecs() if len(marker._markers) > 1: - markers = [ - gen_marker(mkr) for mkr in marker._markers - if not isinstance(mkr, six.string_types) - ] - markers = [cls.from_marker(mkr) for mkr in markers] - return reduce(lambda x, y: x | y, [marker for marker in markers if marker]) + specs = PySpecs() + markers = sorted([ + el for el in marker._markers + if isinstance(el, tuple) + ], key=lambda x: x[2].value) + for mkr in markers: + specs.add(cls.from_marker(gen_marker(mkr))) + return specs if marker._markers[0][0].value != 'python_version': return op = marker._markers[0][1].value @@ -317,13 +362,13 @@ def from_marker(cls, marker): for v in version.split(",") ) elif op == "not in": - versions = version.split(",") + versions = [v.strip() for v in version.split(",")] bad_versions = ["3.0", "3.1", "3.2", "3.3"] if len(versions) >= 2 and any(v in versions for v in bad_versions): versions = bad_versions specset.update( Specifier("!={0}".format(v.strip())) - for v in bad_versions + for v in sorted(bad_versions) ) else: specset.add(Specifier("".join([op, version]))) From 7ea23eeac6901982bffe9b446704852d9914f41f Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 26 Sep 2018 01:12:50 -0400 Subject: [PATCH 30/32] Simplify and cleanup metadata implementation Signed-off-by: Dan Ryan --- src/passa/models/metadata.py | 102 ++++++++++++++++------------------- 1 file changed, 46 insertions(+), 56 deletions(-) diff --git a/src/passa/models/metadata.py b/src/passa/models/metadata.py index 03733b5..27990a8 100644 --- a/src/passa/models/metadata.py +++ b/src/passa/models/metadata.py @@ -13,7 +13,9 @@ from six.moves import reduce -from ..internals.markers import get_without_extra +from ..internals.markers import ( + get_without_extra, get_without_pyversion, get_contained_pyversions +) from ..internals.specifiers import PySpecs @@ -39,7 +41,7 @@ def __repr__(self): ) def __key(self): - return (tuple(self.markerset), hash(self.pyspecset)) + return (tuple(self.markerset), hash(tuple(self.pyspecset))) def __hash__(self): return hash(self.__key()) @@ -66,26 +68,17 @@ def __gt__(self, other): return operator.gt(self.__key(), other.__key()) def __str__(self): - pyspecs = PySpecs() - markerset = set() - for m in self.markerset: - marker_specs = PySpecs.from_marker(packaging.markers.Marker(m)) - if marker_specs: - pyspecs.add(marker_specs) - else: - markerset.add(m) - if pyspecs: - self.pyspecset.add(pyspecs) - self.markerset = frozenset(markerset) + self.markerset = frozenset(filter(None, self.markerset)) return " and ".join(dedup_markers(itertools.chain( # Make sure to always use the same quotes so we can dedup properly. ( "{0}".format(ms) if " or " in ms else ms for ms in (str(m).replace('"', "'") for m in self.markerset) + if ms ), ( - "{0}".format(str(spec)) for spec in self.pyspecset - ), + "{0}".format(str(self.pyspecset)) if self.pyspecset else "", + ) ))) def __bool__(self): @@ -94,39 +87,38 @@ def __bool__(self): def __nonzero__(self): # Python 2. return self.__bool__() - def __or__(self, pair): - metaset = MetaSet() - markerset = set(self.markerset) - pyspecset = PySpecs() - pyspecset |= self.pyspecset - if isinstance(pair, MetaSet): - markerset |= set(pair.markerset) - metaset.markerset = frozenset(markerset) - pyspecset |= pair.pyspecset - pyspecset.clean() - metaset.pyspecset = pyspecset - return metaset + @classmethod + def from_tuple(cls, pair): marker, specset = pair - pyspecset |= PySpecs(specset) + pyspecs = PySpecs(specset) + markerset = set() if marker: - real_markers = set() - pyspec_markers = [] - for mkr in marker._markers: - if isinstance(mkr, tuple): - marker_string = " ".join([elem.serialize() for elem in mkr]) - if mkr[0].value == "python_version": - pyspec_markers.append(marker_string) - else: - real_markers.add(marker_string) - if pyspec_markers: - pyspec_markers = " and ".join(pyspec_markers) - marker_specs = PySpecs.from_marker(packaging.markers.Marker(pyspec_markers)) - pyspecset |= marker_specs - if real_markers: - markerset |= real_markers - pyspecset.clean() - metaset.pyspecset = pyspecset + # Returns a PySpec instance or None + marker_pyversions = get_contained_pyversions(marker) + if marker_pyversions: + pyspecs.add(marker_pyversions) + # The remainder of the marker, if there is any + cleaned_marker = get_without_pyversion(marker) + if cleaned_marker: + markerset.add(str(cleaned_marker)) + metaset = cls() + metaset.markerset = frozenset(markerset) + metaset.pyspecset = pyspecs + return metaset + + def __or__(self, other): + if not isinstance(other, type(self)): + other = self.from_tuple(other) + metaset = MetaSet() + markerset = set() + specset = PySpecs() + for meta in (self, other): + if meta.markerset: + markerset |= set(meta.markerset) + if meta.pyspecset: + specset = specset | meta.pyspecset metaset.markerset = frozenset(markerset) + metaset.pyspecset = specset return metaset @@ -179,19 +171,17 @@ def _calculate_metasets_mapping(dependencies, pythons, traces): def _format_metasets(metasets): - # If there is an unconditional route, this needs to be unconditional. - if not metasets or not all(metasets): - return None + metasets = dedup_markers(metaset for metaset in metasets if metaset) + # If there is an unconditional route, this needs to be unconditional. + if not metasets: + return "" + combined_metaset = str(MetaSet() | reduce(lambda x, y: x | y, metasets)) + if not combined_metaset: + return "" # This extra str(Marker()) call helps simplify the expression. - combined_metaset = MetaSet() - metasets = [combined_metaset,] + metasets - combined_metaset = reduce(lambda x, y: x | y, metasets) - metaset_string = str(combined_metaset) - if not metaset_string: - return metaset_string - metaset_string = str(MetaSet() | (packaging.markers.Marker(metaset_string), None)) - return str(packaging.markers.Marker(metaset_string)) + metaset_string = str(packaging.markers.Marker(combined_metaset)) + return metaset_string def set_metadata(candidates, traces, dependencies, pythons): From f65c6ef7ea6bdb94e1b0cb38efdcd0ece571bacf Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 26 Sep 2018 10:16:25 -0400 Subject: [PATCH 31/32] update lockfile? Signed-off-by: Dan Ryan --- Pipfile.lock | 261 +++++++++++++++++++++++------- src/passa/internals/specifiers.py | 51 +++--- 2 files changed, 231 insertions(+), 81 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index c3a800a..89c3e2d 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -32,20 +32,21 @@ "hashes": [ "sha256:f0b0e4eba956de51238e17573b7087e852dfe9854afd2e9c873f73fc0ca0a6dd" ], + "markers": "python_version < '2.8' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.5" }, "backports-shutil-get-terminal-size": { "hashes": [ "sha256:0975ba55054c15e346944b38956a4c9cbee9009391e41b86c68990effb8c1f64" ], - "markers": "python_version < '3.3' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.3' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.0.0" }, "backports-weakref": { "hashes": [ "sha256:81bc9b51c0abc58edc76aefbbc68c62a787918ffe943a37947e162c3f8e19e82" ], - "markers": "python_version < '3.3' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.3' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.0.post1" }, "cached-property": { @@ -53,13 +54,14 @@ "sha256:3a026f1a54135677e7da5ce819b0c690f156f37976f3e30c5430740725203d7f", "sha256:9217a59f14a5682da7c4b8829deadbfc194ac22e9908ccf7c8820234e80a1504" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.5.1" }, "cerberus": { "hashes": [ "sha256:f5c2e048fb15ecb3c088d192164316093fcfa602a74b3386eefb2983aa7e800a" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.2" }, "certifi": { @@ -67,6 +69,7 @@ "sha256:376690d6f16d32f9d1fe8932551d80b23e9d393a8578c5633a2ed39a64861638", "sha256:456048c7e371c089d0a77a5212fb37a2c2dce1e24146e3b7e0261736aaeaa22a" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2018.8.24" }, "chardet": { @@ -74,12 +77,14 @@ "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==3.0.4" }, "distlib": { "hashes": [ "sha256:cd502c66fc27c535bab62dc4f482e403e2369c2c05281a79cc2d4e2f42a87f20" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.2.7" }, "enum34": { @@ -89,7 +94,7 @@ "sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79", "sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1" ], - "markers": "python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '2.8' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.6" }, "first": { @@ -97,6 +102,7 @@ "sha256:3bb3de3582cb27071cfb514f00ed784dc444b7f96dc21e140de65fe00585c95e", "sha256:41d5b64e70507d0c3ca742d68010a76060eea8a3d863e9b5130ab11a4a91aa0e" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.0.1" }, "idna": { @@ -104,13 +110,14 @@ "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.7" }, "importlib": { "hashes": [ "sha256:b6ee7066fea66e35f8d0acee24d98006de1a0a8a94a8ce6efe73a9a23c8d9826" ], - "markers": "python_version < '2.7' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '2.7' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.0.4" }, "installer": { @@ -118,7 +125,7 @@ "sha256:1ba23de573e9b95a8dcbd04fd026c40a64b77db0aadc48f28a844b4cb87479fe", "sha256:f4f195c9b17ea7d2b631a758451485c6b080975349b4adebe45ef4bb022db069" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.1.1" }, "modutil": { @@ -126,7 +133,7 @@ "sha256:2c85c1666649e92e56de17c00e1e831313602d9b55e8661d39c01e39003b45f7", "sha256:cc3dad264e36ed359fdd67c4588959d2996bd0402ad9c9d974ca906821537218" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '3.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.0.0" }, "mork": { @@ -134,7 +141,7 @@ "sha256:2ac3eed13edb8d854b6be470f0126da38514e9fb50b2112e45fc263e846d1576", "sha256:bd16240d34ae4833cffa44b2671306c24fe830456105b908dac34fd31d2017af" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.1.2" }, "packagebuilder": { @@ -142,7 +149,7 @@ "sha256:1e85c4e0e994322996b93cd6685c12834d30f3558889154f8e3de8fb1f3fd1e7", "sha256:dc525d06ecd102db23ab421b879d7d27021d784ff933e33e8c411a53af5c9dbe" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.1.0" }, "packaging": { @@ -150,6 +157,7 @@ "sha256:e9215d2d2535d3ae866c3d6efc77d5b24a0192cce0ff20e42896cc0664f889c0", "sha256:f019b770dd64e585a99714f1fd5e01c7a8f11b45635aa953fd41c689a657375b" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==17.1" }, "passa": { @@ -164,7 +172,7 @@ "sha256:8eb170f8d0d61825e09a95b38be068299ddeda82f35e96c3301a8a5e7604cb83", "sha256:d1aa2a11ba7b8f7b21ab852b1fb5afb277e1bb99d5dfc663380b5015c0d80c5a" ], - "markers": "python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.3.2" }, "pip-shims": { @@ -172,7 +180,7 @@ "sha256:9c8a568b4a8ce4000a2982224f48a35736fca81214dfdb30dcae24287866a7e4", "sha256:ebc2bb29ddd21fa00c0cf28a5d8c725100f2f7ee98703aba237efd02e205c1c1" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.1.2" }, "plette": { @@ -183,7 +191,7 @@ "sha256:c0e3553c1e581d8423daccbd825789c6e7f29b7d9e00e5331b12e1642a1a26d3", "sha256:dde5d525cf5f0cbad4d938c83b93db17887918daf63c13eafed257c4f61b07b4" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.2.2" }, "pyparsing": { @@ -191,6 +199,7 @@ "sha256:905d8090c335314568b5faee0025b1829f27bb974604a5762a6cdef3a7dfc3b7", "sha256:f493ee323be1e94929416b3585eefcc04943115cecbaaa35a8c86d1a2368af19" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.2.1" }, "recursive-monkey-patch": { @@ -198,7 +207,7 @@ "sha256:546739fea5be2ea9f98b5ec44fafeb697b5cf9fdcda64a03422582ab03ee24c4", "sha256:98922554e77f2e2c85a4f5d873a0f52efdc1b553f32444bd6c788b2ff583bf3e" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.4.0" }, "requests": { @@ -206,6 +215,7 @@ "sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1", "sha256:ec22d826a36ed72a7358ff3fe56cbd4ba69dd7a6718ffd450ff0e9df7a47ce6a" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.19.1" }, "requirementslib": { @@ -213,6 +223,7 @@ "sha256:90151d8963f814e17190e067b60e92fb35fd1bc46c99f8dba3d7b0d93a3dd958", "sha256:c3aeaa4e0b80843ba65a68878293e07ea52a8d0706dbba86b02dad6cd20ef2dd" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.6" }, "resolvelib": { @@ -220,6 +231,7 @@ "sha256:6c4c6690b0bdd78bcc002e1a5d1b6abbde58c694a6ea1838f165b20d2c943db7", "sha256:8734e53271ef98f38a2c99324d5e7905bc00c97dc3fc5bb7d83c82a979e71c04" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.2.2" }, "scandir": { @@ -236,7 +248,7 @@ "sha256:c9009c527929f6e25604aec39b0a43c3f831d2947d89d6caaab22f057b7055c8", "sha256:f5c71e29b4e2af7ccdc03a020c626ede51da471173b4a6ad1e904f2b2e04b4bd" ], - "markers": "python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.9.0" }, "six": { @@ -244,6 +256,7 @@ "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" ], + "markers": "python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.11.0" }, "toml": { @@ -251,6 +264,7 @@ "sha256:380178cde50a6a79f9d2cf6f42a62a5174febe5eea4126fe4038785f1d888d42", "sha256:a7901919d3e4f92ffba7ff40a9d697e35bbbc8a8049fe8da742f34c83606d957" ], + "markers": "python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.9.6" }, "tomlkit": { @@ -258,7 +272,7 @@ "sha256:8ab16e93162fc44d3ad83d2aa29a7140b8f7d996ae1790a73b9a7aed6fb504ac", "sha256:ca181cee7aee805d455628f7c94eb8ae814763769a93e69157f250fe4ebe1926" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.4.4" }, "typing": { @@ -267,7 +281,7 @@ "sha256:57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4", "sha256:a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a" ], - "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==3.6.6" }, "urllib3": { @@ -275,6 +289,7 @@ "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf", "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5" ], + "markers": "python_version < '4' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.23" }, "virtualenv": { @@ -282,7 +297,7 @@ "sha256:2ce32cd126117ce2c539f0134eb89de91a8413a29baac49cbab3eb50e2026669", "sha256:ca07b4c0b54e14a91af9f34d0919790b016923d157afda5efdde55c96718f752" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==16.0.0" }, "vistir": { @@ -293,7 +308,7 @@ "sha256:8a360ac20cbcc0863d6dbbe7a52e8b2c9ebf48abd6833c3813a82c70708244af", "sha256:bc6e10284792485c10585536e6aede9e38996c841cc9d2a67238cd05742c2d0b" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.1.6" }, "wheel": { @@ -301,6 +316,7 @@ "sha256:0a2e54558a0628f2145d2fc822137e322412115173e8a2ddbe1c9024338ae83c", "sha256:80044e51ec5bbf6c894ba0bc48d26a8c20a9ba629f4ca19ea26ecfcf87685f5f" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.31.1" }, "yaspin": { @@ -308,7 +324,7 @@ "sha256:36fdccc5e0637b5baa8892fe2c3d927782df7d504e9020f40eb2c1502518aa5a", "sha256:8e52bf8079a48e2a53f3dfeec9e04addb900c101d1591c85df69cf677d3237e7" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.14.0" } }, @@ -325,6 +341,7 @@ "sha256:37228cda29411948b422fae072f57e31d3396d2ee1c9783775980ee9c9990af6", "sha256:58587dd4dc3daefad0487f6d9ae32b4542b185e1c36db6993290e7c41ca2b47c" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.5" }, "appdirs": { @@ -332,16 +349,9 @@ "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92", "sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e" ], + "markers": "python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.4.3" }, - "argparse": { - "hashes": [ - "sha256:62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4", - "sha256:c31647edb69fd3d465a847ea3157d37bed1f95f19760b11a47aa91c04b666314" - ], - "markers": "python_version == '2.6'", - "version": "==1.4.0" - }, "arpeggio": { "hashes": [ "sha256:a5258b84f76661d558492fa87e42db634df143685a0e51802d59cae7daad8732", @@ -354,6 +364,7 @@ "sha256:0312ad34fcad8fac3704d441f7b317e50af620823353ec657a53e981f92920c0", "sha256:ec9ae8adaae229e4f8446952d204a3e4b5fdd2d099f9be3aaf556120135fb3ee" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.2.1" }, "attrs": { @@ -361,6 +372,7 @@ "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" ], + "markers": "python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==18.2.0" }, "babel": { @@ -374,20 +386,21 @@ "hashes": [ "sha256:f0b0e4eba956de51238e17573b7087e852dfe9854afd2e9c873f73fc0ca0a6dd" ], + "markers": "python_version < '2.8' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.5" }, "backports-shutil-get-terminal-size": { "hashes": [ "sha256:0975ba55054c15e346944b38956a4c9cbee9009391e41b86c68990effb8c1f64" ], - "markers": "python_version < '3.3' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.3' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.0.0" }, "backports-weakref": { "hashes": [ "sha256:81bc9b51c0abc58edc76aefbbc68c62a787918ffe943a37947e162c3f8e19e82" ], - "markers": "python_version < '3.3' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.3' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.0.post1" }, "black": { @@ -398,18 +411,26 @@ "markers": "python_version >= '3.6'", "version": "==18.6b4" }, + "bleach": { + "hashes": [ + "sha256:0ee95f6167129859c5dce9b1ca291ebdb5d8cd7e382ca0e237dfd0dad63f63d8", + "sha256:24754b9a7d530bf30ce7cbc805bc6cce785660b4a10ff3a43633728438c105ab" + ], + "version": "==2.1.4" + }, "cached-property": { "hashes": [ "sha256:3a026f1a54135677e7da5ce819b0c690f156f37976f3e30c5430740725203d7f", "sha256:9217a59f14a5682da7c4b8829deadbfc194ac22e9908ccf7c8820234e80a1504" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.5.1" }, "cerberus": { "hashes": [ "sha256:f5c2e048fb15ecb3c088d192164316093fcfa602a74b3386eefb2983aa7e800a" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.2" }, "certifi": { @@ -417,28 +438,101 @@ "sha256:376690d6f16d32f9d1fe8932551d80b23e9d393a8578c5633a2ed39a64861638", "sha256:456048c7e371c089d0a77a5212fb37a2c2dce1e24146e3b7e0261736aaeaa22a" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2018.8.24" }, + "cffi": { + "hashes": [ + "sha256:151b7eefd035c56b2b2e1eb9963c90c6302dc15fbd8c1c0a83a163ff2c7d7743", + "sha256:1553d1e99f035ace1c0544050622b7bc963374a00c467edafac50ad7bd276aef", + "sha256:1b0493c091a1898f1136e3f4f991a784437fac3673780ff9de3bcf46c80b6b50", + "sha256:2ba8a45822b7aee805ab49abfe7eec16b90587f7f26df20c71dd89e45a97076f", + "sha256:3bb6bd7266598f318063e584378b8e27c67de998a43362e8fce664c54ee52d30", + "sha256:3c85641778460581c42924384f5e68076d724ceac0f267d66c757f7535069c93", + "sha256:3eb6434197633b7748cea30bf0ba9f66727cdce45117a712b29a443943733257", + "sha256:495c5c2d43bf6cebe0178eb3e88f9c4aa48d8934aa6e3cddb865c058da76756b", + "sha256:4c91af6e967c2015729d3e69c2e51d92f9898c330d6a851bf8f121236f3defd3", + "sha256:57b2533356cb2d8fac1555815929f7f5f14d68ac77b085d2326b571310f34f6e", + "sha256:770f3782b31f50b68627e22f91cb182c48c47c02eb405fd689472aa7b7aa16dc", + "sha256:79f9b6f7c46ae1f8ded75f68cf8ad50e5729ed4d590c74840471fc2823457d04", + "sha256:7a33145e04d44ce95bcd71e522b478d282ad0eafaf34fe1ec5bbd73e662f22b6", + "sha256:857959354ae3a6fa3da6651b966d13b0a8bed6bbc87a0de7b38a549db1d2a359", + "sha256:87f37fe5130574ff76c17cab61e7d2538a16f843bb7bca8ebbc4b12de3078596", + "sha256:95d5251e4b5ca00061f9d9f3d6fe537247e145a8524ae9fd30a2f8fbce993b5b", + "sha256:9d1d3e63a4afdc29bd76ce6aa9d58c771cd1599fbba8cf5057e7860b203710dd", + "sha256:a36c5c154f9d42ec176e6e620cb0dd275744aa1d804786a71ac37dc3661a5e95", + "sha256:a6a5cb8809091ec9ac03edde9304b3ad82ad4466333432b16d78ef40e0cce0d5", + "sha256:ae5e35a2c189d397b91034642cb0eab0e346f776ec2eb44a49a459e6615d6e2e", + "sha256:b0f7d4a3df8f06cf49f9f121bead236e328074de6449866515cea4907bbc63d6", + "sha256:b75110fb114fa366b29a027d0c9be3709579602ae111ff61674d28c93606acca", + "sha256:ba5e697569f84b13640c9e193170e89c13c6244c24400fc57e88724ef610cd31", + "sha256:be2a9b390f77fd7676d80bc3cdc4f8edb940d8c198ed2d8c0be1319018c778e1", + "sha256:ca1bd81f40adc59011f58159e4aa6445fc585a32bb8ac9badf7a2c1aa23822f2", + "sha256:d5d8555d9bfc3f02385c1c37e9f998e2011f0db4f90e250e5bc0c0a85a813085", + "sha256:e55e22ac0a30023426564b1059b035973ec82186ddddbac867078435801c7801", + "sha256:e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4", + "sha256:ecbb7b01409e9b782df5ded849c178a0aa7c906cf8c5a67368047daab282b184", + "sha256:ed01918d545a38998bfa5902c7c00e0fee90e957ce036a4000a88e3fe2264917", + "sha256:edabd457cd23a02965166026fd9bfd196f4324fe6032e866d0f3bd0301cd486f", + "sha256:fdf1c1dc5bafc32bc5d08b054f94d659422b05aba244d6be4ddc1c72d9aa70fb" + ], + "version": "==1.11.5" + }, "chardet": { "hashes": [ "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==3.0.4" }, "click": { "hashes": [ - "sha256:29f99fc6125fbc931b758dc053b3114e55c77a6e4c6c3a2674a2dc986016381d", - "sha256:f15516df478d5a56180fbf80e68f206010e6d160fc39fa508b65e035fd75130b" + "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", + "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7" ], - "version": "==6.7" + "markers": "python_version >= '3.6'", + "version": "==7.0" + }, + "cmarkgfm": { + "hashes": [ + "sha256:0186dccca79483e3405217993b83b914ba4559fe9a8396efc4eea56561b74061", + "sha256:1a625afc6f62da428df96ec325dc30866cc5781520cbd904ff4ec44cf018171c", + "sha256:207b7673ff4e177374c572feeae0e4ef33be620ec9171c08fd22e2b796e03e3d", + "sha256:275905bb371a99285c74931700db3f0c078e7603bed383e8cf1a09f3ee05a3de", + "sha256:50098f1c4950722521f0671e54139e0edc1837d63c990cf0f3d2c49607bb51a2", + "sha256:50ed116d0b60a07df0dc7b180c28569064b9d37d1578d4c9021cff04d725cb63", + "sha256:61a72def110eed903cd1848245897bcb80d295cd9d13944d4f9f30cba5b76655", + "sha256:64186fb75d973a06df0e6ea12879533b71f6e7ba1ab01ffee7fc3e7534758889", + "sha256:665303d34d7f14f10d7b0651082f25ebf7107f29ef3d699490cac16cdc0fc8ce", + "sha256:70b18f843aec58e4e64aadce48a897fe7c50426718b7753aaee399e72df64190", + "sha256:761ee7b04d1caee2931344ac6bfebf37102ffb203b136b676b0a71a3f0ea3c87", + "sha256:811527e9b7280b136734ed6cb6845e5fbccaeaa132ddf45f0246cbe544016957", + "sha256:987b0e157f70c72a84f3c2f9ef2d7ab0f26c08f2bf326c12c087ff9eebcb3ff5", + "sha256:9fc6a2183d0a9b0974ec7cdcdad42bd78a3be674cc3e65f87dd694419b3b0ab7", + "sha256:a3d17ee4ae739fe16f7501a52255c2e287ac817cfd88565b9859f70520afffea", + "sha256:ba5b5488719c0f2ced0aa1986376f7baff1a1653a8eb5fdfcf3f84c7ce46ef8d", + "sha256:c573ea89dd95d41b6d8cf36799c34b6d5b1eac4aed0212dee0f0a11fb7b01e8f", + "sha256:c5f1b9e8592d2c448c44e6bc0d91224b16ea5f8293908b1561de1f6d2d0658b1", + "sha256:cbe581456357d8f0674d6a590b1aaf46c11d01dd0a23af147a51a798c3818034", + "sha256:cf219bec69e601fe27e3974b7307d2f06082ab385d42752738ad2eb630a47d65", + "sha256:cf5014eb214d814a83a7a47407272d5db10b719dbeaf4d3cfe5969309d0fcf4b", + "sha256:d08bad67fa18f7e8ff738c090628ee0cbf0505d74a991c848d6d04abfe67b697", + "sha256:d6f716d7b1182bf35862b5065112f933f43dd1aa4f8097c9bcfb246f71528a34", + "sha256:e08e479102627641c7cb4ece421c6ed4124820b1758765db32201136762282d9", + "sha256:e20ac21418af0298437d29599f7851915497ce9f2866bc8e86b084d8911ee061", + "sha256:e25f53c37e319241b9a412382140dffac98ca756ba8f360ac7ab5e30cad9670a", + "sha256:e8932bddf159064f04e946fbb64693753488de21586f20e840b3be51745c8c09", + "sha256:f20900f16377f2109783ae9348d34bc80530808439591c3d3df73d5c7ef1a00c" + ], + "version": "==0.4.2" }, "colorama": { "hashes": [ "sha256:463f8483208e921368c9f306094eb6f725c6ca42b0f97e313cb5d5512459feda", "sha256:48eb22f4f8461b1df5734a074b57042430fb06e1d61bd1e11b078c0fe6d7a1f1" ], - "markers": "sys_platform == 'win32'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and sys_platform == 'win32'", "version": "==0.3.9" }, "coverage": { @@ -475,12 +569,14 @@ "sha256:e05cb4d9aad6233d67e0541caa7e511fa4047ed7750ec2510d466e806e0255d6", "sha256:f3f501f345f24383c0000395b26b726e46758b71393267aeae0bd36f8b3ade80" ], + "markers": "python_version < '4' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==4.5.1" }, "distlib": { "hashes": [ "sha256:cd502c66fc27c535bab62dc4f482e403e2369c2c05281a79cc2d4e2f42a87f20" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.2.7" }, "docutils": { @@ -498,7 +594,7 @@ "sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79", "sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1" ], - "markers": "python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '2.8' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.6" }, "execnet": { @@ -506,6 +602,7 @@ "sha256:a7a84d5fa07a089186a329528f127c9d73b9de57f1a1131b82bb5320ee651f6a", "sha256:fc155a6b553c66c838d1a22dba1dc9f5f505c43285a878c6f74a79c024750b83" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.5.0" }, "first": { @@ -513,6 +610,7 @@ "sha256:3bb3de3582cb27071cfb514f00ed784dc444b7f96dc21e140de65fe00585c95e", "sha256:41d5b64e70507d0c3ca742d68010a76060eea8a3d863e9b5130ab11a4a91aa0e" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.0.1" }, "funcsigs": { @@ -520,14 +618,28 @@ "sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca", "sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50" ], - "markers": "python_version < '3.0'", + "markers": "python_version < '3.0' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.0.2" }, + "future": { + "hashes": [ + "sha256:e39ced1ab767b5936646cedba8bcce582398233d6a627067d4c6a454c90cfedb" + ], + "version": "==0.16.0" + }, + "html5lib": { + "hashes": [ + "sha256:20b159aa3badc9d5ee8f5c647e5efd02ed2a66ab8d354930bd9ff139fc1dc0a3", + "sha256:66cb0dcfdbbc4f9c3ba1a63fdb511ffdbd4f513b2b6d81b80cd26ce6b3fb3736" + ], + "version": "==1.0.1" + }, "idna": { "hashes": [ "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.7" }, "imagesize": { @@ -541,7 +653,7 @@ "hashes": [ "sha256:b6ee7066fea66e35f8d0acee24d98006de1a0a8a94a8ce6efe73a9a23c8d9826" ], - "markers": "python_version < '2.7' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '2.7' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.0.4" }, "incremental": { @@ -556,7 +668,7 @@ "sha256:1ba23de573e9b95a8dcbd04fd026c40a64b77db0aadc48f28a844b4cb87479fe", "sha256:f4f195c9b17ea7d2b631a758451485c6b080975349b4adebe45ef4bb022db069" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.1.1" }, "invoke": { @@ -585,7 +697,7 @@ "sha256:2c85c1666649e92e56de17c00e1e831313602d9b55e8661d39c01e39003b45f7", "sha256:cc3dad264e36ed359fdd67c4588959d2996bd0402ad9c9d974ca906821537218" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '3.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.0.0" }, "more-itertools": { @@ -594,13 +706,14 @@ "sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e", "sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==4.3.0" }, "ordereddict": { "hashes": [ "sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f" ], - "markers": "python_version < '3.0'", + "markers": "python_version < '3.0' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1" }, "packagebuilder": { @@ -608,7 +721,7 @@ "sha256:1e85c4e0e994322996b93cd6685c12834d30f3558889154f8e3de8fb1f3fd1e7", "sha256:dc525d06ecd102db23ab421b879d7d27021d784ff933e33e8c411a53af5c9dbe" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.1.0" }, "packaging": { @@ -616,6 +729,7 @@ "sha256:e9215d2d2535d3ae866c3d6efc77d5b24a0192cce0ff20e42896cc0664f889c0", "sha256:f019b770dd64e585a99714f1fd5e01c7a8f11b45635aa953fd41c689a657375b" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==17.1" }, "parver": { @@ -637,7 +751,7 @@ "sha256:8eb170f8d0d61825e09a95b38be068299ddeda82f35e96c3301a8a5e7604cb83", "sha256:d1aa2a11ba7b8f7b21ab852b1fb5afb277e1bb99d5dfc663380b5015c0d80c5a" ], - "markers": "python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.3.2" }, "pip-shims": { @@ -645,7 +759,7 @@ "sha256:9c8a568b4a8ce4000a2982224f48a35736fca81214dfdb30dcae24287866a7e4", "sha256:ebc2bb29ddd21fa00c0cf28a5d8c725100f2f7ee98703aba237efd02e205c1c1" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.1.2" }, "pkginfo": { @@ -663,7 +777,7 @@ "sha256:c0e3553c1e581d8423daccbd825789c6e7f29b7d9e00e5331b12e1642a1a26d3", "sha256:dde5d525cf5f0cbad4d938c83b93db17887918daf63c13eafed257c4f61b07b4" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.2.2" }, "pluggy": { @@ -671,6 +785,7 @@ "sha256:6e3836e39f4d36ae72840833db137f7b7d35105079aee6ec4a62d9f80d594dd1", "sha256:95eb8364a4708392bae89035f45341871286a333f749c3141c20573d2b3876e1" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.7.1" }, "py": { @@ -678,8 +793,15 @@ "sha256:06a30435d058473046be836d3fc4f27167fd84c45b99704f2fb5509ef61f9af1", "sha256:50402e9d1c9005d759426988a492e0edaadb7f4e68bcddfea586bc7432d009c6" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.6.0" }, + "pycparser": { + "hashes": [ + "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3" + ], + "version": "==2.19" + }, "pygments": { "hashes": [ "sha256:78f3f434bcc5d6ee09020f92ba487f95ba50f1e3ef83ae96b9d5ffa1bab25c5d", @@ -692,6 +814,7 @@ "sha256:905d8090c335314568b5faee0025b1829f27bb974604a5762a6cdef3a7dfc3b7", "sha256:f493ee323be1e94929416b3585eefcc04943115cecbaaa35a8c86d1a2368af19" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.2.1" }, "pytest": { @@ -699,6 +822,7 @@ "sha256:0a72d8a9f559c006ba153e0c9b4838efd7b656cf1f993747ba7128770d6eb12c", "sha256:95529588ff4e85114a0b0ad8e9cf0131ca47d46b28230e25366c5aba66b1d854" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==3.8.1" }, "pytest-cov": { @@ -706,6 +830,7 @@ "sha256:513c425e931a0344944f84ea47f3956be0e416d95acbd897a44970c8d926d5d7", "sha256:e360f048b7dae3f2f2a9a4d067b2dd6b6a015d384d1577c994a43f3f7cbad762" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.6.0" }, "pytest-forked": { @@ -713,6 +838,7 @@ "sha256:e4500cd0509ec4a26535f7d4112a8cc0f17d3a41c29ffd4eab479d2a55b30805", "sha256:f275cb48a73fc61a6710726348e1da6d68a978f0ec0c54ece5a5fae5977e5a08" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.2" }, "pytest-timeout": { @@ -720,14 +846,16 @@ "sha256:1117fc0536e1638862917efbdc0895e6b62fa61e6cf4f39bb655686af7af9627", "sha256:b050a05da96a9992e90e884bc19b4790678b40c25471d2b77015b388417e1fa8" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.3.2" }, "pytest-xdist": { "hashes": [ - "sha256:0875deac20f6d96597036bdf63970887a6f36d28289c2f6682faf652dfea687b", - "sha256:28e25e79698b2662b648319d3971c0f9ae0e6500f88258ccb9b153c31110ba9b" + "sha256:029fbc816cee2887bf9ae6e0874256a8b707409eb54886e02df7cb1944ef101c", + "sha256:7118027c5bb7178fc46db223e67e8dd123b4e3f892b555cf68a66a252b3dc348" ], - "version": "==1.23.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.23.1" }, "pytz": { "hashes": [ @@ -736,11 +864,19 @@ ], "version": "==2018.5" }, + "readme-renderer": { + "hashes": [ + "sha256:237ca8705ffea849870de41101dba41543561da05c0ae45b2f1c547efa9843d2", + "sha256:f75049a3a7afa57165551e030dd8f9882ebf688b9600535a3f7e23596651875d" + ], + "version": "==22.0" + }, "requests": { "hashes": [ "sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1", "sha256:ec22d826a36ed72a7358ff3fe56cbd4ba69dd7a6718ffd450ff0e9df7a47ce6a" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.19.1" }, "requests-toolbelt": { @@ -755,6 +891,7 @@ "sha256:90151d8963f814e17190e067b60e92fb35fd1bc46c99f8dba3d7b0d93a3dd958", "sha256:c3aeaa4e0b80843ba65a68878293e07ea52a8d0706dbba86b02dad6cd20ef2dd" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.6" }, "resolvelib": { @@ -762,6 +899,7 @@ "sha256:6c4c6690b0bdd78bcc002e1a5d1b6abbde58c694a6ea1838f165b20d2c943db7", "sha256:8734e53271ef98f38a2c99324d5e7905bc00c97dc3fc5bb7d83c82a979e71c04" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.2.2" }, "scandir": { @@ -778,7 +916,7 @@ "sha256:c9009c527929f6e25604aec39b0a43c3f831d2947d89d6caaab22f057b7055c8", "sha256:f5c71e29b4e2af7ccdc03a020c626ede51da471173b4a6ad1e904f2b2e04b4bd" ], - "markers": "python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.9.0" }, "six": { @@ -786,6 +924,7 @@ "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" ], + "markers": "python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.11.0" }, "snowballstemmer": { @@ -821,6 +960,7 @@ "sha256:380178cde50a6a79f9d2cf6f42a62a5174febe5eea4126fe4038785f1d888d42", "sha256:a7901919d3e4f92ffba7ff40a9d697e35bbbc8a8049fe8da742f34c83606d957" ], + "markers": "python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.9.6" }, "tomlkit": { @@ -828,7 +968,7 @@ "sha256:8ab16e93162fc44d3ad83d2aa29a7140b8f7d996ae1790a73b9a7aed6fb504ac", "sha256:ca181cee7aee805d455628f7c94eb8ae814763769a93e69157f250fe4ebe1926" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.4.4" }, "towncrier": { @@ -847,10 +987,10 @@ }, "twine": { "hashes": [ - "sha256:08eb132bbaec40c6d25b358f546ec1dc96ebd2638a86eea68769d9e67fe2b129", - "sha256:2fd9a4d9ff0bcacf41fdc40c8cb0cfaef1f1859457c9653fd1b92237cc4e9f25" + "sha256:7d89bc6acafb31d124e6e5b295ef26ac77030bf098960c2a4c4e058335827c5c", + "sha256:fad6f1251195f7ddd1460cb76d6ea106c93adb4e56c41e0da79658e56e547d2c" ], - "version": "==1.11.0" + "version": "==1.12.1" }, "typing": { "hashes": [ @@ -858,7 +998,7 @@ "sha256:57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4", "sha256:a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a" ], - "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==3.6.6" }, "urllib3": { @@ -866,6 +1006,7 @@ "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf", "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5" ], + "markers": "python_version < '4' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.23" }, "virtualenv": { @@ -873,7 +1014,7 @@ "sha256:2ce32cd126117ce2c539f0134eb89de91a8413a29baac49cbab3eb50e2026669", "sha256:ca07b4c0b54e14a91af9f34d0919790b016923d157afda5efdde55c96718f752" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==16.0.0" }, "vistir": { @@ -884,14 +1025,22 @@ "sha256:8a360ac20cbcc0863d6dbbe7a52e8b2c9ebf48abd6833c3813a82c70708244af", "sha256:bc6e10284792485c10585536e6aede9e38996c841cc9d2a67238cd05742c2d0b" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.1.6" }, + "webencodings": { + "hashes": [ + "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78", + "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923" + ], + "version": "==0.5.1" + }, "wheel": { "hashes": [ "sha256:0a2e54558a0628f2145d2fc822137e322412115173e8a2ddbe1c9024338ae83c", "sha256:80044e51ec5bbf6c894ba0bc48d26a8c20a9ba629f4ca19ea26ecfcf87685f5f" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.31.1" }, "yaspin": { @@ -899,7 +1048,7 @@ "sha256:36fdccc5e0637b5baa8892fe2c3d927782df7d504e9020f40eb2c1502518aa5a", "sha256:8e52bf8079a48e2a53f3dfeec9e04addb900c101d1591c85df69cf677d3237e7" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.14.0" } } diff --git a/src/passa/internals/specifiers.py b/src/passa/internals/specifiers.py index 3f2ae8c..76946cf 100644 --- a/src/passa/internals/specifiers.py +++ b/src/passa/internals/specifiers.py @@ -143,30 +143,6 @@ def cleanup_pyspecs(specs, joiner="or"): return results -@lru_cache(maxsize=128) -def pyspec_from_markers(marker): - if marker._markers[0][0] != 'python_version': - return - op = marker._markers[0][1].value - version = marker._markers[0][2].value - specset = set() - if op == "in": - specset.update( - Specifier("=={0}".format(v.strip())) - for v in version.split(",") - ) - elif op == "not in": - specset.update( - Specifier("!={0}".format(v.strip())) - for v in version.split(",") - ) - else: - specset.add(Specifier("".join([op, version]))) - if specset: - return specset - return None - - def gen_marker(mkr): m = Marker("python_version == '1'") m._markers.pop() @@ -175,6 +151,12 @@ def gen_marker(mkr): class PySpecs(Set): + + MAX_VERSIONS = { + 2: 7, + 3: 9 + } + def __init__(self, specs=None): if not specs: specs = SpecifierSet() @@ -210,6 +192,16 @@ def __iter__(self): yield version return + @classmethod + def get_versions(cls): + major_versions = list(cls.MAX_VERSIONS.keys()) + versions = ( + "{0}.{1}".format(major, minor) for major in major_versions + for minor in range(cls.MAX_VERSIONS[major] + 1) + ) + versions = (packaging.version.parse(v) for v in versions) + return versions + def clean(self): if len(set(self.specifierset)) == 1: spec = next(iter(spec for spec in self.specifierset), None) @@ -305,13 +297,22 @@ def __nonzero__(self): # Python 2. @lru_cache(maxsize=128) def __and__(self, other): + # Unintuitive perhaps, but this is for "x or y" and needs to handle the + # widest possible range encapsulated by the two using the intersection if not isinstance(other, PySpecs): specset = PySpecs(other) if self == other: return self new_specset = SpecifierSet() diff_specset = SpecifierSet() - intersection = set(self.as_specset) & set(other.as_specset) + own_versions = [v for v in self.get_versions() if v in self.specifierset] + other_versions = [v for v in self.get_versions() if v in other.specifierset] + intersection = set(own_versions) & set(other_versions) + min_included = None + max_included = None + if intersection: + max_included = max(intersection) + min_included = min(intersection) diff_specset._specs = frozenset(set(self.as_specset) ^ set(other.as_specset)) tuples = cleanup_pyspecs(diff_specset, joiner="or") new_marker_str = " and ".join( From a9b8bfb3e6b01d0ecf9c6917db581a931937cfa6 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 27 Sep 2018 01:13:42 -0400 Subject: [PATCH 32/32] Finalize marker intersection logic Signed-off-by: Dan Ryan --- src/passa/internals/specifiers.py | 174 ++++++++++++++++++++++++------ 1 file changed, 140 insertions(+), 34 deletions(-) diff --git a/src/passa/internals/specifiers.py b/src/passa/internals/specifiers.py index 76946cf..14654e8 100644 --- a/src/passa/internals/specifiers.py +++ b/src/passa/internals/specifiers.py @@ -7,16 +7,16 @@ import six -from cached_property import cached_property from packaging.markers import Marker from packaging.specifiers import Specifier, SpecifierSet +import packaging.version from vistir.misc import dedup - six.add_move(six.MovedAttribute("Set", "collections", "collections.abc")) from six.moves import reduce, Set + try: from functools import lru_cache except ImportError: @@ -55,6 +55,13 @@ def _format_pyspec(specifier): next_tuple = (curr_tuple[0], curr_tuple[1] + 1) except IndexError: next_tuple = (curr_tuple[0], 1) + if not next_tuple[1] <= PySpecs.MAX_VERSIONS[next_tuple[0]]: + if (specifier.operator == "<" + and next_tuple[1] - 1 <= PySpecs.MAX_VERSIONS[next_tuple[0]]): + op = "<=" + next_tuple = (next_tuple[0], next_tuple[1] - 1) + else: + return specifier specifier = Specifier("{0}{1}".format(op, _format_version(next_tuple))) return specifier @@ -143,6 +150,49 @@ def cleanup_pyspecs(specs, joiner="or"): return results +def fix_version_tuple(version_tuple): + op, version = version_tuple + max_allowed = PySpecs.MAX_VERSIONS[version[0]] + if op == "<" and version[1] > max_allowed and version[1] - 1 <= max_allowed: + op = "<=" + version = (version[0], version[1] - 1) + return (op, version) + + +@lru_cache(maxsize=128) +def get_versions(specset, group_by_operator=True): + specs = [_get_specs(x) for x in list(tuple(specset))] + initial_sort_key = lambda k: (k[0], k[1]) + initial_grouping_key = operator.itemgetter(0) + if not group_by_operator: + initial_grouping_key = operator.itemgetter(1) + initial_sort_key = operator.itemgetter(1) + version_tuples = sorted( + set((op, version) for spec in specs for op, version in spec), + key=initial_sort_key + ) + version_tuples = [fix_version_tuple(t) for t in version_tuples] + op_groups = [ + (grp, list(map(operator.itemgetter(1), keys))) + for grp, keys in itertools.groupby(version_tuples, key=initial_grouping_key) + ] + versions = [ + (op, packaging.version.parse(".".join(str(v) for v in val))) + for op, vals in op_groups for val in vals + ] + return versions + + +def get_next_python(version): + version_index = ALL_PYTHON_VERSIONS.index(version) + return ALL_PYTHON_VERSIONS[version_index + 1] + + +def get_previous_python(version): + version_index = ALL_PYTHON_VERSIONS.index(version) + return ALL_PYTHON_VERSIONS[version_index - 1] + + def gen_marker(mkr): m = Marker("python_version == '1'") m._markers.pop() @@ -192,16 +242,6 @@ def __iter__(self): yield version return - @classmethod - def get_versions(cls): - major_versions = list(cls.MAX_VERSIONS.keys()) - versions = ( - "{0}.{1}".format(major, minor) for major in major_versions - for minor in range(cls.MAX_VERSIONS[major] + 1) - ) - versions = (packaging.version.parse(v) for v in versions) - return versions - def clean(self): if len(set(self.specifierset)) == 1: spec = next(iter(spec for spec in self.specifierset), None) @@ -295,36 +335,89 @@ def __bool__(self): def __nonzero__(self): # Python 2. return self.__bool__() + def get_versions(self, group_by_operator=True): + return get_versions(self.specifierset, group_by_operator=group_by_operator) + + def get_versions_in_specset(self): + return set([v[1] for v in self.get_versions() if v[1] in self.specifierset]) + + def get_version_excludes(self): + return set(v[1] for v in self.get_versions() if v[1] not in self.specifierset) + + def group_specs(self, specs=None, handle_exclusions=True): + if not specs: + specs = self.get_versions(group_by_operator=handle_exclusions) + else: + specs = get_versions(specs, group_by_operator=handle_exclusions) + pyversions = enumerate(self.get_versions(group_by_operator=handle_exclusions)) + + def get_version(v): + return ALL_PYTHON_VERSIONS.index(v[1][1]) + + excludes = set() + ranges = set() + + # group the versions on their index from ALL_PYTHON_VERSIONS - their index here + # consecutive elements will share a group, e.g. + # ALL_PYTHON_VERSIONS.index(parse_version("2.7")) == 7, 3.0 == 8, 3.1 == 9 + # if 2.7 is element 1, (7 - 1) = 6, if 3.0 is element 2, (8 - 2) = 6 + # and they will share a group (i.e. they are consecutive) + for k, grp in itertools.groupby(pyversions, lambda t: get_version(t) - t[0]): + version_group = list(grp) + op = next(iter(v[1][0] for v in version_group), None) + _versions = [v[1][1] for v in version_group] + if op == "!=": + excludes.update(set(_versions)) + else: + min_ = min(_versions) + max_ = max(_versions) + if len(_versions) == 1 or str(min_) == str(max_): + ranges.add((min_,)) + else: + ranges.add((min_, max_)) + return ranges, excludes + + def create_specset_from_ranges(self, specset=None, ranges=None, excludes=None): + group_args = {"handle_exclusions": False} + if specset: + group_args["specs"] = specset + if not ranges: + ranges, _ = self.group_specs(**group_args) + if not excludes: + group_args["handle_exclusions"] = True + _, excludes = self.group_specs(**group_args) + spec_ranges = set() + for range_ in ranges: + if len(range_) == 1: + spec_ranges.add(Specifier("=={0}".format(str(next(iter(range_)))))) + else: + min_, max_ = range_ + if min_ == max_: + spec_ranges.add("<={0}").format(min_) + else: + spec_ranges.add(Specifier(">={0}".format(str(min_)))) + spec_ranges.add(Specifier("<={0}".format(str(max_)))) + for exclude in excludes: + spec_ranges.add(Specifier("!={0}".format(str(exclude)))) + new_specset = SpecifierSet() + new_specset._specs = frozenset(spec_ranges) + @lru_cache(maxsize=128) def __and__(self, other): # Unintuitive perhaps, but this is for "x or y" and needs to handle the # widest possible range encapsulated by the two using the intersection if not isinstance(other, PySpecs): - specset = PySpecs(other) + other = PySpecs(other) if self == other: return self new_specset = SpecifierSet() - diff_specset = SpecifierSet() - own_versions = [v for v in self.get_versions() if v in self.specifierset] - other_versions = [v for v in self.get_versions() if v in other.specifierset] - intersection = set(own_versions) & set(other_versions) - min_included = None - max_included = None - if intersection: - max_included = max(intersection) - min_included = min(intersection) - diff_specset._specs = frozenset(set(self.as_specset) ^ set(other.as_specset)) - tuples = cleanup_pyspecs(diff_specset, joiner="or") - new_marker_str = " and ".join( - "python_version {0} '{1}'".format(op, val) - for op, val in tuples - ) - specset = set() - if new_marker_str: - marker = Marker(new_marker_str) - specset = set(PySpecs.from_marker(marker).as_specset) - specset = frozenset(specset | intersection) - new_specset._specs = specset + own_ranges, own_excludes = self.group_specs() + other_ranges, other_excludes = other.group_specs() + # In order to do an "or" propertly we need to intersect the "good" versions + intersection = self.get_versions_in_specset() & other.get_versions_in_specset() + # And then we need to union the "bad" versions + excludes = own_excludes | other_excludes + new_specset = self.create_specset_from_ranges(ranges=intersection, excludes=excludes) return PySpecs(new_specset) @lru_cache(maxsize=128) @@ -379,3 +472,16 @@ def from_marker(cls, marker): newset = cls(specifierset) return newset return None + + +def get_all_python_versions(): + major_versions = list(PySpecs.MAX_VERSIONS.keys()) + versions = ( + "{0}.{1}".format(major, minor) for major in major_versions + for minor in range(PySpecs.MAX_VERSIONS[major] + 1) + ) + versions = (packaging.version.parse(v) for v in versions) + return versions + + +ALL_PYTHON_VERSIONS = sorted(get_all_python_versions())