diff --git a/LICENSE b/LICENSE index e1a278e..cd41e27 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2018, Dan Ryan +Copyright (c) 2018, Dan Ryan and Tzu-ping Chung Permission to use, copy, modify, and distribute this software for any purpose with or without fee is hereby granted, provided that the above diff --git a/Pipfile.lock b/Pipfile.lock index ff1f65c..c83e057 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -24,11 +24,11 @@ }, "attrs": { "hashes": [ - "sha256:4b90b09eeeb9b88c35bc642cbac057e45a5fd85367b985bd2809c62b7b939265", - "sha256:e0d0eb91441a3b53dab4d9b743eafc1ac44476296a2053b6ca3af0b139faf87b" + "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", + "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '3.6'", - "version": "==18.1.0" + "version": "==18.2.0" }, "backports-shutil-get-terminal-size": { "hashes": [ @@ -115,22 +115,6 @@ "markers": "python_version >= '2.6' and python_version >= '3.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.0.0" }, - "more-itertools": { - "hashes": [ - "sha256:c187a73da93e7a8acc0001572aebc7e3c69daf7bf6881a2cea10650bd4420092", - "sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e", - "sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==4.3.0" - }, - "ordereddict": { - "hashes": [ - "sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f" - ], - "markers": "python_version < '3.0' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.1" - }, "packaging": { "hashes": [ "sha256:e9215d2d2535d3ae866c3d6efc77d5b24a0192cce0ff20e42896cc0664f889c0", @@ -149,7 +133,7 @@ "sha256:8eb170f8d0d61825e09a95b38be068299ddeda82f35e96c3301a8a5e7604cb83", "sha256:d1aa2a11ba7b8f7b21ab852b1fb5afb277e1bb99d5dfc663380b5015c0d80c5a" ], - "markers": "python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.6' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.3.2" }, "pip-shims": { @@ -189,19 +173,19 @@ }, "requirementslib": { "hashes": [ - "sha256:7288ec80fa62b78c6193eca85882605c4cdbb60361fc62478bd4a913c0b54400", - "sha256:8bb1b156d8b13337eab359273605b036d60735b350019addabb054e8a89a96f6" + "sha256:90151d8963f814e17190e067b60e92fb35fd1bc46c99f8dba3d7b0d93a3dd958", + "sha256:c3aeaa4e0b80843ba65a68878293e07ea52a8d0706dbba86b02dad6cd20ef2dd" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.1.5" + "version": "==1.1.6" }, "resolvelib": { "hashes": [ - "sha256:d52f2c0762deeb2a4cc34a84371a7a5ac85e111bdc69ce9ae729d8d636606ad6", - "sha256:eb759d43bbf50de9bf36afb9f6c269fabf9ff49084dbfad4ba67252d134bf4b5" + "sha256:6c4c6690b0bdd78bcc002e1a5d1b6abbde58c694a6ea1838f165b20d2c943db7", + "sha256:8734e53271ef98f38a2c99324d5e7905bc00c97dc3fc5bb7d83c82a979e71c04" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.2.1" + "version": "==0.2.2" }, "scandir": { "hashes": [ @@ -217,7 +201,7 @@ "sha256:c9009c527929f6e25604aec39b0a43c3f831d2947d89d6caaab22f057b7055c8", "sha256:f5c71e29b4e2af7ccdc03a020c626ede51da471173b4a6ad1e904f2b2e04b4bd" ], - "markers": "python_version < '3.5' and python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version < '3.6' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.9.0" }, "six": { @@ -225,7 +209,7 @@ "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" ], - "markers": "python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.6' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.11.0" }, "toml": { @@ -237,11 +221,11 @@ }, "tomlkit": { "hashes": [ - "sha256:618b057b96b488bb858102bdb38dd9d451c0617667031e320efea2392a39bec4", - "sha256:6856b5395f7c509baad1911aa3940b3c2fb33f53aff968ed1596c393ccea98e5" + "sha256:8ab16e93162fc44d3ad83d2aa29a7140b8f7d996ae1790a73b9a7aed6fb504ac", + "sha256:ca181cee7aee805d455628f7c94eb8ae814763769a93e69157f250fe4ebe1926" ], "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.4.3" + "version": "==0.4.4" }, "typing": { "hashes": [ @@ -278,6 +262,14 @@ ], "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.31.1" + }, + "yaspin": { + "hashes": [ + "sha256:36fdccc5e0637b5baa8892fe2c3d927782df7d504e9020f40eb2c1502518aa5a", + "sha256:8e52bf8079a48e2a53f3dfeec9e04addb900c101d1591c85df69cf677d3237e7" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.14.0" } }, "develop": { @@ -331,11 +323,11 @@ }, "attrs": { "hashes": [ - "sha256:4b90b09eeeb9b88c35bc642cbac057e45a5fd85367b985bd2809c62b7b939265", - "sha256:e0d0eb91441a3b53dab4d9b743eafc1ac44476296a2053b6ca3af0b139faf87b" + "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", + "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '3.6'", - "version": "==18.1.0" + "version": "==18.2.0" }, "babel": { "hashes": [ @@ -502,11 +494,11 @@ }, "imagesize": { "hashes": [ - "sha256:3620cc0cadba3f7475f9940d22431fc4d407269f1be59ec9b8edcca26440cf18", - "sha256:5b326e4678b6925158ccc66a9fa3122b6106d7c876ee32d7de6ce59385b96315" + "sha256:3f349de3eb99145973fefb7dbe38554414e5c30abd0c8e4b970a7c9d09f3a1d8", + "sha256:f3832918bc3c66617f92e35f5d70729187676313caa60c187eb0f28b8fe5e3b5" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.0.0" + "version": "==1.1.0" }, "importlib": { "hashes": [ @@ -524,11 +516,11 @@ }, "invoke": { "hashes": [ - "sha256:1c2cf54c9b9af973ad9704d8ba81b225117cab612568cacbfb3fc42958cc20a9", - "sha256:334495ea16e73948894e9535019f87a88a44b73e7977492b12c2d1b5085f8197", - "sha256:54bdd3fd0245abd1185e05359fd2e4f26be0657cfe7d7bb1bed735e054fa53ab" + "sha256:4f4de934b15c2276caa4fbc5a3b8a61c0eb0b234f2be1780d2b793321995c2d6", + "sha256:dc492f8f17a0746e92081aec3f86ae0b4750bf41607ea2ad87e5a7b5705121b7", + "sha256:eb6f9262d4d25b40330fb21d1e99bf0f85011ccc3526980f8a3eaedd4b43892e" ], - "version": "==1.1.1" + "version": "==1.2.0" }, "jinja2": { "hashes": [ @@ -596,7 +588,7 @@ "sha256:8eb170f8d0d61825e09a95b38be068299ddeda82f35e96c3301a8a5e7604cb83", "sha256:d1aa2a11ba7b8f7b21ab852b1fb5afb277e1bb99d5dfc663380b5015c0d80c5a" ], - "markers": "python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.6' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.3.2" }, "pip-shims": { @@ -659,19 +651,19 @@ }, "pytest": { "hashes": [ - "sha256:2d7c49e931316cc7d1638a3e5f54f5d7b4e5225972b3c9838f3584788d27f349", - "sha256:ad0c7db7b5d4081631e0155f5c61b80ad76ce148551aaafe3a718d65a7508b18" + "sha256:453cbbbe5ce6db38717d282b758b917de84802af4288910c12442984bde7b823", + "sha256:a8a07f84e680482eb51e244370aaf2caa6301ef265f37c2bdefb3dd3b663f99d" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==3.7.4" + "version": "==3.8.0" }, "pytest-cov": { "hashes": [ - "sha256:03aa752cf11db41d281ea1d807d954c4eda35cfa1b21d6971966cc041bbf6e2d", - "sha256:890fe5565400902b0c78b5357004aab1c814115894f4f21370e2433256a3eeec" + "sha256:513c425e931a0344944f84ea47f3956be0e416d95acbd897a44970c8d926d5d7", + "sha256:e360f048b7dae3f2f2a9a4d067b2dd6b6a015d384d1577c994a43f3f7cbad762" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.5.1" + "version": "==2.6.0" }, "pytest-forked": { "hashes": [ @@ -697,12 +689,6 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.23.0" }, - "pytoml": { - "hashes": [ - "sha256:42f76a696182570e93581e763da033c1de484973bad82458387b6dee6c184cbc" - ], - "version": "==0.1.19" - }, "pytz": { "hashes": [ "sha256:a061aa0a9e06881eb8b3b2b43f05b9439d6583c206d0a6c340ff72a7b6669053", @@ -728,19 +714,19 @@ }, "requirementslib": { "hashes": [ - "sha256:7288ec80fa62b78c6193eca85882605c4cdbb60361fc62478bd4a913c0b54400", - "sha256:8bb1b156d8b13337eab359273605b036d60735b350019addabb054e8a89a96f6" + "sha256:90151d8963f814e17190e067b60e92fb35fd1bc46c99f8dba3d7b0d93a3dd958", + "sha256:c3aeaa4e0b80843ba65a68878293e07ea52a8d0706dbba86b02dad6cd20ef2dd" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.1.5" + "version": "==1.1.6" }, "resolvelib": { "hashes": [ - "sha256:d52f2c0762deeb2a4cc34a84371a7a5ac85e111bdc69ce9ae729d8d636606ad6", - "sha256:eb759d43bbf50de9bf36afb9f6c269fabf9ff49084dbfad4ba67252d134bf4b5" + "sha256:6c4c6690b0bdd78bcc002e1a5d1b6abbde58c694a6ea1838f165b20d2c943db7", + "sha256:8734e53271ef98f38a2c99324d5e7905bc00c97dc3fc5bb7d83c82a979e71c04" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.2.1" + "version": "==0.2.2" }, "scandir": { "hashes": [ @@ -756,7 +742,7 @@ "sha256:c9009c527929f6e25604aec39b0a43c3f831d2947d89d6caaab22f057b7055c8", "sha256:f5c71e29b4e2af7ccdc03a020c626ede51da471173b4a6ad1e904f2b2e04b4bd" ], - "markers": "python_version < '3.5' and python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version < '3.6' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.9.0" }, "six": { @@ -764,7 +750,7 @@ "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" ], - "markers": "python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.6' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.11.0" }, "snowballstemmer": { @@ -807,11 +793,11 @@ }, "tomlkit": { "hashes": [ - "sha256:618b057b96b488bb858102bdb38dd9d451c0617667031e320efea2392a39bec4", - "sha256:6856b5395f7c509baad1911aa3940b3c2fb33f53aff968ed1596c393ccea98e5" + "sha256:8ab16e93162fc44d3ad83d2aa29a7140b8f7d996ae1790a73b9a7aed6fb504ac", + "sha256:ca181cee7aee805d455628f7c94eb8ae814763769a93e69157f250fe4ebe1926" ], "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.4.3" + "version": "==0.4.4" }, "towncrier": { "hashes": [ @@ -870,6 +856,14 @@ ], "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.31.1" + }, + "yaspin": { + "hashes": [ + "sha256:36fdccc5e0637b5baa8892fe2c3d927782df7d504e9020f40eb2c1502518aa5a", + "sha256:8e52bf8079a48e2a53f3dfeec9e04addb900c101d1591c85df69cf677d3237e7" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.14.0" } } } diff --git a/news/53.feature.rst b/news/53.feature.rst new file mode 100644 index 0000000..2589034 --- /dev/null +++ b/news/53.feature.rst @@ -0,0 +1 @@ +Refactored and restructured the internals for improved organization and separation of concerns. diff --git a/setup.cfg b/setup.cfg index 98ea198..4daf9ee 100644 --- a/setup.cfg +++ b/setup.cfg @@ -45,6 +45,7 @@ install_requires = resolvelib>=0.2.1,!=1.0.0.dev0 requirementslib>=1.1.1 six + virtualenv vistir[spinner]>=0.1.4 [options.extras_require] @@ -52,6 +53,7 @@ pack = invoke parver tests = + cached-property pytest-xdist pytest-timeout pytest-cov diff --git a/src/passa/actions/__init__.py b/src/passa/actions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/passa/actions/add.py b/src/passa/actions/add.py new file mode 100644 index 0000000..3efc4dc --- /dev/null +++ b/src/passa/actions/add.py @@ -0,0 +1,57 @@ +# -*- coding=utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + +import itertools +import sys + + +def add_packages(packages=[], editables=[], project=None, dev=False, sync=False, clean=False): + from passa.models.lockers import PinReuseLocker + from passa.operations.lock import lock + + lines = list(itertools.chain( + packages, + ("-e {}".format(e) for e in editables), + )) + + project = project + for line in lines: + try: + project.add_line_to_pipfile(line, develop=dev) + except (TypeError, ValueError) as e: + print("Cannot add {line!r} to Pipfile: {error}".format( + line=line, error=str(e), + ), file=sys.stderr) + return 2 + + prev_lockfile = project.lockfile + + locker = PinReuseLocker(project) + success = lock(locker) + if not success: + return 1 + + project._p.write() + project._l.write() + print("Written to project at", project.root) + + if not sync: + return + + from passa.models.synchronizers import Synchronizer + from passa.operations.sync import sync + + lockfile_diff = project.difference_lockfile(prev_lockfile) + default = any(lockfile_diff.default) + develop = any(lockfile_diff.develop) + + syncer = Synchronizer( + project, default=default, develop=develop, + clean_unneeded=clean + ) + success = sync(syncer) + if not success: + return 1 + + print("Synchronized project at", project.root) diff --git a/src/passa/actions/clean.py b/src/passa/actions/clean.py new file mode 100644 index 0000000..9006f22 --- /dev/null +++ b/src/passa/actions/clean.py @@ -0,0 +1,17 @@ +# -*- coding=utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + + +def clean(project, default=True, dev=False, sync=True): + from passa.models.synchronizers import Cleaner + from passa.operations.sync import clean + + cleaner = Cleaner(project, default=default, develop=dev, sync=sync) + + success = clean(cleaner) + if not success: + return 1 + + if sync: + print("Cleaned project at", project.root) diff --git a/src/passa/actions/freeze.py b/src/passa/actions/freeze.py new file mode 100644 index 0000000..ca4dbb2 --- /dev/null +++ b/src/passa/actions/freeze.py @@ -0,0 +1,93 @@ +# -*- coding=utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + +import contextlib +import io +import itertools +import sys + +import vistir.misc + + +def _source_as_lines(source, extra): + url = source["url"] + if extra: + lines = ["--extra-index-url {}".format(url)] + else: + lines = ["--index-url {}".format(url)] + if not source.get("verify_ssl", True): + lines = ["--trusted-host {}".format(url)] + return lines + + +def _requirement_as_line(requirement, sources, include_hashes): + if requirement.index: + sources = sources + else: + sources = None + line = vistir.misc.to_text( + requirement.as_line(sources=sources, include_hashes=include_hashes) + ) + return line + + +@contextlib.contextmanager +def open_for_output(filename): + if filename is None: + yield sys.stdout + return + with io.open(filename, "w", encoding="utf-8", newline="\n") as f: + yield f + + +def freeze(project=None, default=True, dev=True, include_hashes=None, target=None): + from requirementslib import Requirement + + lockfile = project.lockfile + if not lockfile: + print("Pipfile.lock is required to export.", file=sys.stderr) + return 1 + + section_names = [] + if default: + section_names.append("default") + if dev: + section_names.append("develop") + requirements = [ + Requirement.from_pipfile(key, entry._data) + for key, entry in itertools.chain.from_iterable( + lockfile.get(name, {}).items() + for name in section_names + ) + ] + + if include_hashes is None: + include_hashes = all(r.is_named for r in requirements) + + sources = lockfile.meta.sources._data + + source_lines = list(vistir.misc.dedup(itertools.chain( + itertools.chain.from_iterable( + _source_as_lines(source, False) + for source in sources[:1] + ), + itertools.chain.from_iterable( + _source_as_lines(source, True) + for source in sources[1:] + ), + ))) + + requirement_lines = sorted(vistir.misc.dedup( + _requirement_as_line(requirement, sources, include_hashes) + for requirement in requirements + )) + + with open_for_output(target) as f: + for line in source_lines: + f.write(line) + f.write("\n") + f.write("\n") + for line in requirement_lines: + f.write(line) + f.write("\n") diff --git a/src/passa/actions/init.py b/src/passa/actions/init.py new file mode 100644 index 0000000..bbab009 --- /dev/null +++ b/src/passa/actions/init.py @@ -0,0 +1,59 @@ +# -*- coding=utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + +import io +import os +from pip_shims import Command as PipCommand, cmdoptions +import plette +import six +import vistir + + +class PipCmd(PipCommand): + name = "PipCmd" + + +def get_sources(urls, trusted_hosts): + trusted_hosts = [six.moves.urllib.parse.urlparse(url).netloc for url in trusted_hosts] + sources = [] + for url in urls: + parsed_url = six.moves.urllib.parse.urlparse(url) + netloc = parsed_url.netloc + if '@' in netloc: + _, _, netloc = netloc.rpartition('@') + name, _, _ = netloc.partition('.') # Just use the domain name as the source name + verify_ssl = True + if netloc in trusted_hosts: + verify_ssl = False + sources.append({"url": url, "name": name, "verify_ssl": verify_ssl}) + return sources + + +def init_project(root=None, python_version=None): + pipfile_path = os.path.join(root, "Pipfile") + if os.path.isfile(pipfile_path): + raise RuntimeError("{0!r} is already a Pipfile project".format(root)) + if not os.path.exists(root): + vistir.path.mkdir_p(root, mode=0o755) + pip_command = PipCmd() + cmdoptions.make_option_group(cmdoptions.index_group, pip_command.parser) + parsed, _ = pip_command.parser.parse_args([]) + index_urls = [parsed.index_url] + parsed.extra_index_urls + sources = get_sources(index_urls, parsed.trusted_hosts) + data = { + "source": sources, + "packages": {}, + "dev-packages": {}, + } + if python_version: + data["requires"] = {"python_version": python_version} + return create_project(pipfile_path=pipfile_path, data=data) + + +def create_project(pipfile_path, data={}): + pipfile = plette.pipfiles.Pipfile(data=data) + with io.open(pipfile_path, "w") as fh: + pipfile.dump(fh) + print("Successfully created new pipfile at {0!r}".format(pipfile_path)) + return 0 diff --git a/src/passa/actions/install.py b/src/passa/actions/install.py new file mode 100644 index 0000000..1728dae --- /dev/null +++ b/src/passa/actions/install.py @@ -0,0 +1,32 @@ +# -*- coding=utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + + +def install(project=None, check=True, dev=False, clean=True): + from passa.models.lockers import BasicLocker + from passa.operations.lock import lock + + project = project + + if not check or not project.is_synced(): + locker = BasicLocker(project) + success = lock(locker) + if not success: + return 1 + project._l.write() + print("Written to project at", project.root) + + from passa.models.synchronizers import Synchronizer + from passa.operations.sync import sync + + syncer = Synchronizer( + project, default=True, develop=dev, + clean_unneeded=clean, + ) + + success = sync(syncer) + if not success: + return 1 + + print("Synchronized project at", project.root) diff --git a/src/passa/actions/lock.py b/src/passa/actions/lock.py new file mode 100644 index 0000000..7c09469 --- /dev/null +++ b/src/passa/actions/lock.py @@ -0,0 +1,17 @@ +# -*- coding=utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + + +def lock(project=None): + from passa.models.lockers import BasicLocker + from passa.operations.lock import lock + + project = project + locker = BasicLocker(project) + success = lock(locker) + if not success: + return + + project._l.write() + print("Written to project at", project.root) diff --git a/src/passa/actions/remove.py b/src/passa/actions/remove.py new file mode 100644 index 0000000..17ba1c7 --- /dev/null +++ b/src/passa/actions/remove.py @@ -0,0 +1,38 @@ +# -*- coding=utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + + +def remove(project=None, only="default", packages=[], clean=True, sync=False): + from passa.models.lockers import PinReuseLocker + from passa.operations.lock import lock + + default = (only != "dev") + develop = (only != "default") + + project = project + project.remove_keys_from_pipfile( + packages, default=default, develop=develop, + ) + + locker = PinReuseLocker(project) + success = lock(locker) + if not success: + return 1 + + project._p.write() + project._l.write() + print("Written to project at", project.root) + + if not clean: + return + + from passa.models.synchronizers import Cleaner + from passa.operations.sync import clean + + cleaner = Cleaner(project, default=True, develop=True) + success = clean(cleaner) + if not success: + return 1 + + print("Cleaned project at", project.root) diff --git a/src/passa/actions/sync.py b/src/passa/actions/sync.py new file mode 100644 index 0000000..23e36ee --- /dev/null +++ b/src/passa/actions/sync.py @@ -0,0 +1,20 @@ +# -*- coding=utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + + +def sync(project=None, dev=False, clean=True): + from passa.models.synchronizers import Synchronizer + from passa.operations.sync import sync + + project = project + syncer = Synchronizer( + project, default=True, develop=dev, + clean_unneeded=clean, + ) + + success = sync(syncer) + if not success: + return 1 + + print("Synchronized project at", project.root) diff --git a/src/passa/actions/upgrade.py b/src/passa/actions/upgrade.py new file mode 100644 index 0000000..fb3ad7f --- /dev/null +++ b/src/passa/actions/upgrade.py @@ -0,0 +1,52 @@ +# -*- coding=utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + +import sys + + +def upgrade(project=None, strategy="only-if-needed", sync=True, packages=[]): + from passa.models.lockers import EagerUpgradeLocker, PinReuseLocker + from passa.operations.lock import lock + + for package in packages: + if not project.contains_key_in_pipfile(package): + print("{package!r} not found in Pipfile".format( + package=package, + ), file=sys.stderr) + return 2 + + project.remove_keys_from_lockfile(packages) + + prev_lockfile = project.lockfile + + if strategy == "eager": + locker = EagerUpgradeLocker(project, packages) + else: + locker = PinReuseLocker(project) + success = lock(locker) + if not success: + return 1 + + project._l.write() + print("Written to project at", project.root) + + if not sync: + return + + from passa.operations.sync import sync + from passa.models.synchronizers import Synchronizer + + lockfile_diff = project.difference_lockfile(prev_lockfile) + default = bool(any(lockfile_diff.default)) + develop = bool(any(lockfile_diff.develop)) + + syncer = Synchronizer( + project, default=default, develop=develop, + clean_unneeded=False, + ) + success = sync(syncer) + if not success: + return 1 + + print("Synchronized project at", project.root) diff --git a/src/passa/cli/__init__.py b/src/passa/cli/__init__.py index cb503e8..b6891d3 100644 --- a/src/passa/cli/__init__.py +++ b/src/passa/cli/__init__.py @@ -34,7 +34,7 @@ def main(argv=None): continue parser = subparsers.add_parser(klass.name, help=klass.description) command = klass(parser) - parser.set_defaults(func=command.main) + parser.set_defaults(func=command.run) options = root_parser.parse_args(argv) diff --git a/src/passa/cli/_base.py b/src/passa/cli/_base.py index 68e0e34..0ca4868 100644 --- a/src/passa/cli/_base.py +++ b/src/passa/cli/_base.py @@ -6,31 +6,7 @@ import os import sys -import tomlkit.exceptions - - -def build_project(root): - # This is imported lazily to reduce import overhead. Not evey command - # needs the project instance. - from passa.internals.projects import Project - root = os.path.abspath(root) - if not os.path.isfile(os.path.join(root, "Pipfile")): - raise argparse.ArgumentError( - "{0!r} is not a Pipfile project".format(root), - ) - try: - project = Project(root) - except tomlkit.exceptions.ParseError as e: - raise argparse.ArgumentError( - "failed to parse Pipfile: {0!r}".format(str(e)), - ) - return project - - -# Better error reporting. Recent argparse would emit something like -# "invalid project root value: 'xxxxxx'". The str() wrapper is needed to -# keep Python 2 happy :( -build_project.__name__ = str("project root") +from .options import project class BaseCommand(object): @@ -38,19 +14,30 @@ class BaseCommand(object): """ name = None description = None - parsed_main = None - - def __init__(self, parser): + default_arguments = [project] + arguments = [] + + def __init__(self, parser=None): + if not parser: + parser = argparse.ArgumentParser( + prog=os.path.basename(sys.argv[0]), + description="Base argument parser for passa" + ) self.parser = parser self.add_arguments() @classmethod - def run_current_module(cls): + def build_parser(cls): parser = argparse.ArgumentParser( prog="passa {}".format(cls.name), description=cls.description, ) - cls(parser)() + return cls(parser) + + @classmethod + def run_parser(cls): + parser = cls.build_parser() + parser() def __call__(self, argv=None): options = self.parser.parse_args(argv) @@ -58,16 +45,17 @@ def __call__(self, argv=None): if result is not None: sys.exit(result) + def add_default_arguments(self): + for arg in self.default_arguments: + arg.add_to_parser(self.parser) + def add_arguments(self): - self.parser.add_argument( - "--project", - metavar="project", - default=os.getcwd(), - type=build_project, - help="path to project root (directory containing Pipfile)", - ) + self.add_default_arguments() + for arg in self.arguments: + arg.add_to_parser(self.parser) def main(self, options): - # This __dict__ access is needed for Python 2 to prevent Python from - # wrapping parsed_main into an unbounded method. - return type(self).__dict__["parsed_main"](options) + return self.run(options) + + def run(self, options): + raise NotImplementedError diff --git a/src/passa/cli/add.py b/src/passa/cli/add.py index 26ce0ed..2635b98 100644 --- a/src/passa/cli/add.py +++ b/src/passa/cli/add.py @@ -2,98 +2,28 @@ from __future__ import absolute_import, print_function, unicode_literals -import itertools -import sys - +from ..actions.add import add_packages from ._base import BaseCommand - - -def main(options): - from passa.internals.lockers import PinReuseLocker - from passa.operations.lock import lock - - lines = list(itertools.chain( - options.requirement_lines, - ("-e {}".format(e) for e in options.editable_lines), - )) - - project = options.project - for line in lines: - try: - project.add_line_to_pipfile(line, develop=options.dev) - except (TypeError, ValueError) as e: - print("Cannot add {line!r} to Pipfile: {error}".format( - line=line, error=str(e), - ), file=sys.stderr) - return 2 - - prev_lockfile = project.lockfile - - locker = PinReuseLocker(project) - success = lock(locker) - if not success: - return 1 - - project._p.write() - project._l.write() - print("Written to project at", project.root) - - if not options.sync: - return - - from passa.internals.synchronizers import Synchronizer - from passa.operations.sync import sync - - lockfile_diff = project.difference_lockfile(prev_lockfile) - default = bool(any(lockfile_diff.default)) - develop = bool(any(lockfile_diff.develop)) - - syncer = Synchronizer( - project, default=default, develop=develop, - clean_unneeded=False, - ) - success = sync(syncer) - if not success: - return 1 - - print("Synchronized project at", project.root) +from .options import package_group, clean_group class Command(BaseCommand): name = "add" description = "Add packages to project." - parsed_main = main + arguments = [package_group, clean_group] - def add_arguments(self): - super(Command, self).add_arguments() - self.parser.add_argument( - "requirement_lines", metavar="requirement", - nargs="*", - help="requirement to add (can be used multiple times)", - ) - self.parser.add_argument( - "-e", "--editable", - metavar="requirement", dest="editable_lines", - action="append", default=[], - help="editable requirement to add (can be used multiple times)", - ) - self.parser.add_argument( - "--dev", - action="store_true", - help="add packages to [dev-packages]", - ) - self.parser.add_argument( - "--no-sync", dest="sync", - action="store_false", default=True, - help="do not synchronize the environment", - ) - - def main(self, options): - if not options.editable_lines and not options.requirement_lines: + def run(self, options): + if not options.editables and not options.packages: self.parser.error("Must supply either a requirement or --editable") - return super(Command, self).main(options) + return add_packages( + packages=options.packages, + editables=options.editables, + project=options.project, + dev=options.dev, + clean=options.clean + ) if __name__ == "__main__": - Command.run_current_module() + Command.run_parser() diff --git a/src/passa/cli/clean.py b/src/passa/cli/clean.py index cd1b679..a74d814 100644 --- a/src/passa/cli/clean.py +++ b/src/passa/cli/clean.py @@ -2,37 +2,23 @@ from __future__ import absolute_import, print_function, unicode_literals +from ..actions.clean import clean from ._base import BaseCommand - - -def main(options): - from passa.internals.synchronizers import Cleaner - from passa.operations.sync import clean - - project = options.project - cleaner = Cleaner(project, default=True, develop=options.dev) - - success = clean(cleaner) - if not success: - return 1 - - print("Cleaned project at", project.root) +from .options import dev, no_default, sync_group class Command(BaseCommand): name = "clean" description = "Uninstall unlisted packages from the environment." - parsed_main = main - - def add_arguments(self): - super(Command, self).add_arguments() - self.parser.add_argument( - "--no-dev", dest="dev", - action="store_false", default=True, - help="uninstall develop packages, only keep default ones", + arguments = [dev, no_default, sync_group] + + def run(self, options): + return clean( + project=options.project, default=options.default, dev=options.dev, + sync=options.sync ) if __name__ == "__main__": - Command.run_current_module() + Command.run_parser() diff --git a/src/passa/cli/freeze.py b/src/passa/cli/freeze.py index 6ca5717..053c727 100644 --- a/src/passa/cli/freeze.py +++ b/src/passa/cli/freeze.py @@ -2,137 +2,23 @@ from __future__ import absolute_import, print_function, unicode_literals -import contextlib -import io -import itertools -import sys - -import six -import vistir.misc - +from ..actions.freeze import freeze from ._base import BaseCommand - - -def _source_as_lines(source, extra): - url = source["url"] - if extra: - lines = ["--extra-index-url {}".format(url)] - else: - lines = ["--index-url {}".format(url)] - if not source.get("verify_ssl", True): - lines = ["--trusted-host {}".format(url)] - return lines - - -def _requirement_as_line(requirement, sources, include_hashes): - if requirement.index: - sources = sources - else: - sources = None - line = requirement.as_line(sources=sources, include_hashes=include_hashes) - if not isinstance(line, six.text_type): - line = line.decode("utf-8") - return line - - -@contextlib.contextmanager -def open_for_output(filename): - if filename is None: - yield sys.stdout - return - with io.open(filename, "w", encoding="utf-8", newline="\n") as f: - yield f - - -def main(options): - from requirementslib import Requirement - - lockfile = options.project.lockfile - if not lockfile: - print("Pipfile.lock is required to export.", file=sys.stderr) - return 1 - - section_names = [] - if options.default: - section_names.append("default") - if options.dev: - section_names.append("develop") - requirements = [ - Requirement.from_pipfile(key, entry._data) - for key, entry in itertools.chain.from_iterable( - lockfile.get(name, {}).items() - for name in section_names - ) - ] - - include_hashes = options.include_hashes - if include_hashes is None: - include_hashes = all(r.is_named for r in requirements) - - sources = lockfile.meta.sources._data - - source_lines = list(vistir.misc.dedup(itertools.chain( - itertools.chain.from_iterable( - _source_as_lines(source, False) - for source in sources[:1] - ), - itertools.chain.from_iterable( - _source_as_lines(source, True) - for source in sources[1:] - ), - ))) - - requirement_lines = sorted(vistir.misc.dedup( - _requirement_as_line(requirement, sources, include_hashes) - for requirement in requirements - )) - - with open_for_output(options.target) as f: - for line in source_lines: - f.write(line) - f.write("\n") - f.write("\n") - for line in requirement_lines: - f.write(line) - f.write("\n\n") +from .options import dev, include_hashes_group, no_default, target class Command(BaseCommand): name = "freeze" description = "Export project depenencies to requirements.txt." - parsed_main = main + arguments = [dev, no_default, target, include_hashes_group] - def add_arguments(self): - super(Command, self).add_arguments() - self.parser.add_argument( - "--target", - default=None, - help="file to export into (default is to print to stdout)", - ) - self.parser.add_argument( - "--dev", - action="store_true", default=False, - help="include development packages in requirements.txt", - ) - self.parser.add_argument( - "--no-default", dest="default", - action="store_false", default=True, - help="do not include default packages in requirements.txt", - ) - include_hashes_group = self.parser.add_mutually_exclusive_group() - include_hashes_group.add_argument( - "--include-hashes", dest="include_hashes", - action="store_true", - help="output hashes in requirements.txt (default is to guess)", - ) - include_hashes_group.add_argument( - "--no-include-hashes", dest="include_hashes", - action="store_false", - help=("do not output hashes in requirements.txt " - "(default is to guess)"), + def run(self, options): + return freeze( + project=options.project, default=options.default, dev=options.dev, + include_hashes=options.include_hashes ) if __name__ == "__main__": - Command.run_current_module() + Command.run_parser() diff --git a/src/passa/cli/init.py b/src/passa/cli/init.py new file mode 100644 index 0000000..95ce8d8 --- /dev/null +++ b/src/passa/cli/init.py @@ -0,0 +1,32 @@ +# -*- coding=utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + +import argparse +import os + +from ..actions.init import init_project +from ._base import BaseCommand +from .options import new_project_group + + +class Command(BaseCommand): + + name = "init" + description = "Create a new project." + default_arguments = [] + arguments = [new_project_group] + + def run(self, options): + pipfile_path = os.path.join(options.project, "Pipfile") + if os.path.exists(pipfile_path): + raise argparse.ArgumentError( + "{0!r} is already a Pipfile project".format(options.project), + ) + return init_project( + root=options.project, python_version=options.python_version + ) + + +if __name__ == "__main__": + Command.run_parser() diff --git a/src/passa/cli/install.py b/src/passa/cli/install.py index f47377b..1c0b459 100644 --- a/src/passa/cli/install.py +++ b/src/passa/cli/install.py @@ -2,62 +2,21 @@ from __future__ import absolute_import, print_function, unicode_literals +from ..actions.install import install from ._base import BaseCommand - - -def main(options): - from passa.internals.lockers import BasicLocker - from passa.operations.lock import lock - - project = options.project - - if not options.check or not project.is_synced(): - locker = BasicLocker(project) - success = lock(locker) - if not success: - return 1 - project._l.write() - print("Written to project at", project.root) - - from passa.internals.synchronizers import Synchronizer - from passa.operations.sync import sync - - syncer = Synchronizer( - project, default=True, develop=options.dev, - clean_unneeded=options.clean, - ) - - success = sync(syncer) - if not success: - return 1 - - print("Synchronized project at", project.root) +from .options import dev, no_check, no_clean class Command(BaseCommand): name = "install" description = "Generate Pipfile.lock to synchronize the environment." - parsed_main = main - - def add_arguments(self): - super(Command, self).add_arguments() - self.parser.add_argument( - "--no-check", dest="check", - action="store_false", default=True, - help="do not check if Pipfile.lock is update, always resolve", - ) - self.parser.add_argument( - "--dev", - action="store_true", - help="install develop packages", - ) - self.parser.add_argument( - "--no-clean", dest="clean", - action="store_false", default=True, - help="do not uninstall packages not specified in Pipfile.lock", - ) + arguments = [no_check, dev, no_clean] + + def run(self, options): + return install(project=options.project, check=options.check, dev=options.dev, + clean=options.clean) if __name__ == "__main__": - Command.run_current_module() + Command.run_parser() diff --git a/src/passa/cli/lock.py b/src/passa/cli/lock.py index 67b1d11..9b0651a 100644 --- a/src/passa/cli/lock.py +++ b/src/passa/cli/lock.py @@ -2,28 +2,17 @@ from __future__ import absolute_import, print_function, unicode_literals +from ..actions.lock import lock from ._base import BaseCommand -def main(options): - from passa.internals.lockers import BasicLocker - from passa.operations.lock import lock - - project = options.project - locker = BasicLocker(project) - success = lock(locker) - if not success: - return - - project._l.write() - print("Written to project at", project.root) - - class Command(BaseCommand): name = "lock" description = "Generate Pipfile.lock." - parsed_main = main + + def run(self, options): + return lock(project=options.project) if __name__ == "__main__": - Command.run_current_module() + Command.run_parser() diff --git a/src/passa/cli/options.py b/src/passa/cli/options.py new file mode 100644 index 0000000..07782db --- /dev/null +++ b/src/passa/cli/options.py @@ -0,0 +1,214 @@ +# -*- coding=utf-8 -*- +from __future__ import absolute_import + +import argparse +import inspect +import os +import sys + +import six +import tomlkit.exceptions + +import passa.models.projects +import passa.models.virtualenv +import vistir + + +PYTHON_VERSION = ".".join(str(v) for v in sys.version_info[:2]) + + +class Project(passa.models.projects.Project): + def __init__(self, root, *args, **kwargs): + root = vistir.compat.Path(root).absolute() + pipfile = root.joinpath("Pipfile") + if not pipfile.is_file(): + raise argparse.ArgumentError( + project, "{0!r} is not a Pipfile project".format(root.as_posix()), + ) + self.venv = self.get_venv(root) + try: + super(Project, self).__init__(root.as_posix(), env_prefix=self.venv.venv_dir, + *args, **kwargs) + except tomlkit.exceptions.ParseError as e: + raise argparse.ArgumentError( + project, "failed to parse Pipfile: {0!r}".format(str(e)), + ) + + def get_venv(self, root): + if 'VIRTUAL_ENV' in os.environ: + return passa.models.virtualenv.VirtualEnv(os.environ['VIRTUAL_ENV']) + return passa.models.virtualenv.VirtualEnv.from_project_path(root) + + def __name__(self): + return "Project Root" + + +class OptionMeta(type): + + @property + def action_map(self): + action_map = getattr(self, '_action_map', None) + if not action_map: + self.action_map = { + name.strip("_").replace("Action", ""): obj + for name, obj in inspect.getmembers(argparse) + if name.startswith('_') and name.endswith('Action') + } + return self._action_map + + @action_map.setter + def action_map(self, action_map): + self._action_map = action_map + + +@six.add_metaclass(OptionMeta) +class Option(object): + def __init__(self, *args, **kwargs): + self.args = list(args) + self.kwargs = kwargs.copy() + if "dest" not in kwargs and not args[0].startswith("-"): + dest = list(args).pop(0) + else: + dest = kwargs.pop("dest", args[0].lstrip("-").replace("-", "_")) + action = kwargs.pop("action", None) + if not action: + if 'const' in kwargs: + action = 'store_const' + else: + action = 'store' + self.action = self.get_option(action, args, dest, **kwargs) + + @classmethod + def get_option(cls, action, option_strings, dest, *args, **kwargs): + if action: + action = action.title().replace("_", "") + return cls.action_map[action](list(option_strings), dest, *args, **kwargs) + return + + def add_to_parser(self, parser): + parser.add_argument(*self.args, **self.kwargs) + + def add_to_group(self, group): + group.add_argument(*self.args, **self.kwargs) + + +class ArgumentGroup(object): + def __init__(self, name, parser=None, is_mutually_exclusive=False, required=False, options=[]): + self.name = name + self.options = options + self.parser = parser + self.required = required + self.is_mutually_exclusive = is_mutually_exclusive + self.argument_group = None + + def add_to_parser(self, parser): + group = None + if self.is_mutually_exclusive: + group = parser.add_mutually_exclusive_group(required=self.required) + else: + group = parser.add_argument_group() + for option in self.options: + option.add_to_group(group) + self.argument_group = group + self.parser = parser + + def add_to_group(self, group): + self.add_to_parser(group) + + +project = Option( + "--project", metavar="project", default=os.getcwd(), type=Project, + help="path to project root (directory containing Pipfile)", +) + +new_project = Option( + "--project", metavar="project", default=os.getcwd(), type=str, + help="path to project root (directory containing Pipfile)", +) + +python_version = Option( + "--py-version", "--python-version", "--requires-python", metavar="python_version", + dest="python_version", default=PYTHON_VERSION, type=str, + help="required minor python version for the project" +) + +packages = Option( + "packages", metavar="package", nargs="*", + help="requirement to add (can be used multiple times)", +) + +editable = Option( + '-e', '--editable', dest='editables', nargs="*", default=[], metavar='path/vcs', + help="editable requirement to add (can be used multiple times)", +) + +dev = Option( + "--dev", action="store_true", default=False, + help="Use [dev-packages] for install/freeze/uninstall operations", +) + +no_sync = Option( + "--no-sync", dest="sync", action="store_false", default=True, + help="do not synchronize the environment", +) + +sync = Option( + "--sync", dest="sync", action="store_true", help="synchronize the environment", + default=False +) + +target = Option( + "-t", "--target", default=None, + help="file to export into (default is to print to stdout)" +) + +no_default = Option( + "--no-default", dest="default", action="store_false", default=True, + help="do not include default packages when exporting, importing, or cleaning" +) + +include_hashes = Option( + "--include-hashes", dest="include_hashes", action="store_true", + help="output hashes in requirements.txt (default is to guess)", +) + +no_include_hashes = Option( + "--no-include-hashes", dest="include_hashes", action="store_false", + help="do not output hashes in requirements.txt (default is to guess)", +) + +no_check = Option( + "--no-check", dest="check", action="store_false", default=True, + help="do not check if Pipfile.lock is up to date, always resolve", +) + +no_clean = Option( + "--no-clean", dest="clean", action="store_false", default=True, + help="do not remove packages not specified in Pipfile.lock", +) + +clean = Option( + "--clean", dest="clean", action="store_true", default=False, + help="remove packages not specified in Pipfile.lock", +) +dev_only = Option( + "--dev", dest="only", action="store_const", const="dev", + help="only try to modify [dev-packages]", +) + +default_only = Option( + "--default", dest="only", action="store_const", const="default", + help="only try to modify [default]", +) + +strategy = Option( + "--strategy", choices=["eager", "only-if-needed"], default="only-if-needed", + help="how dependency upgrading is handled", +) + +include_hashes_group = ArgumentGroup("include_hashes", is_mutually_exclusive=True, options=[include_hashes, no_include_hashes]) +dev_group = ArgumentGroup("dev", is_mutually_exclusive="True", options=[dev_only, default_only]) +new_project_group = ArgumentGroup("new-project", options=[new_project, python_version]) +clean_group = ArgumentGroup("clean", is_mutually_exclusive=True, options=[clean, no_clean]) +sync_group = ArgumentGroup("sync", is_mutually_exclusive=True, options=[sync, no_sync]) +package_group = ArgumentGroup("packages", options=[packages, editable, dev, sync_group]) diff --git a/src/passa/cli/remove.py b/src/passa/cli/remove.py index b1dbfd7..041c195 100644 --- a/src/passa/cli/remove.py +++ b/src/passa/cli/remove.py @@ -2,74 +2,21 @@ from __future__ import absolute_import, print_function, unicode_literals +from ..actions.remove import remove from ._base import BaseCommand - - -def main(options): - from passa.internals.lockers import PinReuseLocker - from passa.operations.lock import lock - - default = (options.only != "dev") - develop = (options.only != "default") - - project = options.project - project.remove_keys_from_pipfile( - options.packages, default=default, develop=develop, - ) - - locker = PinReuseLocker(project) - success = lock(locker) - if not success: - return 1 - - project._p.write() - project._l.write() - print("Written to project at", project.root) - - if not options.clean: - return - - from passa.internals.synchronizers import Cleaner - from passa.operations.sync import clean - - cleaner = Cleaner(project, default=True, develop=True) - success = clean(cleaner) - if not success: - return 1 - - print("Cleaned project at", project.root) +from .options import dev_group, clean_group, sync_group, packages class Command(BaseCommand): name = "remove" description = "Remove packages from project." - parsed_main = main + arguments = [dev_group, clean_group, sync_group, packages] - def add_arguments(self): - super(Command, self).add_arguments() - self.parser.add_argument( - "packages", metavar="package", - nargs="+", - help="package to remove (can be used multiple times)", - ) - dev_group = self.parser.add_mutually_exclusive_group() - dev_group.add_argument( - "--dev", dest="only", - action="store_const", const="dev", - help="only try to remove from [dev-packages]", - ) - dev_group.add_argument( - "--default", dest="only", - action="store_const", const="default", - help="only try to remove from [packages]", - ) - self.parser.add_argument( - "--no-clean", dest="clean", - action="store_false", default=True, - help="do not uninstall packages not specified in Pipfile.lock", - ) + def run(self, options): + return remove(project=options.project, only=options.only, + packages=options.packages, clean=options.clean, sync=options.sync) if __name__ == "__main__": - Command.run_current_module() + Command.run_parser() diff --git a/src/passa/cli/sync.py b/src/passa/cli/sync.py index ade81e0..9a31fe1 100644 --- a/src/passa/cli/sync.py +++ b/src/passa/cli/sync.py @@ -2,45 +2,20 @@ from __future__ import absolute_import, print_function, unicode_literals +from ..actions.sync import sync from ._base import BaseCommand - - -def main(options): - from passa.internals.synchronizers import Synchronizer - from passa.operations.sync import sync - - project = options.project - syncer = Synchronizer( - project, default=True, develop=options.dev, - clean_unneeded=options.clean, - ) - - success = sync(syncer) - if not success: - return 1 - - print("Synchronized project at", project.root) +from .options import dev, clean_group class Command(BaseCommand): name = "sync" description = "Install Pipfile.lock into the environment." - parsed_main = main - - def add_arguments(self): - super(Command, self).add_arguments() - self.parser.add_argument( - "--dev", - action="store_true", - help="install develop packages", - ) - self.parser.add_argument( - "--no-clean", dest="clean", - action="store_false", default=True, - help="do not uninstall packages not specified in Pipfile.lock", - ) + arguments = [dev, clean_group] + + def run(self, options): + return sync(project=options.project, dev=options.dev, clean=options.clean) if __name__ == "__main__": - Command.run_current_module() + Command.run_parser() diff --git a/src/passa/cli/upgrade.py b/src/passa/cli/upgrade.py index 011fff6..c7696c2 100644 --- a/src/passa/cli/upgrade.py +++ b/src/passa/cli/upgrade.py @@ -1,91 +1,21 @@ # -*- coding=utf-8 -*- - from __future__ import absolute_import, print_function, unicode_literals -import sys - +from ..actions.upgrade import upgrade from ._base import BaseCommand - - -def main(options): - from passa.internals.lockers import EagerUpgradeLocker, PinReuseLocker - from passa.operations.lock import lock - - project = options.project - packages = options.packages - for package in packages: - if not project.contains_key_in_pipfile(package): - print("{package!r} not found in Pipfile".format( - package=package, - ), file=sys.stderr) - return 2 - - project.remove_keys_from_lockfile(packages) - - prev_lockfile = project.lockfile - - if options.strategy == "eager": - locker = EagerUpgradeLocker(project, packages) - else: - locker = PinReuseLocker(project) - success = lock(locker) - if not success: - return 1 - - project._l.write() - print("Written to project at", project.root) - - if not options.sync: - return - - from passa.operations.sync import sync - from passa.internals.synchronizers import Synchronizer - - lockfile_diff = project.difference_lockfile(prev_lockfile) - default = bool(any(lockfile_diff.default)) - develop = bool(any(lockfile_diff.develop)) - - syncer = Synchronizer( - project, default=default, develop=develop, - clean_unneeded=False, - ) - success = sync(syncer) - if not success: - return 1 - - print("Synchronized project at", project.root) +from .options import clean_group, sync_group, packages, strategy class Command(BaseCommand): name = "upgrade" description = "Upgrade packages in project." - parsed_main = main + arguments = [packages, strategy, clean_group, sync_group] - def add_arguments(self): - super(Command, self).add_arguments() - self.parser.add_argument( - "packages", metavar="package", - nargs="+", - help="package to upgrade (can be used multiple times)", - ) - self.parser.add_argument( - "--strategy", - choices=["eager", "only-if-needed"], - default="only-if-needed", - help="how dependency upgrading is handled", - ) - self.parser.add_argument( - "--no-sync", dest="sync", - action="store_false", default=True, - help="do not synchronize the environment", - ) - self.parser.add_argument( - "--no-clean", dest="clean", - action="store_false", default=True, - help="do not uninstall packages not specified in Pipfile.lock", - ) + def run(self, options): + return upgrade(project=options.project, strategy=options.strategy, + sync=options.sync, packages=options.packages) if __name__ == "__main__": - Command.run_current_module() + Command.run_parser() diff --git a/src/passa/internals/_pip.py b/src/passa/internals/_pip.py index f73e241..2aa143a 100644 --- a/src/passa/internals/_pip.py +++ b/src/passa/internals/_pip.py @@ -17,8 +17,8 @@ import six import vistir +from ..models.caches import CACHE_DIR from ._pip_shims import VCS_SUPPORT, build_wheel as _build_wheel, unpack_url -from .caches import CACHE_DIR from .utils import filter_sources diff --git a/src/passa/internals/dependencies.py b/src/passa/internals/dependencies.py index 1fa3b0b..410a5e6 100644 --- a/src/passa/internals/dependencies.py +++ b/src/passa/internals/dependencies.py @@ -13,8 +13,8 @@ import requirementslib import six +from ..models.caches import DependencyCache, RequiresPythonCache from ._pip import WheelBuildError, build_wheel, read_sdist_metadata -from .caches import DependencyCache, RequiresPythonCache from .markers import contains_extra, get_contained_extras, get_without_extra from .utils import get_pinned_version, is_pinned diff --git a/src/passa/models/__init__.py b/src/passa/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/passa/internals/caches.py b/src/passa/models/caches.py similarity index 98% rename from src/passa/internals/caches.py rename to src/passa/models/caches.py index 6d3131f..c6d29b5 100644 --- a/src/passa/internals/caches.py +++ b/src/passa/models/caches.py @@ -13,8 +13,8 @@ import requests import vistir -from ._pip_shims import VCS_SUPPORT -from .utils import get_pinned_version +from ..internals._pip_shims import VCS_SUPPORT +from ..internals.utils import get_pinned_version CACHE_DIR = os.environ.get("PASSA_CACHE_DIR", appdirs.user_cache_dir("passa")) diff --git a/src/passa/internals/lockers.py b/src/passa/models/lockers.py similarity index 97% rename from src/passa/internals/lockers.py rename to src/passa/models/lockers.py index b567c46..c25ca60 100644 --- a/src/passa/internals/lockers.py +++ b/src/passa/models/lockers.py @@ -4,18 +4,19 @@ import itertools +import resolvelib + import plette import requirementslib -import resolvelib import vistir +from ..internals.hashes import get_hashes +from ..internals.reporters import StdOutReporter +from ..internals.traces import trace_graph +from ..internals.utils import identify_requirment from .caches import HashCache -from .hashes import get_hashes from .metadata import set_metadata from .providers import BasicProvider, EagerUpgradeProvider, PinReuseProvider -from .reporters import StdOutReporter -from .traces import trace_graph -from .utils import identify_requirment def _get_requirements(model, section_name): diff --git a/src/passa/internals/metadata.py b/src/passa/models/metadata.py similarity index 97% rename from src/passa/internals/metadata.py rename to src/passa/models/metadata.py index 9709c53..a949f1e 100644 --- a/src/passa/internals/metadata.py +++ b/src/passa/models/metadata.py @@ -10,8 +10,8 @@ import vistir import vistir.misc -from .markers import get_without_extra -from .specifiers import cleanup_pyspecs, pyspec_from_markers +from ..internals.markers import get_without_extra +from ..internals.specifiers import cleanup_pyspecs, pyspec_from_markers def dedup_markers(s): diff --git a/src/passa/internals/projects.py b/src/passa/models/projects.py similarity index 99% rename from src/passa/internals/projects.py rename to src/passa/models/projects.py index f6e037d..7ff6f31 100644 --- a/src/passa/internals/projects.py +++ b/src/passa/models/projects.py @@ -84,6 +84,7 @@ def dumps(self): class Project(object): root = attr.ib() + env_prefix = attr.ib(default=None) _p = attr.ib(init=False) _l = attr.ib(init=False) diff --git a/src/passa/internals/providers.py b/src/passa/models/providers.py similarity index 97% rename from src/passa/internals/providers.py rename to src/passa/models/providers.py index bf4fd22..36b2f2e 100644 --- a/src/passa/internals/providers.py +++ b/src/passa/models/providers.py @@ -6,11 +6,10 @@ import resolvelib -from .candidates import find_candidates -from .dependencies import get_dependencies -from .utils import ( - filter_sources, get_allow_prereleases, - are_requirements_equal, identify_requirment, strip_extras, +from ..internals.candidates import find_candidates +from ..internals.dependencies import get_dependencies +from ..internals.utils import ( + filter_sources, get_allow_prereleases, identify_requirment, strip_extras, ) diff --git a/src/passa/internals/synchronizers.py b/src/passa/models/synchronizers.py similarity index 70% rename from src/passa/internals/synchronizers.py rename to src/passa/models/synchronizers.py index 7ccce23..9c8aa9c 100644 --- a/src/passa/internals/synchronizers.py +++ b/src/passa/models/synchronizers.py @@ -14,15 +14,19 @@ import packaging.version import requirementslib -from ._pip import uninstall, EditableInstaller, WheelInstaller +from .virtualenv import VirtualEnv +from ..internals._pip import uninstall, EditableInstaller, WheelInstaller -def _is_installation_local(name): + +def _is_installation_local(name, venv=None): """Check whether the distribution is in the current Python installation. This is used to distinguish packages seen by a virtual environment. A venv may be able to see global packages, but we don't want to mess with them. """ + if venv: + return venv.is_installed(name) loc = os.path.normcase(pkg_resources.working_set.by_key[name].location) pre = os.path.normcase(sys.prefix) return os.path.commonprefix([loc, pre]) == pre @@ -38,12 +42,14 @@ def _is_up_to_date(distro, version): ]) -def _group_installed_names(packages): +def _group_installed_names(packages, venv=None): """Group locally installed packages based on given specifications. `packages` is a name-package mapping that are used as baseline to determine how the installed package should be grouped. + `venv` is the virtual environment object of the virtualenv being installed into. + Returns a 3-tuple of disjoint sets, all containing names of installed packages: @@ -54,8 +60,13 @@ def _group_installed_names(packages): """ groupcoll = GroupCollection(set(), set(), set(), set()) - for distro in pkg_resources.working_set: - name = distro.key + if venv: + working_set = venv.get_working_set() + else: + working_set = pkg_resources.working_set + + for dist in working_set: + name = dist.key try: package = packages[name] except KeyError: @@ -66,7 +77,7 @@ def _group_installed_names(packages): if not r.is_named: # Always mark non-named. I think pip does something similar? groupcoll.outdated.add(name) - elif not _is_up_to_date(distro, r.get_version()): + elif not _is_up_to_date(dist, r.get_version()): groupcoll.outdated.add(name) else: groupcoll.uptodate.add(name) @@ -75,11 +86,14 @@ def _group_installed_names(packages): @contextlib.contextmanager -def _remove_package(name): - if name is None or not _is_installation_local(name): +def _remove_package(name, venv=None): + if name is None or not _is_installation_local(name, venv=venv): yield None return - with uninstall(name, auto_confirm=True, verbose=False) as uninstaller: + _uninstall = uninstall + if venv: + _uninstall = venv.uninstall + with _uninstall(name, auto_confirm=True, verbose=False) as uninstaller: yield uninstaller @@ -88,19 +102,22 @@ def _get_packages(lockfile, default, develop): # Extras don't matter because they only affect dependencies, and we # don't install dependencies anyway! packages = {} - if default: - packages.update(lockfile.default._data) if develop: packages.update(lockfile.develop._data) + if default: + packages.update(lockfile.default._data) return packages -def _build_paths(): +def _build_paths(venv=None): """Prepare paths for distlib.wheel.Wheel to install into. """ - paths = sysconfig.get_paths() + if venv: + paths = venv.paths + else: + paths = sysconfig.get_paths() return { - "prefix": sys.prefix, + "prefix": sys.prefix if not venv else venv.venv_dir.as_posix(), "data": paths["data"], "scripts": paths["scripts"], "headers": paths["include"], @@ -112,13 +129,13 @@ def _build_paths(): PROTECTED_FROM_CLEAN = {"setuptools", "pip", "wheel"} -def _clean(names): +def _clean(names, venv=None): cleaned = set() for name in names: if name in PROTECTED_FROM_CLEAN: continue - with _remove_package(name) as uninst: - if uninst: + with _remove_package(name, venv=venv) as uninst: + if uninst.paths: cleaned.add(name) return cleaned @@ -126,18 +143,34 @@ def _clean(names): class Synchronizer(object): """Helper class to install packages from a project's lock file. """ - def __init__(self, project, default, develop, clean_unneeded): + def __init__(self, project, default, develop, clean_unneeded, venv=None): self._root = project.root # Only for repr. self.packages = _get_packages(project.lockfile, default, develop) self.sources = project.lockfile.meta.sources._data - self.paths = _build_paths() self.clean_unneeded = clean_unneeded + if not venv: + self._venv = getattr(project, "venv", None) + else: + self._venv = venv + self.paths = _build_paths(venv=self.venv) + + @property + def venv(self): + if self._venv: + return self._venv + return self.project.venv def __repr__(self): return "<{0} @ {1!r}>".format(type(self).__name__, self._root) def sync(self): - groupcoll = _group_installed_names(self.packages) + if not self.venv: + return self._sync() + with self.venv.activated(): + return self._sync() + + def _sync(self): + groupcoll = _group_installed_names(self.packages, venv=self.venv) installed = set() updated = set() @@ -146,7 +179,7 @@ def sync(self): # TODO: Show a prompt to confirm cleaning. We will need to implement a # reporter pattern for this as well. if self.clean_unneeded: - names = _clean(groupcoll.unneeded) + names = _clean(groupcoll.unneeded, venv=self.venv) cleaned.update(names) # TODO: Specify installation order? (pypa/pipenv#2274) @@ -161,7 +194,7 @@ def sync(self): continue r.markers = None if r.editable: - installer = EditableInstaller(r) + installer = EditableInstaller(r, venv=self.venv) else: installer = WheelInstaller(r, self.sources, self.paths) try: @@ -181,7 +214,7 @@ def sync(self): else: name_to_remove = None try: - with _remove_package(name_to_remove): + with _remove_package(name_to_remove, venv=self.venv): installer.install() except Exception as e: if os.environ.get("PASSA_NO_SUPPRESS_EXCEPTIONS"): @@ -201,14 +234,27 @@ def sync(self): class Cleaner(object): """Helper class to clean packages not in a project's lock file. """ - def __init__(self, project, default, develop): + def __init__(self, project, default, develop, sync=True, verbose=False): self._root = project.root # Only for repr. self.packages = _get_packages(project.lockfile, default, develop) + self.sync = sync + self.project = project def __repr__(self): return "<{0} @ {1!r}>".format(type(self).__name__, self._root) + def print(self, packages): + if not self.sync: + message = "Would clean: {0}" + else: + message = "Cleaned: {0}" + print(message.format(", ".join(sorted(set(packages))))) + def clean(self): - groupcoll = _group_installed_names(self.packages) - cleaned = _clean(groupcoll.unneeded) + groupcoll = _group_installed_names(self.packages, venv=self.project.venv) + cleaned = set() + if self.sync: + cleaned = _clean(groupcoll.unneeded, venv=self.project.venv) + else: + return groupcoll.unneeded return cleaned diff --git a/src/passa/models/virtualenv.py b/src/passa/models/virtualenv.py new file mode 100644 index 0000000..a4f3954 --- /dev/null +++ b/src/passa/models/virtualenv.py @@ -0,0 +1,311 @@ +# -*- coding=utf-8 -*- + +import base64 +import contextlib +import distlib.scripts +import hashlib +import importlib +import json +import posixpath +import os +import re +import six +import sys +import sysconfig + +import passa.internals._pip +from cached_property import cached_property + +import vistir + + +class VirtualEnv(object): + def __init__(self, venv_dir): + self.recursive_monkey_patch = self.safe_import("recursive_monkey_patch") + self.venv_dir = vistir.compat.Path(venv_dir) + + @classmethod + def from_project_path(cls, path): + path = vistir.compat.Path(path) + if path.name == 'Pipfile': + pipfile_path = path + path = path.parent + else: + pipfile_path = path / 'Pipfile' + pipfile_location = cls.normalize_path(pipfile_path) + venv_path = path / '.venv' + if venv_path.exists(): + if not venv_path.is_dir(): + possible_path = vistir.compat.Path(venv_path.read_text().strip()) + if possible_path.exists(): + return cls(possible_path.as_posix()) + else: + if venv_path.joinpath('lib').exists(): + return cls(venv_path.as_posix()) + sanitized = re.sub(r'[ $`!*@"\\\r\n\t]', "_", path.name)[0:42] + hash_ = hashlib.sha256(pipfile_location.encode()).digest()[:6] + encoded_hash = base64.urlsafe_b64encode(hash_).decode() + hash_fragment = encoded_hash[:8] + venv_name = "{0}-{1}".format(sanitized, hash_fragment) + return cls(cls.get_workon_home().joinpath(venv_name).as_posix()) + + @classmethod + def normalize_path(cls, path): + if not path: + return + if not path.is_absolute(): + try: + path = path.resolve() + except OSError: + path = path.absolute() + path = vistir.path.unicode_path("{0}".format(path)) + if os.name != "nt": + return path + + drive, tail = os.path.splitdrive(path) + # Only match (lower cased) local drives (e.g. 'c:'), not UNC mounts. + if drive.islower() and len(drive) == 2 and drive[1] == ":": + path = "{}{}".format(drive.upper(), tail) + + return vistir.path.unicode_path(path) + + @classmethod + def get_workon_home(cls): + workon_home = os.environ.get("WORKON_HOME") + if not workon_home: + if os.name == "nt": + workon_home = "~/.virtualenvs" + else: + workon_home = os.path.join( + os.environ.get("XDG_DATA_HOME", "~/.local/share"), "virtualenvs" + ) + return vistir.compat.Path(os.path.expandvars(workon_home)).expanduser() + + @classmethod + def filter_sources(cls, requirement, sources): + if not sources or not requirement.index: + return sources + filtered_sources = [ + source for source in sources + if source.get("name") == requirement.index + ] + return filtered_sources or sources + + @classmethod + def safe_import(cls, name): + module = None + if name not in sys.modules: + module = importlib.import_module(name) + else: + module = sys.modules[name] + six.moves.reload_module(module) + return module + + @cached_property + def script_basedir(self): + script_dir = os.path.basename(sysconfig.get_paths()["scripts"]) + return script_dir + + @property + def python(self): + return self.venv_dir.joinpath(self.script_basedir).joinpath("python").as_posix() + + @cached_property + def sys_path(self): + c = vistir.misc.run([self.python, "-c", "import json,sys; print(json.dumps(sys.path))"], + return_object=True, nospin=True) + assert c.returncode == 0, "failed loading virtualenv path" + path = [ + path for path in json.loads(c.out.strip()) + if posixpath.normpath(path).startswith(posixpath.normpath(str(self.venv_dir))) + ] + return path + + @cached_property + def system_paths(self): + paths = {} + sysconfig = self.safe_import("sysconfig") + paths = sysconfig.get_paths() + return paths + + @cached_property + def sys_prefix(self): + c = self.run_py(["-c", "'import sys; print(sys.prefix)'"]) + sys_prefix = vistir.misc.to_text(c.out).strip() + return sys_prefix + + @cached_property + def paths(self): + paths = {} + with vistir.contextmanagers.temp_environ(), vistir.contextmanagers.temp_path(): + os.environ["PYTHONUSERBASE"] = vistir.compat.fs_str(self.venv_dir.as_posix()) + os.environ["PYTHONIOENCODING"] = vistir.compat.fs_str("utf-8") + os.environ["PYTHONDONTWRITEBYTECODE"] = vistir.compat.fs_str("1") + sysconfig = self.safe_import("sysconfig") + scheme, _, _ = sysconfig._get_default_scheme().partition('_') + scheme = "{0}_user".format(scheme) + paths = sysconfig.get_paths(scheme=scheme) + return paths + + @property + def scripts_dir(self): + return self.paths["scripts"] + + @cached_property + def passa_entry(self): + pkg_resources = self.safe_import("pkg_resources") + return pkg_resources.working_set.by_key['passa'].location + + def get_distributions(self): + pkg_resources = self.safe_import("pkg_resources") + return pkg_resources.find_distributions(self.paths["purelib"], only=True) + + def get_working_set(self): + working_set = None + import pkg_resources + passa_entry = self.passa_entry + working_set = pkg_resources.WorkingSet(self.sys_path + [passa_entry]) + return working_set + + @cached_property + def python_version(self): + with self.activated(): + sysconfig = self.safe_import("sysconfig") + py_version = sysconfig.get_python_version() + return py_version + + def get_setup_install_args(self, pkgname, setup_py, develop=False): + headers = vistir.compat.Path(self.sys_prefix) / "include" / "site" + headers = headers / "python{0}".format(self.python_version) / pkgname + install_arg = "install" if not develop else "develop" + return [ + self.python, "-u", "-c", SETUPTOOLS_SHIM % setup_py, install_arg, + "--single-version-externally-managed", "root={0}".format(), + "--install-headers={0}".format(headers.as_posix()), + "--install-purelib={0}".format(self.paths["purelib"]), + "--install-platlib={0}".format(self.paths["platlib"]), + "--install-scripts={0}".format(self.scripts_dir), + "--install-data={0}".format(self.paths["data"]), + ] + + def install(self, req, editable=False, sources=[]): + with self.activated(): + install_options = ["--prefix={0}".format(self.venv_dir),] + passa_pip = self.safe_import("passa.internals._pip") + ireq = req.as_ireq() + if editable: + with vistir.contextmanagers.cd(ireq.setup_py_dir, ireq.setup_py): + c = self.run( + install_options + self.get_setup_install_args( + req.name, develop=editable + ), cwd=ireq.setup_py_dir + ) + return c.returncode + distlib_scripts = self.safe_import("distlib.scripts") + sources = self.filter_sources(req, sources) + hashes = req.hashes + wheel = passa_pip.build_wheel(ireq, sources, hashes) + wheel.install(self.paths, distlib_scripts.ScriptMaker(None, None)) + + @contextlib.contextmanager + def activated(self): + original_path = sys.path + original_prefix = sys.prefix + original_user_base = os.environ.get("PYTHONUSERBASE", None) + original_venv = os.environ.get("VIRTUAL_ENV", None) + passa_path = vistir.compat.Path(__file__).absolute().parent.parent.as_posix() + with vistir.contextmanagers.temp_environ(), vistir.contextmanagers.temp_path(): + os.environ["PYTHONIOENCODING"] = vistir.compat.fs_str("utf-8") + os.environ["PYTHONDONTWRITEBYTECODE"] = vistir.compat.fs_str("1") + activate_this = os.path.join(self.scripts_dir, "activate_this.py") + with open(activate_this, "r") as f: + code = compile(f.read(), activate_this, "exec") + exec(code, dict(__file__=activate_this)) + os.environ["PYTHONUSERBASE"] = vistir.compat.fs_str(self.venv_dir.as_posix()) + os.environ["VIRTUAL_ENV"] = vistir.compat.fs_str(self.venv_dir.as_posix()) + sys.path = self.sys_path + sys.prefix = self.venv_dir + site = self.safe_import("site") + site.addsitedir(passa_path) + sys.modules["recursive_monkey_patch"] = self.recursive_monkey_patch + pkg_resources = self.safe_import("pkg_resources") + try: + yield + finally: + print("Deactivating virtualenv...") + del os.environ["VIRTUAL_ENV"] + del os.environ["PYTHONUSERBASE"] + if original_user_base: + os.environ["PYTHONUSERBASE"] = original_user_base + if original_venv: + os.environ["VIRTUAL_ENV"] = original_venv + sys.path = original_path + sys.prefix = original_prefix + six.moves.reload_module(pkg_resources) + + def run(self, cmd, cwd=os.curdir): + c = None + with self.activated(): + script = vistir.cmdparse.Script.parse(cmd) + c = vistir.misc.run(script._parts, return_object=True, nospin=True, cwd=cwd) + return c + + def run_py(self, cmd, cwd=os.curdir): + c = None + if isinstance(cmd, six.string_types): + script = vistir.cmdparse.Script.parse("{0} {1}".format(self.python, cmd)) + else: + script = vistir.cmdparse.Script.parse([self.python,] + list(cmd)) + with self.activated(): + c = vistir.misc.run(script._parts, return_object=True, nospin=True, cwd=cwd) + return c + + def is_installed(self, pkgname): + return any(d for d in self.get_distributions() if d.project_name == pkgname) + + def get_monkeypatched_pathset(self): + from pip_shims.shims import req_install + req_uninstall_name = "{0}.req_uninstall".format(req_install.__package__) + req_uninstall = self.safe_import(req_uninstall_name) + self.recursive_monkey_patch.monkey_patch( + PatchedUninstaller, req_uninstall.UninstallPathSet + ) + return req_uninstall.UninstallPathSet + + @contextlib.contextmanager + def uninstall(self, pkgname, *args, **kwargs): + auto_confirm = kwargs.pop("auto_confirm", True) + verbose = kwargs.pop("verbose", False) + with self.activated(): + pathset_base = self.get_monkeypatched_pathset() + dist = next( + iter(filter(lambda d: d.project_name == pkgname, self.get_working_set())), + None + ) + pathset = pathset_base.from_dist(dist) + if pathset is not None: + pathset.remove(auto_confirm=auto_confirm, verbose=verbose) + try: + yield pathset + except Exception as e: + if pathset is not None: + pathset.rollback() + else: + if pathset is not None: + pathset.commit() + if pathset is None: + return + + +SETUPTOOLS_SHIM = ( + "import setuptools, tokenize;__file__=%r;" + "f=getattr(tokenize, 'open', open)(__file__);" + "code=f.read().replace('\\r\\n', '\\n');" + "f.close();" + "exec(compile(code, __file__, 'exec'))" +) + + +class PatchedUninstaller(object): + def _permitted(self, path): + return True diff --git a/src/passa/operations/sync.py b/src/passa/operations/sync.py index 3014e8d..45502a4 100644 --- a/src/passa/operations/sync.py +++ b/src/passa/operations/sync.py @@ -16,8 +16,8 @@ def sync(syncer): def clean(cleaner): - print("Cleaning") + print("Cleaning...") cleaned = cleaner.clean() if cleaned: - print("Uninstalled: {}".format(", ".join(sorted(cleaned)))) + cleaner.print(cleaned) return True diff --git a/tests/actions/__init__.py b/tests/actions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/actions/test_add.py b/tests/actions/test_add.py new file mode 100644 index 0000000..e7cd883 --- /dev/null +++ b/tests/actions/test_add.py @@ -0,0 +1,20 @@ +# -*- coding=utf-8 -*- +import passa.actions.init +import passa.actions.add +import passa.cli.options +import passa.models.projects + + +def test_add_one(project_directory): + project = passa.cli.options.Project(project_directory.strpath) + retcode = passa.actions.add.add_packages(["pytz"], project=project) + assert not retcode + assert 'pytz' in project.lockfile.default + + +def test_add_one_with_deps(project_directory): + project = passa.cli.options.Project(project_directory.strpath) + retcode = passa.actions.add.add_packages(["requests"], project=project) + assert not retcode + assert 'requests' in project.lockfile.default + assert 'idna' in project.lockfile.default diff --git a/tests/actions/test_clean.py b/tests/actions/test_clean.py new file mode 100644 index 0000000..95b0b14 --- /dev/null +++ b/tests/actions/test_clean.py @@ -0,0 +1,18 @@ +# -*- coding=utf-8 -*- + + +def test_clean_subset(project): + from passa.actions.add import add_packages + from passa.actions.clean import clean + retcode = add_packages(["requests"], project=project) + assert not retcode + packages = ["requests", "chardet", "certifi", "idna"] + c = project.venv.run("pip install pytz") + assert c.returncode == 0 + c = project.venv.run("python -c 'import pytz'") + assert c.returncode == 0 + clean_retcode = clean(project=project) + assert not clean_retcode + c = project.venv.run("python -c 'import pytz'") + assert c.returncode != 0 + assert all(pkg in project.lockfile.default for pkg in packages) diff --git a/tests/actions/test_freeze.py b/tests/actions/test_freeze.py new file mode 100644 index 0000000..53d34f9 --- /dev/null +++ b/tests/actions/test_freeze.py @@ -0,0 +1,21 @@ +# -*- coding=utf-8 -*- +import passa.actions.add +import passa.actions.freeze +import passa.cli.options +import passa.models.projects + + +def test_freeze(project_directory): + project = passa.cli.options.Project(project_directory.strpath) + retcode = passa.actions.add.add_packages(["requests"], project=project) + assert not retcode + packages = ["requests", "chardet", "certifi", "idna"] + assert all(pkg in project.lockfile.default for pkg in packages) + freeze_file = project_directory.join("requirements.txt") + freeze_retcode = passa.actions.freeze.freeze( + project=project, include_hashes=False, target=freeze_file.strpath + ) + assert not freeze_retcode + lines = [line.strip() for line in freeze_file.readlines() if line.strip() != ''] + for pkg in packages: + assert any(line.startswith(pkg) for line in lines) diff --git a/tests/actions/test_init.py b/tests/actions/test_init.py new file mode 100644 index 0000000..2f12530 --- /dev/null +++ b/tests/actions/test_init.py @@ -0,0 +1,19 @@ +# -*- coding=utf-8 -*- + +import pytest + +import passa.actions.init +import passa.cli.options + + +def test_init(tmpdir): + init_retcode = passa.actions.init.init_project(root=tmpdir.strpath) + assert init_retcode == 0 + project = passa.cli.options.Project(tmpdir.strpath) + assert project.pipfile.packages._data == {} + assert project.pipfile.dev_packages._data == {} + + +def test_init_exists(project_directory): + with pytest.raises(RuntimeError, match=r'.* is already a Pipfile project'): + passa.actions.init.init_project(root=project_directory.strpath) diff --git a/tests/actions/test_install.py b/tests/actions/test_install.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/actions/test_lock.py b/tests/actions/test_lock.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/actions/test_remove.py b/tests/actions/test_remove.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/actions/test_sync.py b/tests/actions/test_sync.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/actions/test_upgrade.py b/tests/actions/test_upgrade.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/__init__.py b/tests/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/test_add.py b/tests/cli/test_add.py new file mode 100644 index 0000000..a6bcbda --- /dev/null +++ b/tests/cli/test_add.py @@ -0,0 +1 @@ +# -*- coding=utf-8 -*- diff --git a/tests/cli/test_clean.py b/tests/cli/test_clean.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/test_freeze.py b/tests/cli/test_freeze.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/test_init.py b/tests/cli/test_init.py new file mode 100644 index 0000000..a6bcbda --- /dev/null +++ b/tests/cli/test_init.py @@ -0,0 +1 @@ +# -*- coding=utf-8 -*- diff --git a/tests/cli/test_install.py b/tests/cli/test_install.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/test_lock.py b/tests/cli/test_lock.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/test_remove.py b/tests/cli/test_remove.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/test_sync.py b/tests/cli/test_sync.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/test_upgrade.py b/tests/cli/test_upgrade.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..e1326ca --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,57 @@ +# -*- coding=utf-8 -*- +import os +import pytest +import passa.cli.options +import passa.models.virtualenv +import sys +import vistir + + +DEFAULT_PIPFILE_CONTENTS = """ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[packages] + +[dev-packages] +""".strip() + + +@pytest.fixture(scope="function") +def project_directory(tmpdir_factory): + project_dir = tmpdir_factory.mktemp("passa-project") + project_dir.join("Pipfile").write(DEFAULT_PIPFILE_CONTENTS) + with vistir.contextmanagers.cd(project_dir.strpath): + yield project_dir + + +@pytest.fixture(scope="function") +def virtualenv(tmpdir_factory): + venv_dir = tmpdir_factory.mktemp("passa-testenv") + print("Creating virtualenv {0!r}".format(venv_dir.strpath)) + c = vistir.misc.run([sys.executable, "-m", "virtualenv", venv_dir.strpath], + return_object=True, block=True, nospin=True) + if c.returncode == 0: + print("Virtualenv created...") + return venv_dir + raise RuntimeError("Failed creating virtualenv for testing...{0!r}".format(c.err.strip())) + + +class _Project(passa.cli.options.Project): + def __init__(self, root, venv=None): + self.path = os.path.abspath(root) + self.venv = venv + super(_Project, self).__init__(self.path) + + +@pytest.fixture +def tmpvenv(virtualenv): + return passa.models.virtualenv.VirtualEnv(virtualenv) + + +@pytest.fixture(scope="function") +def project(project_directory, tmpvenv): + with tmpvenv.activated(): + yield _Project(project_directory, tmpvenv) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000..e69de29