From 17fdda635ecf847440fca3f4735fff6b0946f9e2 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 17 Sep 2018 01:16:01 -0400 Subject: [PATCH 01/34] Add initial tests Signed-off-by: Dan Ryan --- tests/actions/__init__.py | 0 tests/actions/test_add.py | 20 ++++++++++++++++++++ tests/actions/test_clean.py | 18 ++++++++++++++++++ tests/actions/test_freeze.py | 21 +++++++++++++++++++++ tests/actions/test_init.py | 19 +++++++++++++++++++ tests/actions/test_install.py | 0 tests/actions/test_lock.py | 0 tests/actions/test_remove.py | 0 tests/actions/test_sync.py | 0 tests/actions/test_upgrade.py | 0 tests/cli/__init__.py | 0 tests/cli/test_add.py | 1 + tests/cli/test_clean.py | 0 tests/cli/test_freeze.py | 0 tests/cli/test_init.py | 1 + tests/cli/test_install.py | 0 tests/cli/test_lock.py | 0 tests/cli/test_remove.py | 0 tests/cli/test_sync.py | 0 tests/cli/test_upgrade.py | 0 tests/conftest.py | 21 +++++++++++++++++++++ tests/integration/__init__.py | 0 22 files changed, 101 insertions(+) create mode 100644 tests/actions/__init__.py create mode 100644 tests/actions/test_add.py create mode 100644 tests/actions/test_clean.py create mode 100644 tests/actions/test_freeze.py create mode 100644 tests/actions/test_init.py create mode 100644 tests/actions/test_install.py create mode 100644 tests/actions/test_lock.py create mode 100644 tests/actions/test_remove.py create mode 100644 tests/actions/test_sync.py create mode 100644 tests/actions/test_upgrade.py create mode 100644 tests/cli/__init__.py create mode 100644 tests/cli/test_add.py create mode 100644 tests/cli/test_clean.py create mode 100644 tests/cli/test_freeze.py create mode 100644 tests/cli/test_init.py create mode 100644 tests/cli/test_install.py create mode 100644 tests/cli/test_lock.py create mode 100644 tests/cli/test_remove.py create mode 100644 tests/cli/test_sync.py create mode 100644 tests/cli/test_upgrade.py create mode 100644 tests/conftest.py create mode 100644 tests/integration/__init__.py diff --git a/tests/actions/__init__.py b/tests/actions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/actions/test_add.py b/tests/actions/test_add.py new file mode 100644 index 0000000..e7cd883 --- /dev/null +++ b/tests/actions/test_add.py @@ -0,0 +1,20 @@ +# -*- coding=utf-8 -*- +import passa.actions.init +import passa.actions.add +import passa.cli.options +import passa.models.projects + + +def test_add_one(project_directory): + project = passa.cli.options.Project(project_directory.strpath) + retcode = passa.actions.add.add_packages(["pytz"], project=project) + assert not retcode + assert 'pytz' in project.lockfile.default + + +def test_add_one_with_deps(project_directory): + project = passa.cli.options.Project(project_directory.strpath) + retcode = passa.actions.add.add_packages(["requests"], project=project) + assert not retcode + assert 'requests' in project.lockfile.default + assert 'idna' in project.lockfile.default diff --git a/tests/actions/test_clean.py b/tests/actions/test_clean.py new file mode 100644 index 0000000..3062ef5 --- /dev/null +++ b/tests/actions/test_clean.py @@ -0,0 +1,18 @@ +# -*- coding=utf-8 -*- +from passa.actions.add import add_packages +from passa.cli.options import Project +import vistir + + +def test_clean_subset(project_directory): + with vistir.contextmanagers.cd(project_directory.strpath): + project = Project(project_directory.strpath) + retcode = add_packages(["requests"], project=project) + assert not retcode + packages = ["requests", "chardet", "certifi", "idna"] + assert all(pkg in project.lockfile.default for pkg in packages) + import passa.actions.clean + pass + # clean_retcode = passa.actions.clean.clean(project=project) + # assert not clean_retcode + # assert project.lockfile.default._data == {} diff --git a/tests/actions/test_freeze.py b/tests/actions/test_freeze.py new file mode 100644 index 0000000..53d34f9 --- /dev/null +++ b/tests/actions/test_freeze.py @@ -0,0 +1,21 @@ +# -*- coding=utf-8 -*- +import passa.actions.add +import passa.actions.freeze +import passa.cli.options +import passa.models.projects + + +def test_freeze(project_directory): + project = passa.cli.options.Project(project_directory.strpath) + retcode = passa.actions.add.add_packages(["requests"], project=project) + assert not retcode + packages = ["requests", "chardet", "certifi", "idna"] + assert all(pkg in project.lockfile.default for pkg in packages) + freeze_file = project_directory.join("requirements.txt") + freeze_retcode = passa.actions.freeze.freeze( + project=project, include_hashes=False, target=freeze_file.strpath + ) + assert not freeze_retcode + lines = [line.strip() for line in freeze_file.readlines() if line.strip() != ''] + for pkg in packages: + assert any(line.startswith(pkg) for line in lines) diff --git a/tests/actions/test_init.py b/tests/actions/test_init.py new file mode 100644 index 0000000..2f12530 --- /dev/null +++ b/tests/actions/test_init.py @@ -0,0 +1,19 @@ +# -*- coding=utf-8 -*- + +import pytest + +import passa.actions.init +import passa.cli.options + + +def test_init(tmpdir): + init_retcode = passa.actions.init.init_project(root=tmpdir.strpath) + assert init_retcode == 0 + project = passa.cli.options.Project(tmpdir.strpath) + assert project.pipfile.packages._data == {} + assert project.pipfile.dev_packages._data == {} + + +def test_init_exists(project_directory): + with pytest.raises(RuntimeError, match=r'.* is already a Pipfile project'): + passa.actions.init.init_project(root=project_directory.strpath) diff --git a/tests/actions/test_install.py b/tests/actions/test_install.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/actions/test_lock.py b/tests/actions/test_lock.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/actions/test_remove.py b/tests/actions/test_remove.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/actions/test_sync.py b/tests/actions/test_sync.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/actions/test_upgrade.py b/tests/actions/test_upgrade.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/__init__.py b/tests/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/test_add.py b/tests/cli/test_add.py new file mode 100644 index 0000000..a6bcbda --- /dev/null +++ b/tests/cli/test_add.py @@ -0,0 +1 @@ +# -*- coding=utf-8 -*- diff --git a/tests/cli/test_clean.py b/tests/cli/test_clean.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/test_freeze.py b/tests/cli/test_freeze.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/test_init.py b/tests/cli/test_init.py new file mode 100644 index 0000000..a6bcbda --- /dev/null +++ b/tests/cli/test_init.py @@ -0,0 +1 @@ +# -*- coding=utf-8 -*- diff --git a/tests/cli/test_install.py b/tests/cli/test_install.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/test_lock.py b/tests/cli/test_lock.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/test_remove.py b/tests/cli/test_remove.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/test_sync.py b/tests/cli/test_sync.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/test_upgrade.py b/tests/cli/test_upgrade.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..577c942 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,21 @@ +# -*- coding=utf-8 -*- +import pytest + + +DEFAULT_PIPFILE_CONTENTS = """ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[packages] + +[dev-packages] +""".strip() + + +@pytest.fixture(scope="function") +def project_directory(tmpdir_factory): + project_dir = tmpdir_factory.mktemp("passa-project") + project_dir.join("Pipfile").write(DEFAULT_PIPFILE_CONTENTS) + return project_dir diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000..e69de29 From 4186a0ce7dbfacfd11094c54808c01d76f16cfe7 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 18 Sep 2018 05:23:23 -0400 Subject: [PATCH 02/34] Clean up options and add virtualenv module Signed-off-by: Dan Ryan --- src/passa/actions/add.py | 2 +- src/passa/actions/clean.py | 4 +- src/passa/actions/remove.py | 2 +- src/passa/cli/add.py | 7 +- src/passa/cli/clean.py | 9 +- src/passa/cli/options.py | 77 +++++++++-- src/passa/cli/remove.py | 6 +- src/passa/cli/sync.py | 4 +- src/passa/cli/upgrade.py | 4 +- src/passa/models/projects.py | 1 + src/passa/models/synchronizers.py | 85 ++++++++---- src/passa/models/virtualenv.py | 212 ++++++++++++++++++++++++++++++ 12 files changed, 364 insertions(+), 49 deletions(-) create mode 100644 src/passa/models/virtualenv.py diff --git a/src/passa/actions/add.py b/src/passa/actions/add.py index 6338466..3efc4dc 100644 --- a/src/passa/actions/add.py +++ b/src/passa/actions/add.py @@ -48,7 +48,7 @@ def add_packages(packages=[], editables=[], project=None, dev=False, sync=False, syncer = Synchronizer( project, default=default, develop=develop, - clean_unneeded=clean, + clean_unneeded=clean ) success = sync(syncer) if not success: diff --git a/src/passa/actions/clean.py b/src/passa/actions/clean.py index 3570e4d..8c6ae17 100644 --- a/src/passa/actions/clean.py +++ b/src/passa/actions/clean.py @@ -3,11 +3,11 @@ from __future__ import absolute_import, print_function, unicode_literals -def clean(project, dev=False): +def clean(project, default=True, dev=False, sync=True): from passa.models.synchronizers import Cleaner from passa.operations.sync import clean - cleaner = Cleaner(project, default=True, develop=dev) + cleaner = Cleaner(project, default=default, develop=dev, sync=sync) success = clean(cleaner) if not success: diff --git a/src/passa/actions/remove.py b/src/passa/actions/remove.py index 158f5e6..17ba1c7 100644 --- a/src/passa/actions/remove.py +++ b/src/passa/actions/remove.py @@ -3,7 +3,7 @@ from __future__ import absolute_import, print_function, unicode_literals -def remove(project=None, only="default", packages=[], clean=True): +def remove(project=None, only="default", packages=[], clean=True, sync=False): from passa.models.lockers import PinReuseLocker from passa.operations.lock import lock diff --git a/src/passa/cli/add.py b/src/passa/cli/add.py index d5596cd..2635b98 100644 --- a/src/passa/cli/add.py +++ b/src/passa/cli/add.py @@ -4,14 +4,14 @@ from ..actions.add import add_packages from ._base import BaseCommand -from .options import package_group +from .options import package_group, clean_group class Command(BaseCommand): name = "add" description = "Add packages to project." - arguments = [package_group] + arguments = [package_group, clean_group] def run(self, options): if not options.editables and not options.packages: @@ -20,7 +20,8 @@ def run(self, options): packages=options.packages, editables=options.editables, project=options.project, - dev=options.dev + dev=options.dev, + clean=options.clean ) diff --git a/src/passa/cli/clean.py b/src/passa/cli/clean.py index e23d5ee..a74d814 100644 --- a/src/passa/cli/clean.py +++ b/src/passa/cli/clean.py @@ -4,17 +4,20 @@ from ..actions.clean import clean from ._base import BaseCommand -from .options import dev, no_default +from .options import dev, no_default, sync_group class Command(BaseCommand): name = "clean" description = "Uninstall unlisted packages from the environment." - arguments = [dev, no_default] + arguments = [dev, no_default, sync_group] def run(self, options): - return clean(project=options.project, default=options.default, dev=options.dev) + return clean( + project=options.project, default=options.default, dev=options.dev, + sync=options.sync + ) if __name__ == "__main__": diff --git a/src/passa/cli/options.py b/src/passa/cli/options.py index da89a3b..07782db 100644 --- a/src/passa/cli/options.py +++ b/src/passa/cli/options.py @@ -2,12 +2,15 @@ from __future__ import absolute_import import argparse +import inspect import os import sys +import six import tomlkit.exceptions import passa.models.projects +import passa.models.virtualenv import vistir @@ -20,23 +23,67 @@ def __init__(self, root, *args, **kwargs): pipfile = root.joinpath("Pipfile") if not pipfile.is_file(): raise argparse.ArgumentError( - "{0!r} is not a Pipfile project".format(root), + project, "{0!r} is not a Pipfile project".format(root.as_posix()), ) + self.venv = self.get_venv(root) try: - super(Project, self).__init__(root.as_posix(), *args, **kwargs) + super(Project, self).__init__(root.as_posix(), env_prefix=self.venv.venv_dir, + *args, **kwargs) except tomlkit.exceptions.ParseError as e: raise argparse.ArgumentError( - "failed to parse Pipfile: {0!r}".format(str(e)), + project, "failed to parse Pipfile: {0!r}".format(str(e)), ) + def get_venv(self, root): + if 'VIRTUAL_ENV' in os.environ: + return passa.models.virtualenv.VirtualEnv(os.environ['VIRTUAL_ENV']) + return passa.models.virtualenv.VirtualEnv.from_project_path(root) + def __name__(self): return "Project Root" +class OptionMeta(type): + + @property + def action_map(self): + action_map = getattr(self, '_action_map', None) + if not action_map: + self.action_map = { + name.strip("_").replace("Action", ""): obj + for name, obj in inspect.getmembers(argparse) + if name.startswith('_') and name.endswith('Action') + } + return self._action_map + + @action_map.setter + def action_map(self, action_map): + self._action_map = action_map + + +@six.add_metaclass(OptionMeta) class Option(object): def __init__(self, *args, **kwargs): - self.args = args - self.kwargs = kwargs + self.args = list(args) + self.kwargs = kwargs.copy() + if "dest" not in kwargs and not args[0].startswith("-"): + dest = list(args).pop(0) + else: + dest = kwargs.pop("dest", args[0].lstrip("-").replace("-", "_")) + action = kwargs.pop("action", None) + if not action: + if 'const' in kwargs: + action = 'store_const' + else: + action = 'store' + self.action = self.get_option(action, args, dest, **kwargs) + + @classmethod + def get_option(cls, action, option_strings, dest, *args, **kwargs): + if action: + action = action.title().replace("_", "") + return cls.action_map[action](list(option_strings), dest, *args, **kwargs) + return def add_to_parser(self, parser): parser.add_argument(*self.args, **self.kwargs) @@ -46,7 +93,7 @@ def add_to_group(self, group): class ArgumentGroup(object): - def __init__(self, name, parser=None, is_mutually_exclusive=False, required=None, options=[]): + def __init__(self, name, parser=None, is_mutually_exclusive=False, required=False, options=[]): self.name = name self.options = options self.parser = parser @@ -65,6 +112,9 @@ def add_to_parser(self, parser): self.argument_group = group self.parser = parser + def add_to_group(self, group): + self.add_to_parser(group) + project = Option( "--project", metavar="project", default=os.getcwd(), type=Project, @@ -77,7 +127,7 @@ def add_to_parser(self, parser): ) python_version = Option( - "--py-version", "--python-version", "--requires-python", metavar="python-version", + "--py-version", "--python-version", "--requires-python", metavar="python_version", dest="python_version", default=PYTHON_VERSION, type=str, help="required minor python version for the project" ) @@ -102,6 +152,11 @@ def add_to_parser(self, parser): help="do not synchronize the environment", ) +sync = Option( + "--sync", dest="sync", action="store_true", help="synchronize the environment", + default=False +) + target = Option( "-t", "--target", default=None, help="file to export into (default is to print to stdout)" @@ -132,6 +187,10 @@ def add_to_parser(self, parser): help="do not remove packages not specified in Pipfile.lock", ) +clean = Option( + "--clean", dest="clean", action="store_true", default=False, + help="remove packages not specified in Pipfile.lock", +) dev_only = Option( "--dev", dest="only", action="store_const", const="dev", help="only try to modify [dev-packages]", @@ -149,5 +208,7 @@ def add_to_parser(self, parser): include_hashes_group = ArgumentGroup("include_hashes", is_mutually_exclusive=True, options=[include_hashes, no_include_hashes]) dev_group = ArgumentGroup("dev", is_mutually_exclusive="True", options=[dev_only, default_only]) -package_group = ArgumentGroup("packages", options=[packages, editable, dev, no_sync]) new_project_group = ArgumentGroup("new-project", options=[new_project, python_version]) +clean_group = ArgumentGroup("clean", is_mutually_exclusive=True, options=[clean, no_clean]) +sync_group = ArgumentGroup("sync", is_mutually_exclusive=True, options=[sync, no_sync]) +package_group = ArgumentGroup("packages", options=[packages, editable, dev, sync_group]) diff --git a/src/passa/cli/remove.py b/src/passa/cli/remove.py index 538acbf..041c195 100644 --- a/src/passa/cli/remove.py +++ b/src/passa/cli/remove.py @@ -4,18 +4,18 @@ from ..actions.remove import remove from ._base import BaseCommand -from .options import dev_group, no_clean, packages +from .options import dev_group, clean_group, sync_group, packages class Command(BaseCommand): name = "remove" description = "Remove packages from project." - arguments = [dev_group, no_clean, packages] + arguments = [dev_group, clean_group, sync_group, packages] def run(self, options): return remove(project=options.project, only=options.only, - packages=options.packages, clean=options.clean) + packages=options.packages, clean=options.clean, sync=options.sync) if __name__ == "__main__": diff --git a/src/passa/cli/sync.py b/src/passa/cli/sync.py index a09b784..9a31fe1 100644 --- a/src/passa/cli/sync.py +++ b/src/passa/cli/sync.py @@ -4,14 +4,14 @@ from ..actions.sync import sync from ._base import BaseCommand -from .options import dev, no_clean +from .options import dev, clean_group class Command(BaseCommand): name = "sync" description = "Install Pipfile.lock into the environment." - arguments = [dev, no_clean] + arguments = [dev, clean_group] def run(self, options): return sync(project=options.project, dev=options.dev, clean=options.clean) diff --git a/src/passa/cli/upgrade.py b/src/passa/cli/upgrade.py index cf7f502..c7696c2 100644 --- a/src/passa/cli/upgrade.py +++ b/src/passa/cli/upgrade.py @@ -3,14 +3,14 @@ from ..actions.upgrade import upgrade from ._base import BaseCommand -from .options import no_clean, no_sync, packages, strategy +from .options import clean_group, sync_group, packages, strategy class Command(BaseCommand): name = "upgrade" description = "Upgrade packages in project." - arguments = [packages, strategy, no_clean, no_sync] + arguments = [packages, strategy, clean_group, sync_group] def run(self, options): return upgrade(project=options.project, strategy=options.strategy, diff --git a/src/passa/models/projects.py b/src/passa/models/projects.py index f6e037d..7ff6f31 100644 --- a/src/passa/models/projects.py +++ b/src/passa/models/projects.py @@ -84,6 +84,7 @@ def dumps(self): class Project(object): root = attr.ib() + env_prefix = attr.ib(default=None) _p = attr.ib(init=False) _l = attr.ib(init=False) diff --git a/src/passa/models/synchronizers.py b/src/passa/models/synchronizers.py index bad4905..8fc3e0a 100644 --- a/src/passa/models/synchronizers.py +++ b/src/passa/models/synchronizers.py @@ -14,15 +14,19 @@ import packaging.version import requirementslib +from .virtualenv import VirtualEnv + from ..internals._pip import uninstall, EditableInstaller, WheelInstaller -def _is_installation_local(name): +def _is_installation_local(name, venv=None): """Check whether the distribution is in the current Python installation. This is used to distinguish packages seen by a virtual environment. A venv may be able to see global packages, but we don't want to mess with them. """ + if venv: + return venv.is_installed(name) loc = os.path.normcase(pkg_resources.working_set.by_key[name].location) pre = os.path.normcase(sys.prefix) return os.path.commonprefix([loc, pre]) == pre @@ -38,12 +42,14 @@ def _is_up_to_date(distro, version): ]) -def _group_installed_names(packages): +def _group_installed_names(packages, venv=None): """Group locally installed packages based on given specifications. `packages` is a name-package mapping that are used as baseline to determine how the installed package should be grouped. + `venv` is the virtual environment object of the virtualenv being installed into. + Returns a 3-tuple of disjoint sets, all containing names of installed packages: @@ -54,8 +60,13 @@ def _group_installed_names(packages): """ groupcoll = GroupCollection(set(), set(), set(), set()) - for distro in pkg_resources.working_set: - name = distro.key + if venv: + working_set = venv.get_working_set() + else: + working_set = pkg_resources.working_set + + for dist in working_set: + name = dist.key try: package = packages[name] except KeyError: @@ -66,7 +77,7 @@ def _group_installed_names(packages): if not r.is_named: # Always mark non-named. I think pip does something similar? groupcoll.outdated.add(name) - elif not _is_up_to_date(distro, r.get_version()): + elif not _is_up_to_date(dist, r.get_version()): groupcoll.outdated.add(name) else: groupcoll.uptodate.add(name) @@ -75,11 +86,14 @@ def _group_installed_names(packages): @contextlib.contextmanager -def _remove_package(name): - if name is None or not _is_installation_local(name): +def _remove_package(name, venv=None): + if name is None or not _is_installation_local(name, venv=venv): yield None return - with uninstall(name, auto_confirm=True, verbose=False) as uninstaller: + _uninstall = uninstall + if venv: + _uninstall = venv.uninstall + with _uninstall(name, auto_confirm=True, verbose=False) as uninstaller: yield uninstaller @@ -88,19 +102,22 @@ def _get_packages(lockfile, default, develop): # Extras don't matter because they only affect dependencies, and we # don't install dependencies anyway! packages = {} - if default: - packages.update(lockfile.default._data) if develop: packages.update(lockfile.develop._data) + if default: + packages.update(lockfile.default._data) return packages -def _build_paths(): +def _build_paths(venv=None): """Prepare paths for distlib.wheel.Wheel to install into. """ - paths = sysconfig.get_paths() + if venv: + paths = venv.paths + else: + paths = sysconfig.get_paths() return { - "prefix": sys.prefix, + "prefix": sys.prefix if not venv else venv.venv_dir.as_posix(), "data": paths["data"], "scripts": paths["scripts"], "headers": paths["include"], @@ -112,12 +129,12 @@ def _build_paths(): PROTECTED_FROM_CLEAN = {"setuptools", "pip", "wheel"} -def _clean(names): +def _clean(names, venv=None): cleaned = set() for name in names: if name in PROTECTED_FROM_CLEAN: continue - with _remove_package(name) as uninst: + with _remove_package(name, venv=venv) as uninst: if uninst: cleaned.add(name) return cleaned @@ -126,18 +143,34 @@ def _clean(names): class Synchronizer(object): """Helper class to install packages from a project's lock file. """ - def __init__(self, project, default, develop, clean_unneeded): + def __init__(self, project, default, develop, clean_unneeded, venv=None): self._root = project.root # Only for repr. self.packages = _get_packages(project.lockfile, default, develop) self.sources = project.lockfile.meta.sources._data - self.paths = _build_paths() self.clean_unneeded = clean_unneeded + if not venv: + self._venv = getattr(project, "venv", None) + else: + self._venv = venv + self.paths = _build_paths(venv=self.venv) + + @property + def venv(self): + if self._venv: + return self._venv + return self.project.venv def __repr__(self): return "<{0} @ {1!r}>".format(type(self).__name__, self._root) def sync(self): - groupcoll = _group_installed_names(self.packages) + if not self.venv: + return self._sync() + with self.venv.activated(): + return self._sync() + + def _sync(self): + groupcoll = _group_installed_names(self.packages, venv=self.venv) installed = set() updated = set() @@ -146,7 +179,7 @@ def sync(self): # TODO: Show a prompt to confirm cleaning. We will need to implement a # reporter pattern for this as well. if self.clean_unneeded: - names = _clean(groupcoll.unneeded) + names = _clean(groupcoll.unneeded, venv=self.venv) cleaned.update(names) # TODO: Specify installation order? (pypa/pipenv#2274) @@ -161,7 +194,7 @@ def sync(self): continue r.markers = None if r.editable: - installer = EditableInstaller(r) + installer = EditableInstaller(r, venv=self.venv) else: installer = WheelInstaller(r, self.sources, self.paths) try: @@ -181,7 +214,7 @@ def sync(self): else: name_to_remove = None try: - with _remove_package(name_to_remove): + with _remove_package(name_to_remove, venv=self.venv): installer.install() except Exception as e: if os.environ.get("PASSA_NO_SUPPRESS_EXCEPTIONS"): @@ -201,14 +234,18 @@ def sync(self): class Cleaner(object): """Helper class to clean packages not in a project's lock file. """ - def __init__(self, project, default, develop): + def __init__(self, project, default, develop, sync=True): self._root = project.root # Only for repr. self.packages = _get_packages(project.lockfile, default, develop) + self.sync = sync + self.project = project def __repr__(self): return "<{0} @ {1!r}>".format(type(self).__name__, self._root) def clean(self): - groupcoll = _group_installed_names(self.packages) - cleaned = _clean(groupcoll.unneeded) + groupcoll = _group_installed_names(self.packages, venv=self.project.venv) + cleaned = set() + if self.sync: + cleaned = _clean(groupcoll.unneeded, venv=self.project.venv) return cleaned diff --git a/src/passa/models/virtualenv.py b/src/passa/models/virtualenv.py new file mode 100644 index 0000000..78484c2 --- /dev/null +++ b/src/passa/models/virtualenv.py @@ -0,0 +1,212 @@ +# -*- coding=utf-8 -*- + +import base64 +import contextlib +import distlib.scripts +import hashlib +import importlib +import json +import os +import re +import sys +import sysconfig + +from cached_property import cached_property + +import vistir + +from ..internals._pip import RequirementUninstaller + + +class VirtualEnv(object): + def __init__(self, venv_dir): + self.venv_dir = vistir.compat.Path(venv_dir) + + @classmethod + def from_project_path(cls, path): + path = vistir.compat.Path(path) + if path.name == 'Pipfile': + pipfile_path = path + path = path.parent + else: + pipfile_path = path / 'Pipfile' + pipfile_location = cls.normalize_path(pipfile_path) + venv_path = path / '.venv' + if venv_path.exists(): + if not venv_path.is_dir(): + possible_path = vistir.compat.Path(venv_path.read_text().strip()) + if possible_path.exists(): + return cls(possible_path.as_posix()) + else: + if venv_path.joinpath('lib').exists(): + return cls(venv_path.as_posix()) + sanitized = re.sub(r'[ $`!*@"\\\r\n\t]', "_", path.name)[0:42] + hash_ = hashlib.sha256(pipfile_location.encode()).digest()[:6] + encoded_hash = base64.urlsafe_b64encode(hash_).decode() + hash_fragment = encoded_hash[:8] + venv_name = "{0}-{1}".format(sanitized, hash_fragment) + return cls(cls.get_workon_home().joinpath(venv_name).as_posix()) + + @classmethod + def normalize_path(cls, path): + if not path: + return + if not path.is_absolute(): + try: + path = path.resolve() + except OSError: + path = path.absolute() + path = vistir.path.unicode_path("{0}".format(path)) + if os.name != "nt": + return path + + drive, tail = os.path.splitdrive(path) + # Only match (lower cased) local drives (e.g. 'c:'), not UNC mounts. + if drive.islower() and len(drive) == 2 and drive[1] == ":": + path = "{}{}".format(drive.upper(), tail) + + return vistir.path.unicode_path(path) + + @classmethod + def get_workon_home(cls): + workon_home = os.environ.get("WORKON_HOME") + if not workon_home: + if os.name == "nt": + workon_home = "~/.virtualenvs" + else: + workon_home = os.path.join( + os.environ.get("XDG_DATA_HOME", "~/.local/share"), "virtualenvs" + ) + return vistir.compat.Path(os.path.expandvars(workon_home)).expanduser() + + @cached_property + def script_basedir(self): + script_dir = os.path.basename(sysconfig.get_paths()["scripts"]) + return script_dir + + @property + def python(self): + return self.venv_dir.joinpath(self.script_basedir).joinpath("python").as_posix() + + @cached_property + def sys_path(self): + c = vistir.misc.run([self.python, "-c", "import json,sys; print(json.dumps(sys.path))"], + return_object=True, nospin=True) + assert c.returncode == 0, "failed loading virtualenv path" + path = json.loads(c.out.strip()) + return path + + @cached_property + def paths(self): + paths = {} + with vistir.contextmanagers.temp_environ(), vistir.contextmanagers.temp_path(): + os.environ["PYTHONUSERBASE"] = vistir.compat.fs_str(self.venv_dir.as_posix()) + os.environ["PYTHONIOENCODING"] = vistir.compat.fs_str("utf-8") + os.environ["PYTHONDONTWRITEBYTECODE"] = vistir.compat.fs_str("1") + importlib.reload(sysconfig) + scheme, _, _ = sysconfig._get_default_scheme().partition('_') + scheme = "{0}_user".format(scheme) + paths = sysconfig.get_paths(scheme=scheme) + return paths + + @property + def scripts_dir(self): + return self.paths["scripts"] + + @cached_property + def passa_entry(self): + import pkg_resources + return pkg_resources.working_set.by_key['passa'].location + + def get_distributions(self): + import pkg_resources + importlib.reload(pkg_resources) + return pkg_resources.find_distributions(self.paths["purelib"], only=True) + + def get_working_set(self): + working_set = None + import pkg_resources + passa_entry = self.passa_entry + with self.activated(): + working_set = pkg_resources.WorkingSet(self.sys_path + [passa_entry,]) + return working_set + + @classmethod + def filter_sources(cls, requirement, sources): + if not sources or not requirement.index: + return sources + filtered_sources = [ + source for source in sources + if source.get("name") == requirement.index + ] + return filtered_sources or sources + + def install(self, req, editable=False, sources=[]): + with self.activated(): + import passa.internals._pip_shims + importlib.reload(passa.internals._pip_shms) + ireq = req.as_ireq() + if editable: + with vistir.contextmanagers.cd(ireq.setup_py_dir): + c = self.run([self.python, "setup.py", "develop", "--no-deps"], cwd=ireq.setup_py_dir) + return c.returncode + importlib.reload(distlib.scripts) + sources = self.filter_sources(req, sources) + hashes = req.hashes + wheel = passa.internals._pip_shims.build_wheel(ireq, sources, hashes) + wheel.install(self.paths, distlib.scripts.ScriptMaker(None, None)) + + @contextlib.contextmanager + def activated(self): + original_path = sys.path + original_prefix = sys.prefix + original_user_base = os.environ.get("PYTHONUSERBASE", None) + original_venv = os.environ.get("VIRTUAL_ENV", None) + passa_path = vistir.compat.Path(__file__).absolute().parent.parent.as_posix() + with vistir.contextmanagers.temp_environ(), vistir.contextmanagers.temp_path(): + os.environ["PYTHONUSERBASE"] = vistir.compat.fs_str(self.venv_dir.as_posix()) + os.environ["PYTHONIOENCODING"] = vistir.compat.fs_str("utf-8") + os.environ["PYTHONDONTWRITEBYTECODE"] = vistir.compat.fs_str("1") + os.environ["VIRTUAL_ENV"] = vistir.compat.fs_str(self.venv_dir.as_posix()) + sys.path = self.sys_path + sys.prefix = self.venv_dir + sys.path.append(passa_path) + activate_this = os.path.join(self.scripts_dir, "activate_this.py") + with open(activate_this, "r") as f: + code = compile(f.read(), activate_this, "exec") + exec(code, dict(__file__=activate_this)) + try: + yield + finally: + print("Deactivating virtualenv...") + del os.environ["VIRTUAL_ENV"] + del os.environ["PYTHONUSERBASE"] + if original_user_base: + os.environ["PYTHONUSERBASE"] = original_user_base + if original_venv: + os.environ["VIRTUAL_ENV"] = original_venv + sys.path = original_path + sys.prefix = original_prefix + + def run(self, cmd, cwd=os.curdir): + c = None + with self.activated(): + script = vistir.cmdparse.Script.parse(cmd) + c = vistir.misc.run(script._parts, return_object=True, nospin=True, cwd=cwd) + return c + + def is_installed(self, pkgname): + return any(d for d in self.get_distributions() if d.project_name == pkgname) + + def uninstall(self, pkgname, *args, **kwargs): + with self.activated(): + from pip_shims.shims import InstallRequirement + module_name = InstallRequirement.__module__ + del InstallRequirement + if module_name not in sys.modules: + pip_req_install = importlib.import_module(module_name) + else: + pip_req_install = sys.modules[module_name] + importlib.reload(sys.modules[module_name]) + ireq = pip_req_install.InstallRequirement.from_line(pkgname) + return RequirementUninstaller(ireq, *args, **kwargs) From c431ae683a53d07111bb22893a3562b0b9d0b13a Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 18 Sep 2018 05:23:46 -0400 Subject: [PATCH 03/34] Add tests (clean is still failing) Signed-off-by: Dan Ryan --- tests/actions/test_clean.py | 30 ++++++++++++++--------------- tests/conftest.py | 38 ++++++++++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 16 deletions(-) diff --git a/tests/actions/test_clean.py b/tests/actions/test_clean.py index 3062ef5..95b0b14 100644 --- a/tests/actions/test_clean.py +++ b/tests/actions/test_clean.py @@ -1,18 +1,18 @@ # -*- coding=utf-8 -*- -from passa.actions.add import add_packages -from passa.cli.options import Project -import vistir -def test_clean_subset(project_directory): - with vistir.contextmanagers.cd(project_directory.strpath): - project = Project(project_directory.strpath) - retcode = add_packages(["requests"], project=project) - assert not retcode - packages = ["requests", "chardet", "certifi", "idna"] - assert all(pkg in project.lockfile.default for pkg in packages) - import passa.actions.clean - pass - # clean_retcode = passa.actions.clean.clean(project=project) - # assert not clean_retcode - # assert project.lockfile.default._data == {} +def test_clean_subset(project): + from passa.actions.add import add_packages + from passa.actions.clean import clean + retcode = add_packages(["requests"], project=project) + assert not retcode + packages = ["requests", "chardet", "certifi", "idna"] + c = project.venv.run("pip install pytz") + assert c.returncode == 0 + c = project.venv.run("python -c 'import pytz'") + assert c.returncode == 0 + clean_retcode = clean(project=project) + assert not clean_retcode + c = project.venv.run("python -c 'import pytz'") + assert c.returncode != 0 + assert all(pkg in project.lockfile.default for pkg in packages) diff --git a/tests/conftest.py b/tests/conftest.py index 577c942..e1326ca 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,10 @@ # -*- coding=utf-8 -*- +import os import pytest +import passa.cli.options +import passa.models.virtualenv +import sys +import vistir DEFAULT_PIPFILE_CONTENTS = """ @@ -18,4 +23,35 @@ def project_directory(tmpdir_factory): project_dir = tmpdir_factory.mktemp("passa-project") project_dir.join("Pipfile").write(DEFAULT_PIPFILE_CONTENTS) - return project_dir + with vistir.contextmanagers.cd(project_dir.strpath): + yield project_dir + + +@pytest.fixture(scope="function") +def virtualenv(tmpdir_factory): + venv_dir = tmpdir_factory.mktemp("passa-testenv") + print("Creating virtualenv {0!r}".format(venv_dir.strpath)) + c = vistir.misc.run([sys.executable, "-m", "virtualenv", venv_dir.strpath], + return_object=True, block=True, nospin=True) + if c.returncode == 0: + print("Virtualenv created...") + return venv_dir + raise RuntimeError("Failed creating virtualenv for testing...{0!r}".format(c.err.strip())) + + +class _Project(passa.cli.options.Project): + def __init__(self, root, venv=None): + self.path = os.path.abspath(root) + self.venv = venv + super(_Project, self).__init__(self.path) + + +@pytest.fixture +def tmpvenv(virtualenv): + return passa.models.virtualenv.VirtualEnv(virtualenv) + + +@pytest.fixture(scope="function") +def project(project_directory, tmpvenv): + with tmpvenv.activated(): + yield _Project(project_directory, tmpvenv) From bf4fa23b1d8909312dccf05ebbe9bbf05bc9da2e Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 18 Sep 2018 18:32:56 -0400 Subject: [PATCH 04/34] Wheel installation into virtualenv works Signed-off-by: Dan Ryan --- src/passa/models/virtualenv.py | 69 +++++++++++++++++++++++++++++++--- 1 file changed, 64 insertions(+), 5 deletions(-) diff --git a/src/passa/models/virtualenv.py b/src/passa/models/virtualenv.py index 78484c2..54d5682 100644 --- a/src/passa/models/virtualenv.py +++ b/src/passa/models/virtualenv.py @@ -8,6 +8,7 @@ import json import os import re +import six import sys import sysconfig @@ -96,6 +97,19 @@ def sys_path(self): path = json.loads(c.out.strip()) return path + @cached_property + def system_paths(self): + paths = {} + importlib.reload(sysconfig) + paths = sysconfig.get_paths() + return paths + + @cached_property + def sys_prefix(self): + c = self.run_py(["-c", "'import sys; print(sys.prefix)'"]) + sys_prefix = vistir.misc.to_text(c.out).strip() + return sys_prefix + @cached_property def paths(self): paths = {} @@ -141,19 +155,45 @@ def filter_sources(cls, requirement, sources): ] return filtered_sources or sources + @cached_property + def python_version(self): + with self.activated(): + importlib.reload(sysconfig) + py_version = sysconfig.get_python_version() + return py_version + + def get_setup_install_args(self, pkgname, setup_py, develop=False): + headers = vistir.compat.Path(self.sys_prefix) / "include" / "site" + headers = headers / "python{0}".format(self.python_version) / pkgname + install_arg = "install" if not develop else "develop" + return [ + self.python, "-u", "-c", SETUPTOOLS_SHIM % setup_py, install_arg, + "--single-version-externally-managed", "root={0}".format(), + "--install-headers={0}".format(headers.as_posix()), + "--install-purelib={0}".format(self.paths["purelib"]), + "--install-platlib={0}".format(self.paths["platlib"]), + "--install-scripts={0}".format(self.scripts_dir), + "--install-data={0}".format(self.paths["data"]), + ] + def install(self, req, editable=False, sources=[]): with self.activated(): - import passa.internals._pip_shims - importlib.reload(passa.internals._pip_shms) + import passa.internals._pip + install_options = ["--prefix={0}".format(self.venv_dir),] + importlib.reload(passa.internals._pip) ireq = req.as_ireq() if editable: - with vistir.contextmanagers.cd(ireq.setup_py_dir): - c = self.run([self.python, "setup.py", "develop", "--no-deps"], cwd=ireq.setup_py_dir) + with vistir.contextmanagers.cd(ireq.setup_py_dir, ireq.setup_py): + c = self.run( + install_options + self.get_setup_install_args( + req.name, develop=editable + ), cwd=ireq.setup_py_dir + ) return c.returncode importlib.reload(distlib.scripts) sources = self.filter_sources(req, sources) hashes = req.hashes - wheel = passa.internals._pip_shims.build_wheel(ireq, sources, hashes) + wheel = passa.internals._pip.build_wheel(ireq, sources, hashes) wheel.install(self.paths, distlib.scripts.ScriptMaker(None, None)) @contextlib.contextmanager @@ -195,6 +235,16 @@ def run(self, cmd, cwd=os.curdir): c = vistir.misc.run(script._parts, return_object=True, nospin=True, cwd=cwd) return c + def run_py(self, cmd, cwd=os.curdir): + c = None + if isinstance(cmd, six.string_types): + script = vistir.cmdparse.Script.parse("{0} {1}".format(self.python, cmd)) + else: + script = vistir.cmdparse.Script.parse([self.python,] + list(cmd)) + with self.activated(): + c = vistir.misc.run(script._parts, return_object=True, nospin=True, cwd=cwd) + return c + def is_installed(self, pkgname): return any(d for d in self.get_distributions() if d.project_name == pkgname) @@ -210,3 +260,12 @@ def uninstall(self, pkgname, *args, **kwargs): importlib.reload(sys.modules[module_name]) ireq = pip_req_install.InstallRequirement.from_line(pkgname) return RequirementUninstaller(ireq, *args, **kwargs) + + +SETUPTOOLS_SHIM = ( + "import setuptools, tokenize;__file__=%r;" + "f=getattr(tokenize, 'open', open)(__file__);" + "code=f.read().replace('\\r\\n', '\\n');" + "f.close();" + "exec(compile(code, __file__, 'exec'))" +) From ac6929bf7bd784d802779f64cfd38fcca69e54be Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 18 Sep 2018 18:33:15 -0400 Subject: [PATCH 05/34] Update setup.cfg Signed-off-by: Dan Ryan --- setup.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.cfg b/setup.cfg index 98ea198..4daf9ee 100644 --- a/setup.cfg +++ b/setup.cfg @@ -45,6 +45,7 @@ install_requires = resolvelib>=0.2.1,!=1.0.0.dev0 requirementslib>=1.1.1 six + virtualenv vistir[spinner]>=0.1.4 [options.extras_require] @@ -52,6 +53,7 @@ pack = invoke parver tests = + cached-property pytest-xdist pytest-timeout pytest-cov From 194cd50c776e51f5edb2347edcda1508d9d45fcb Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 19 Sep 2018 01:19:33 -0400 Subject: [PATCH 06/34] Implement working versions of clean and install inside virtualenvs Signed-off-by: Dan Ryan --- src/passa/models/synchronizers.py | 2 +- src/passa/models/virtualenv.py | 91 ++++++++++++++++++++++++------- 2 files changed, 71 insertions(+), 22 deletions(-) diff --git a/src/passa/models/synchronizers.py b/src/passa/models/synchronizers.py index 8fc3e0a..33fcfc3 100644 --- a/src/passa/models/synchronizers.py +++ b/src/passa/models/synchronizers.py @@ -135,7 +135,7 @@ def _clean(names, venv=None): if name in PROTECTED_FROM_CLEAN: continue with _remove_package(name, venv=venv) as uninst: - if uninst: + if uninst.paths: cleaned.add(name) return cleaned diff --git a/src/passa/models/virtualenv.py b/src/passa/models/virtualenv.py index 54d5682..c179e3c 100644 --- a/src/passa/models/virtualenv.py +++ b/src/passa/models/virtualenv.py @@ -12,12 +12,11 @@ import sys import sysconfig +import passa.internals._pip from cached_property import cached_property import vistir -from ..internals._pip import RequirementUninstaller - class VirtualEnv(object): def __init__(self, venv_dir): @@ -100,7 +99,7 @@ def sys_path(self): @cached_property def system_paths(self): paths = {} - importlib.reload(sysconfig) + six.moves.reload_module(sysconfig) paths = sysconfig.get_paths() return paths @@ -117,7 +116,7 @@ def paths(self): os.environ["PYTHONUSERBASE"] = vistir.compat.fs_str(self.venv_dir.as_posix()) os.environ["PYTHONIOENCODING"] = vistir.compat.fs_str("utf-8") os.environ["PYTHONDONTWRITEBYTECODE"] = vistir.compat.fs_str("1") - importlib.reload(sysconfig) + six.moves.reload_module(sysconfig) scheme, _, _ = sysconfig._get_default_scheme().partition('_') scheme = "{0}_user".format(scheme) paths = sysconfig.get_paths(scheme=scheme) @@ -134,15 +133,15 @@ def passa_entry(self): def get_distributions(self): import pkg_resources - importlib.reload(pkg_resources) + six.moves.reload_module(pkg_resources) return pkg_resources.find_distributions(self.paths["purelib"], only=True) def get_working_set(self): working_set = None import pkg_resources passa_entry = self.passa_entry - with self.activated(): - working_set = pkg_resources.WorkingSet(self.sys_path + [passa_entry,]) + monkeypatch = self.monkeypatch_dist + working_set = pkg_resources.WorkingSet(self.sys_path + [passa_entry, monkeypatch]) return working_set @classmethod @@ -158,10 +157,26 @@ def filter_sources(cls, requirement, sources): @cached_property def python_version(self): with self.activated(): - importlib.reload(sysconfig) + six.moves.reload_module(sysconfig) py_version = sysconfig.get_python_version() return py_version + @classmethod + def safe_import(cls, name): + module = None + if name not in sys.modules: + module = importlib.import_module(name) + else: + module = sys.modules[name] + six.moves.reload_module(module) + return module + + @cached_property + def monkeypatch_dist(self): + pkg_resources = self.safe_import("pkg_resources") + monkey_patch = pkg_resources.get_distribution('recursive-monkey-patch').location + return monkey_patch + def get_setup_install_args(self, pkgname, setup_py, develop=False): headers = vistir.compat.Path(self.sys_prefix) / "include" / "site" headers = headers / "python{0}".format(self.python_version) / pkgname @@ -178,9 +193,8 @@ def get_setup_install_args(self, pkgname, setup_py, develop=False): def install(self, req, editable=False, sources=[]): with self.activated(): - import passa.internals._pip install_options = ["--prefix={0}".format(self.venv_dir),] - importlib.reload(passa.internals._pip) + six.moves.reload_module(passa.internals._pip) ireq = req.as_ireq() if editable: with vistir.contextmanagers.cd(ireq.setup_py_dir, ireq.setup_py): @@ -190,7 +204,7 @@ def install(self, req, editable=False, sources=[]): ), cwd=ireq.setup_py_dir ) return c.returncode - importlib.reload(distlib.scripts) + six.moves.reload_module(distlib.scripts) sources = self.filter_sources(req, sources) hashes = req.hashes wheel = passa.internals._pip.build_wheel(ireq, sources, hashes) @@ -203,6 +217,7 @@ def activated(self): original_user_base = os.environ.get("PYTHONUSERBASE", None) original_venv = os.environ.get("VIRTUAL_ENV", None) passa_path = vistir.compat.Path(__file__).absolute().parent.parent.as_posix() + monkeypatch_dist = self.monkeypatch_dist with vistir.contextmanagers.temp_environ(), vistir.contextmanagers.temp_path(): os.environ["PYTHONUSERBASE"] = vistir.compat.fs_str(self.venv_dir.as_posix()) os.environ["PYTHONIOENCODING"] = vistir.compat.fs_str("utf-8") @@ -210,11 +225,15 @@ def activated(self): os.environ["VIRTUAL_ENV"] = vistir.compat.fs_str(self.venv_dir.as_posix()) sys.path = self.sys_path sys.prefix = self.venv_dir - sys.path.append(passa_path) + import site + # sys.path.append(passa_path) activate_this = os.path.join(self.scripts_dir, "activate_this.py") with open(activate_this, "r") as f: code = compile(f.read(), activate_this, "exec") exec(code, dict(__file__=activate_this)) + site.addsitedir(passa_path) + site.addsitedir(monkeypatch_dist) + pkg_resources = self.safe_import("pkg_resources") try: yield finally: @@ -227,6 +246,7 @@ def activated(self): os.environ["VIRTUAL_ENV"] = original_venv sys.path = original_path sys.prefix = original_prefix + six.moves.reload_module(pkg_resources) def run(self, cmd, cwd=os.curdir): c = None @@ -248,18 +268,42 @@ def run_py(self, cmd, cwd=os.curdir): def is_installed(self, pkgname): return any(d for d in self.get_distributions() if d.project_name == pkgname) + def get_monkeypatched_pathset(self): + import recursive_monkey_patch + from pip_shims.shims import req_install + req_uninstall_name = "{0}.req_uninstall".format(req_install.__package__) + if req_uninstall_name not in sys.modules: + req_uninstall = importlib.import_module(req_uninstall_name) + else: + req_uninstall = sys.modules[req_uninstall_name] + six.moves.reload_module(req_uninstall) + recursive_monkey_patch.monkey_patch(PatchedUninstaller, req_uninstall.UninstallPathSet) + return req_uninstall.UninstallPathSet + + @contextlib.contextmanager def uninstall(self, pkgname, *args, **kwargs): + auto_confirm = kwargs.pop("auto_confirm", True) + verbose = kwargs.pop("verbose", False) with self.activated(): - from pip_shims.shims import InstallRequirement - module_name = InstallRequirement.__module__ - del InstallRequirement - if module_name not in sys.modules: - pip_req_install = importlib.import_module(module_name) + pathset_base = self.get_monkeypatched_pathset() + dist = next( + iter(filter(lambda d: d.project_name == pkgname, self.get_working_set())), + None + ) + pathset = pathset_base.from_dist(dist) + print(pathset.paths) + if pathset is not None: + pathset.remove(auto_confirm=auto_confirm, verbose=True) + try: + yield pathset + except Exception as e: + if pathset is not None: + pathset.rollback() else: - pip_req_install = sys.modules[module_name] - importlib.reload(sys.modules[module_name]) - ireq = pip_req_install.InstallRequirement.from_line(pkgname) - return RequirementUninstaller(ireq, *args, **kwargs) + if pathset is not None: + pathset.commit() + if pathset is None: + return SETUPTOOLS_SHIM = ( @@ -269,3 +313,8 @@ def uninstall(self, pkgname, *args, **kwargs): "f.close();" "exec(compile(code, __file__, 'exec'))" ) + + +class PatchedUninstaller(object): + def _permitted(self, path): + return True From 8ee990eb01a58adb5f24d065efeabeb18ea87903 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 20 Sep 2018 01:15:03 -0400 Subject: [PATCH 07/34] Fix cleaning working set and messaging Signed-off-by: Dan Ryan --- src/passa/actions/clean.py | 3 +- src/passa/actions/init.py | 2 +- src/passa/models/synchronizers.py | 11 +++- src/passa/models/virtualenv.py | 103 ++++++++++++++---------------- src/passa/operations/sync.py | 4 +- 5 files changed, 62 insertions(+), 61 deletions(-) diff --git a/src/passa/actions/clean.py b/src/passa/actions/clean.py index 8c6ae17..9006f22 100644 --- a/src/passa/actions/clean.py +++ b/src/passa/actions/clean.py @@ -13,4 +13,5 @@ def clean(project, default=True, dev=False, sync=True): if not success: return 1 - print("Cleaned project at", project.root) + if sync: + print("Cleaned project at", project.root) diff --git a/src/passa/actions/init.py b/src/passa/actions/init.py index 1d9f592..bbab009 100644 --- a/src/passa/actions/init.py +++ b/src/passa/actions/init.py @@ -42,7 +42,7 @@ def init_project(root=None, python_version=None): index_urls = [parsed.index_url] + parsed.extra_index_urls sources = get_sources(index_urls, parsed.trusted_hosts) data = { - "sources": sources, + "source": sources, "packages": {}, "dev-packages": {}, } diff --git a/src/passa/models/synchronizers.py b/src/passa/models/synchronizers.py index 33fcfc3..9c8aa9c 100644 --- a/src/passa/models/synchronizers.py +++ b/src/passa/models/synchronizers.py @@ -234,7 +234,7 @@ def _sync(self): class Cleaner(object): """Helper class to clean packages not in a project's lock file. """ - def __init__(self, project, default, develop, sync=True): + def __init__(self, project, default, develop, sync=True, verbose=False): self._root = project.root # Only for repr. self.packages = _get_packages(project.lockfile, default, develop) self.sync = sync @@ -243,9 +243,18 @@ def __init__(self, project, default, develop, sync=True): def __repr__(self): return "<{0} @ {1!r}>".format(type(self).__name__, self._root) + def print(self, packages): + if not self.sync: + message = "Would clean: {0}" + else: + message = "Cleaned: {0}" + print(message.format(", ".join(sorted(set(packages))))) + def clean(self): groupcoll = _group_installed_names(self.packages, venv=self.project.venv) cleaned = set() if self.sync: cleaned = _clean(groupcoll.unneeded, venv=self.project.venv) + else: + return groupcoll.unneeded return cleaned diff --git a/src/passa/models/virtualenv.py b/src/passa/models/virtualenv.py index c179e3c..a4f3954 100644 --- a/src/passa/models/virtualenv.py +++ b/src/passa/models/virtualenv.py @@ -6,6 +6,7 @@ import hashlib import importlib import json +import posixpath import os import re import six @@ -20,6 +21,7 @@ class VirtualEnv(object): def __init__(self, venv_dir): + self.recursive_monkey_patch = self.safe_import("recursive_monkey_patch") self.venv_dir = vistir.compat.Path(venv_dir) @classmethod @@ -79,6 +81,26 @@ def get_workon_home(cls): ) return vistir.compat.Path(os.path.expandvars(workon_home)).expanduser() + @classmethod + def filter_sources(cls, requirement, sources): + if not sources or not requirement.index: + return sources + filtered_sources = [ + source for source in sources + if source.get("name") == requirement.index + ] + return filtered_sources or sources + + @classmethod + def safe_import(cls, name): + module = None + if name not in sys.modules: + module = importlib.import_module(name) + else: + module = sys.modules[name] + six.moves.reload_module(module) + return module + @cached_property def script_basedir(self): script_dir = os.path.basename(sysconfig.get_paths()["scripts"]) @@ -93,13 +115,16 @@ def sys_path(self): c = vistir.misc.run([self.python, "-c", "import json,sys; print(json.dumps(sys.path))"], return_object=True, nospin=True) assert c.returncode == 0, "failed loading virtualenv path" - path = json.loads(c.out.strip()) + path = [ + path for path in json.loads(c.out.strip()) + if posixpath.normpath(path).startswith(posixpath.normpath(str(self.venv_dir))) + ] return path @cached_property def system_paths(self): paths = {} - six.moves.reload_module(sysconfig) + sysconfig = self.safe_import("sysconfig") paths = sysconfig.get_paths() return paths @@ -116,7 +141,7 @@ def paths(self): os.environ["PYTHONUSERBASE"] = vistir.compat.fs_str(self.venv_dir.as_posix()) os.environ["PYTHONIOENCODING"] = vistir.compat.fs_str("utf-8") os.environ["PYTHONDONTWRITEBYTECODE"] = vistir.compat.fs_str("1") - six.moves.reload_module(sysconfig) + sysconfig = self.safe_import("sysconfig") scheme, _, _ = sysconfig._get_default_scheme().partition('_') scheme = "{0}_user".format(scheme) paths = sysconfig.get_paths(scheme=scheme) @@ -128,55 +153,27 @@ def scripts_dir(self): @cached_property def passa_entry(self): - import pkg_resources + pkg_resources = self.safe_import("pkg_resources") return pkg_resources.working_set.by_key['passa'].location def get_distributions(self): - import pkg_resources - six.moves.reload_module(pkg_resources) + pkg_resources = self.safe_import("pkg_resources") return pkg_resources.find_distributions(self.paths["purelib"], only=True) def get_working_set(self): working_set = None import pkg_resources passa_entry = self.passa_entry - monkeypatch = self.monkeypatch_dist - working_set = pkg_resources.WorkingSet(self.sys_path + [passa_entry, monkeypatch]) + working_set = pkg_resources.WorkingSet(self.sys_path + [passa_entry]) return working_set - @classmethod - def filter_sources(cls, requirement, sources): - if not sources or not requirement.index: - return sources - filtered_sources = [ - source for source in sources - if source.get("name") == requirement.index - ] - return filtered_sources or sources - @cached_property def python_version(self): with self.activated(): - six.moves.reload_module(sysconfig) + sysconfig = self.safe_import("sysconfig") py_version = sysconfig.get_python_version() return py_version - @classmethod - def safe_import(cls, name): - module = None - if name not in sys.modules: - module = importlib.import_module(name) - else: - module = sys.modules[name] - six.moves.reload_module(module) - return module - - @cached_property - def monkeypatch_dist(self): - pkg_resources = self.safe_import("pkg_resources") - monkey_patch = pkg_resources.get_distribution('recursive-monkey-patch').location - return monkey_patch - def get_setup_install_args(self, pkgname, setup_py, develop=False): headers = vistir.compat.Path(self.sys_prefix) / "include" / "site" headers = headers / "python{0}".format(self.python_version) / pkgname @@ -194,7 +191,7 @@ def get_setup_install_args(self, pkgname, setup_py, develop=False): def install(self, req, editable=False, sources=[]): with self.activated(): install_options = ["--prefix={0}".format(self.venv_dir),] - six.moves.reload_module(passa.internals._pip) + passa_pip = self.safe_import("passa.internals._pip") ireq = req.as_ireq() if editable: with vistir.contextmanagers.cd(ireq.setup_py_dir, ireq.setup_py): @@ -204,11 +201,11 @@ def install(self, req, editable=False, sources=[]): ), cwd=ireq.setup_py_dir ) return c.returncode - six.moves.reload_module(distlib.scripts) + distlib_scripts = self.safe_import("distlib.scripts") sources = self.filter_sources(req, sources) hashes = req.hashes - wheel = passa.internals._pip.build_wheel(ireq, sources, hashes) - wheel.install(self.paths, distlib.scripts.ScriptMaker(None, None)) + wheel = passa_pip.build_wheel(ireq, sources, hashes) + wheel.install(self.paths, distlib_scripts.ScriptMaker(None, None)) @contextlib.contextmanager def activated(self): @@ -217,22 +214,20 @@ def activated(self): original_user_base = os.environ.get("PYTHONUSERBASE", None) original_venv = os.environ.get("VIRTUAL_ENV", None) passa_path = vistir.compat.Path(__file__).absolute().parent.parent.as_posix() - monkeypatch_dist = self.monkeypatch_dist with vistir.contextmanagers.temp_environ(), vistir.contextmanagers.temp_path(): - os.environ["PYTHONUSERBASE"] = vistir.compat.fs_str(self.venv_dir.as_posix()) os.environ["PYTHONIOENCODING"] = vistir.compat.fs_str("utf-8") os.environ["PYTHONDONTWRITEBYTECODE"] = vistir.compat.fs_str("1") - os.environ["VIRTUAL_ENV"] = vistir.compat.fs_str(self.venv_dir.as_posix()) - sys.path = self.sys_path - sys.prefix = self.venv_dir - import site - # sys.path.append(passa_path) activate_this = os.path.join(self.scripts_dir, "activate_this.py") with open(activate_this, "r") as f: code = compile(f.read(), activate_this, "exec") exec(code, dict(__file__=activate_this)) + os.environ["PYTHONUSERBASE"] = vistir.compat.fs_str(self.venv_dir.as_posix()) + os.environ["VIRTUAL_ENV"] = vistir.compat.fs_str(self.venv_dir.as_posix()) + sys.path = self.sys_path + sys.prefix = self.venv_dir + site = self.safe_import("site") site.addsitedir(passa_path) - site.addsitedir(monkeypatch_dist) + sys.modules["recursive_monkey_patch"] = self.recursive_monkey_patch pkg_resources = self.safe_import("pkg_resources") try: yield @@ -269,15 +264,12 @@ def is_installed(self, pkgname): return any(d for d in self.get_distributions() if d.project_name == pkgname) def get_monkeypatched_pathset(self): - import recursive_monkey_patch from pip_shims.shims import req_install req_uninstall_name = "{0}.req_uninstall".format(req_install.__package__) - if req_uninstall_name not in sys.modules: - req_uninstall = importlib.import_module(req_uninstall_name) - else: - req_uninstall = sys.modules[req_uninstall_name] - six.moves.reload_module(req_uninstall) - recursive_monkey_patch.monkey_patch(PatchedUninstaller, req_uninstall.UninstallPathSet) + req_uninstall = self.safe_import(req_uninstall_name) + self.recursive_monkey_patch.monkey_patch( + PatchedUninstaller, req_uninstall.UninstallPathSet + ) return req_uninstall.UninstallPathSet @contextlib.contextmanager @@ -291,9 +283,8 @@ def uninstall(self, pkgname, *args, **kwargs): None ) pathset = pathset_base.from_dist(dist) - print(pathset.paths) if pathset is not None: - pathset.remove(auto_confirm=auto_confirm, verbose=True) + pathset.remove(auto_confirm=auto_confirm, verbose=verbose) try: yield pathset except Exception as e: diff --git a/src/passa/operations/sync.py b/src/passa/operations/sync.py index 3014e8d..45502a4 100644 --- a/src/passa/operations/sync.py +++ b/src/passa/operations/sync.py @@ -16,8 +16,8 @@ def sync(syncer): def clean(cleaner): - print("Cleaning") + print("Cleaning...") cleaned = cleaner.clean() if cleaned: - print("Uninstalled: {}".format(", ".join(sorted(cleaned)))) + cleaner.print(cleaned) return True From b1a7bf7ca8fb9a23f5675b96a60fba0700d738f7 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 27 Sep 2018 19:52:41 -0400 Subject: [PATCH 08/34] Fix tests Signed-off-by: Dan Ryan --- tests/actions/test_clean.py | 8 ++++---- tests/conftest.py | 12 ++++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/actions/test_clean.py b/tests/actions/test_clean.py index 95b0b14..c56d06d 100644 --- a/tests/actions/test_clean.py +++ b/tests/actions/test_clean.py @@ -1,17 +1,17 @@ # -*- coding=utf-8 -*- +import passa.actions.add +import passa.actions.clean def test_clean_subset(project): - from passa.actions.add import add_packages - from passa.actions.clean import clean - retcode = add_packages(["requests"], project=project) + retcode = passa.actions.add.add_packages(["requests"], project=project) assert not retcode packages = ["requests", "chardet", "certifi", "idna"] c = project.venv.run("pip install pytz") assert c.returncode == 0 c = project.venv.run("python -c 'import pytz'") assert c.returncode == 0 - clean_retcode = clean(project=project) + clean_retcode = passa.actions.clean.clean(project=project) assert not clean_retcode c = project.venv.run("python -c 'import pytz'") assert c.returncode != 0 diff --git a/tests/conftest.py b/tests/conftest.py index e1326ca..1aaf419 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,9 @@ # -*- coding=utf-8 -*- import os import pytest +import passa import passa.cli.options -import passa.models.virtualenv +import mork.virtualenv import sys import vistir @@ -41,17 +42,20 @@ def virtualenv(tmpdir_factory): class _Project(passa.cli.options.Project): def __init__(self, root, venv=None): - self.path = os.path.abspath(root) + self.path = root.strpath self.venv = venv super(_Project, self).__init__(self.path) @pytest.fixture def tmpvenv(virtualenv): - return passa.models.virtualenv.VirtualEnv(virtualenv) + return mork.virtualenv.VirtualEnv(virtualenv.strpath) @pytest.fixture(scope="function") def project(project_directory, tmpvenv): - with tmpvenv.activated(): + venv_working_set = tmpvenv.initial_working_set + passa_dist = venv_working_set.by_key["passa"] + resolved = tmpvenv.resolve_dist(passa_dist, venv_working_set) + with tmpvenv.activated(extra_dists=list(resolved)): yield _Project(project_directory, tmpvenv) From 2a40250fe6173a655e0f33c319b2319c52164a28 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 27 Sep 2018 19:58:07 -0400 Subject: [PATCH 09/34] Update tox and add mork, installer and virtualenv dependencies Signed-off-by: Dan Ryan --- docs/requirements.txt | 2 ++ setup.cfg | 4 ++++ tox.ini | 4 ++-- 3 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 docs/requirements.txt diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..8213302 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,2 @@ +sphinx +sphinx_rtd_theme diff --git a/setup.cfg b/setup.cfg index 4daf9ee..ace8c27 100644 --- a/setup.cfg +++ b/setup.cfg @@ -39,6 +39,8 @@ install_requires = appdirs distlib packaging + packagebuilder + installer pip-shims>=0.1.2 plette[validation]>=0.2.2 requests @@ -52,6 +54,8 @@ install_requires = pack = invoke parver +virtualenv = + mork tests = cached-property pytest-xdist diff --git a/tox.ini b/tox.ini index 2bc8e1d..d436e33 100644 --- a/tox.ini +++ b/tox.ini @@ -8,7 +8,7 @@ setenv = LC_ALL = en_US.UTF-8 deps = coverage - -e .[tests] + -e .[tests,virtualenv] commands = coverage run --parallel -m pytest --timeout 300 [] install_command = python -m pip install {opts} {packages} usedevelop = True @@ -23,7 +23,7 @@ commands = [testenv:docs] deps = -r{toxinidir}/docs/requirements.txt - -e .[tests] + -e .[tests,virtualenv] commands = sphinx-build -d {envtmpdir}/doctrees -b html docs docs/build/html sphinx-build -d {envtmpdir}/doctrees -b man docs docs/build/man From afe8e519d85b3468d096198e98fbebd98fa2e545 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 28 Sep 2018 01:48:24 -0400 Subject: [PATCH 10/34] Swap to mork implementation for testing Signed-off-by: Dan Ryan --- src/passa/cli/options.py | 6 +- src/passa/models/synchronizers.py | 10 +- src/passa/models/virtualenv.py | 311 ------------------------------ tests/conftest.py | 7 +- 4 files changed, 12 insertions(+), 322 deletions(-) delete mode 100644 src/passa/models/virtualenv.py diff --git a/src/passa/cli/options.py b/src/passa/cli/options.py index 07782db..3abcadb 100644 --- a/src/passa/cli/options.py +++ b/src/passa/cli/options.py @@ -10,7 +10,7 @@ import tomlkit.exceptions import passa.models.projects -import passa.models.virtualenv +import mork import vistir @@ -36,8 +36,8 @@ def __init__(self, root, *args, **kwargs): def get_venv(self, root): if 'VIRTUAL_ENV' in os.environ: - return passa.models.virtualenv.VirtualEnv(os.environ['VIRTUAL_ENV']) - return passa.models.virtualenv.VirtualEnv.from_project_path(root) + return mork.VirtualEnv(os.environ['VIRTUAL_ENV']) + return mork.VirtualEnv.from_project_path(root) def __name__(self): return "Project Root" diff --git a/src/passa/models/synchronizers.py b/src/passa/models/synchronizers.py index 9c8aa9c..78b3bf7 100644 --- a/src/passa/models/synchronizers.py +++ b/src/passa/models/synchronizers.py @@ -1,6 +1,6 @@ # -*- coding=utf-8 -*- -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, unicode_literals, print_function import collections import contextlib @@ -14,8 +14,6 @@ import packaging.version import requirementslib -from .virtualenv import VirtualEnv - from ..internals._pip import uninstall, EditableInstaller, WheelInstaller @@ -135,7 +133,7 @@ def _clean(names, venv=None): if name in PROTECTED_FROM_CLEAN: continue with _remove_package(name, venv=venv) as uninst: - if uninst.paths: + if uninst: cleaned.add(name) return cleaned @@ -244,11 +242,13 @@ def __repr__(self): return "<{0} @ {1!r}>".format(type(self).__name__, self._root) def print(self, packages): + message = "" if not self.sync: message = "Would clean: {0}" else: message = "Cleaned: {0}" - print(message.format(", ".join(sorted(set(packages))))) + packages = ", ".join(sorted(set(packages))) if packages else "" + print(message.format(packages)) def clean(self): groupcoll = _group_installed_names(self.packages, venv=self.project.venv) diff --git a/src/passa/models/virtualenv.py b/src/passa/models/virtualenv.py deleted file mode 100644 index a4f3954..0000000 --- a/src/passa/models/virtualenv.py +++ /dev/null @@ -1,311 +0,0 @@ -# -*- coding=utf-8 -*- - -import base64 -import contextlib -import distlib.scripts -import hashlib -import importlib -import json -import posixpath -import os -import re -import six -import sys -import sysconfig - -import passa.internals._pip -from cached_property import cached_property - -import vistir - - -class VirtualEnv(object): - def __init__(self, venv_dir): - self.recursive_monkey_patch = self.safe_import("recursive_monkey_patch") - self.venv_dir = vistir.compat.Path(venv_dir) - - @classmethod - def from_project_path(cls, path): - path = vistir.compat.Path(path) - if path.name == 'Pipfile': - pipfile_path = path - path = path.parent - else: - pipfile_path = path / 'Pipfile' - pipfile_location = cls.normalize_path(pipfile_path) - venv_path = path / '.venv' - if venv_path.exists(): - if not venv_path.is_dir(): - possible_path = vistir.compat.Path(venv_path.read_text().strip()) - if possible_path.exists(): - return cls(possible_path.as_posix()) - else: - if venv_path.joinpath('lib').exists(): - return cls(venv_path.as_posix()) - sanitized = re.sub(r'[ $`!*@"\\\r\n\t]', "_", path.name)[0:42] - hash_ = hashlib.sha256(pipfile_location.encode()).digest()[:6] - encoded_hash = base64.urlsafe_b64encode(hash_).decode() - hash_fragment = encoded_hash[:8] - venv_name = "{0}-{1}".format(sanitized, hash_fragment) - return cls(cls.get_workon_home().joinpath(venv_name).as_posix()) - - @classmethod - def normalize_path(cls, path): - if not path: - return - if not path.is_absolute(): - try: - path = path.resolve() - except OSError: - path = path.absolute() - path = vistir.path.unicode_path("{0}".format(path)) - if os.name != "nt": - return path - - drive, tail = os.path.splitdrive(path) - # Only match (lower cased) local drives (e.g. 'c:'), not UNC mounts. - if drive.islower() and len(drive) == 2 and drive[1] == ":": - path = "{}{}".format(drive.upper(), tail) - - return vistir.path.unicode_path(path) - - @classmethod - def get_workon_home(cls): - workon_home = os.environ.get("WORKON_HOME") - if not workon_home: - if os.name == "nt": - workon_home = "~/.virtualenvs" - else: - workon_home = os.path.join( - os.environ.get("XDG_DATA_HOME", "~/.local/share"), "virtualenvs" - ) - return vistir.compat.Path(os.path.expandvars(workon_home)).expanduser() - - @classmethod - def filter_sources(cls, requirement, sources): - if not sources or not requirement.index: - return sources - filtered_sources = [ - source for source in sources - if source.get("name") == requirement.index - ] - return filtered_sources or sources - - @classmethod - def safe_import(cls, name): - module = None - if name not in sys.modules: - module = importlib.import_module(name) - else: - module = sys.modules[name] - six.moves.reload_module(module) - return module - - @cached_property - def script_basedir(self): - script_dir = os.path.basename(sysconfig.get_paths()["scripts"]) - return script_dir - - @property - def python(self): - return self.venv_dir.joinpath(self.script_basedir).joinpath("python").as_posix() - - @cached_property - def sys_path(self): - c = vistir.misc.run([self.python, "-c", "import json,sys; print(json.dumps(sys.path))"], - return_object=True, nospin=True) - assert c.returncode == 0, "failed loading virtualenv path" - path = [ - path for path in json.loads(c.out.strip()) - if posixpath.normpath(path).startswith(posixpath.normpath(str(self.venv_dir))) - ] - return path - - @cached_property - def system_paths(self): - paths = {} - sysconfig = self.safe_import("sysconfig") - paths = sysconfig.get_paths() - return paths - - @cached_property - def sys_prefix(self): - c = self.run_py(["-c", "'import sys; print(sys.prefix)'"]) - sys_prefix = vistir.misc.to_text(c.out).strip() - return sys_prefix - - @cached_property - def paths(self): - paths = {} - with vistir.contextmanagers.temp_environ(), vistir.contextmanagers.temp_path(): - os.environ["PYTHONUSERBASE"] = vistir.compat.fs_str(self.venv_dir.as_posix()) - os.environ["PYTHONIOENCODING"] = vistir.compat.fs_str("utf-8") - os.environ["PYTHONDONTWRITEBYTECODE"] = vistir.compat.fs_str("1") - sysconfig = self.safe_import("sysconfig") - scheme, _, _ = sysconfig._get_default_scheme().partition('_') - scheme = "{0}_user".format(scheme) - paths = sysconfig.get_paths(scheme=scheme) - return paths - - @property - def scripts_dir(self): - return self.paths["scripts"] - - @cached_property - def passa_entry(self): - pkg_resources = self.safe_import("pkg_resources") - return pkg_resources.working_set.by_key['passa'].location - - def get_distributions(self): - pkg_resources = self.safe_import("pkg_resources") - return pkg_resources.find_distributions(self.paths["purelib"], only=True) - - def get_working_set(self): - working_set = None - import pkg_resources - passa_entry = self.passa_entry - working_set = pkg_resources.WorkingSet(self.sys_path + [passa_entry]) - return working_set - - @cached_property - def python_version(self): - with self.activated(): - sysconfig = self.safe_import("sysconfig") - py_version = sysconfig.get_python_version() - return py_version - - def get_setup_install_args(self, pkgname, setup_py, develop=False): - headers = vistir.compat.Path(self.sys_prefix) / "include" / "site" - headers = headers / "python{0}".format(self.python_version) / pkgname - install_arg = "install" if not develop else "develop" - return [ - self.python, "-u", "-c", SETUPTOOLS_SHIM % setup_py, install_arg, - "--single-version-externally-managed", "root={0}".format(), - "--install-headers={0}".format(headers.as_posix()), - "--install-purelib={0}".format(self.paths["purelib"]), - "--install-platlib={0}".format(self.paths["platlib"]), - "--install-scripts={0}".format(self.scripts_dir), - "--install-data={0}".format(self.paths["data"]), - ] - - def install(self, req, editable=False, sources=[]): - with self.activated(): - install_options = ["--prefix={0}".format(self.venv_dir),] - passa_pip = self.safe_import("passa.internals._pip") - ireq = req.as_ireq() - if editable: - with vistir.contextmanagers.cd(ireq.setup_py_dir, ireq.setup_py): - c = self.run( - install_options + self.get_setup_install_args( - req.name, develop=editable - ), cwd=ireq.setup_py_dir - ) - return c.returncode - distlib_scripts = self.safe_import("distlib.scripts") - sources = self.filter_sources(req, sources) - hashes = req.hashes - wheel = passa_pip.build_wheel(ireq, sources, hashes) - wheel.install(self.paths, distlib_scripts.ScriptMaker(None, None)) - - @contextlib.contextmanager - def activated(self): - original_path = sys.path - original_prefix = sys.prefix - original_user_base = os.environ.get("PYTHONUSERBASE", None) - original_venv = os.environ.get("VIRTUAL_ENV", None) - passa_path = vistir.compat.Path(__file__).absolute().parent.parent.as_posix() - with vistir.contextmanagers.temp_environ(), vistir.contextmanagers.temp_path(): - os.environ["PYTHONIOENCODING"] = vistir.compat.fs_str("utf-8") - os.environ["PYTHONDONTWRITEBYTECODE"] = vistir.compat.fs_str("1") - activate_this = os.path.join(self.scripts_dir, "activate_this.py") - with open(activate_this, "r") as f: - code = compile(f.read(), activate_this, "exec") - exec(code, dict(__file__=activate_this)) - os.environ["PYTHONUSERBASE"] = vistir.compat.fs_str(self.venv_dir.as_posix()) - os.environ["VIRTUAL_ENV"] = vistir.compat.fs_str(self.venv_dir.as_posix()) - sys.path = self.sys_path - sys.prefix = self.venv_dir - site = self.safe_import("site") - site.addsitedir(passa_path) - sys.modules["recursive_monkey_patch"] = self.recursive_monkey_patch - pkg_resources = self.safe_import("pkg_resources") - try: - yield - finally: - print("Deactivating virtualenv...") - del os.environ["VIRTUAL_ENV"] - del os.environ["PYTHONUSERBASE"] - if original_user_base: - os.environ["PYTHONUSERBASE"] = original_user_base - if original_venv: - os.environ["VIRTUAL_ENV"] = original_venv - sys.path = original_path - sys.prefix = original_prefix - six.moves.reload_module(pkg_resources) - - def run(self, cmd, cwd=os.curdir): - c = None - with self.activated(): - script = vistir.cmdparse.Script.parse(cmd) - c = vistir.misc.run(script._parts, return_object=True, nospin=True, cwd=cwd) - return c - - def run_py(self, cmd, cwd=os.curdir): - c = None - if isinstance(cmd, six.string_types): - script = vistir.cmdparse.Script.parse("{0} {1}".format(self.python, cmd)) - else: - script = vistir.cmdparse.Script.parse([self.python,] + list(cmd)) - with self.activated(): - c = vistir.misc.run(script._parts, return_object=True, nospin=True, cwd=cwd) - return c - - def is_installed(self, pkgname): - return any(d for d in self.get_distributions() if d.project_name == pkgname) - - def get_monkeypatched_pathset(self): - from pip_shims.shims import req_install - req_uninstall_name = "{0}.req_uninstall".format(req_install.__package__) - req_uninstall = self.safe_import(req_uninstall_name) - self.recursive_monkey_patch.monkey_patch( - PatchedUninstaller, req_uninstall.UninstallPathSet - ) - return req_uninstall.UninstallPathSet - - @contextlib.contextmanager - def uninstall(self, pkgname, *args, **kwargs): - auto_confirm = kwargs.pop("auto_confirm", True) - verbose = kwargs.pop("verbose", False) - with self.activated(): - pathset_base = self.get_monkeypatched_pathset() - dist = next( - iter(filter(lambda d: d.project_name == pkgname, self.get_working_set())), - None - ) - pathset = pathset_base.from_dist(dist) - if pathset is not None: - pathset.remove(auto_confirm=auto_confirm, verbose=verbose) - try: - yield pathset - except Exception as e: - if pathset is not None: - pathset.rollback() - else: - if pathset is not None: - pathset.commit() - if pathset is None: - return - - -SETUPTOOLS_SHIM = ( - "import setuptools, tokenize;__file__=%r;" - "f=getattr(tokenize, 'open', open)(__file__);" - "code=f.read().replace('\\r\\n', '\\n');" - "f.close();" - "exec(compile(code, __file__, 'exec'))" -) - - -class PatchedUninstaller(object): - def _permitted(self, path): - return True diff --git a/tests/conftest.py b/tests/conftest.py index 1aaf419..0ba621b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -54,8 +54,9 @@ def tmpvenv(virtualenv): @pytest.fixture(scope="function") def project(project_directory, tmpvenv): - venv_working_set = tmpvenv.initial_working_set - passa_dist = venv_working_set.by_key["passa"] - resolved = tmpvenv.resolve_dist(passa_dist, venv_working_set) + import pkg_resources + passa_dist = pkg_resources.get_distribution(pkg_resources.Requirement('passa')) + # passa_dist = tmpvenv.initial_working_set.by_key["passa"] + resolved = tmpvenv.resolve_dist(passa_dist, tmpvenv.base_working_set) with tmpvenv.activated(extra_dists=list(resolved)): yield _Project(project_directory, tmpvenv) From 28bf2f035b1e40bec3bf51d5050d1b5d5fd15338 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 30 Sep 2018 16:53:47 -0400 Subject: [PATCH 11/34] Update CI configs Signed-off-by: Dan Ryan --- .gitignore | 114 ++++++++++++++++++++++++++++++++--- .travis.yml | 11 ++-- appveyor.yml | 4 +- tests/actions/test_remove.py | 0 tests/actions/test_sync.py | 0 tests/conftest.py | 6 +- tox.ini | 7 ++- 7 files changed, 122 insertions(+), 20 deletions(-) delete mode 100644 tests/actions/test_remove.py delete mode 100644 tests/actions/test_sync.py diff --git a/.gitignore b/.gitignore index b858aca..741f1b4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,114 @@ -.env -.venv -__pycache__ - /build /dist /docs/_build /pack +htmlcov/ +.pytest_cache/ +.vscode/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so -*.egg-info +# Distribution / packaging +.Python +develop-eggs/ +downloads/ +eggs/ +.eggs/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST -*.py[co] +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ .pytest_cache/ -.vscode/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json diff --git a/.travis.yml b/.travis.yml index 8e38f96..3acd3a4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,8 +7,8 @@ matrix: fast_finish: true install: - - "python -m pip install --upgrade pip pytest-timeout" - - "python -m pip install --upgrade -e .[tests]" + - "python -m pip install --upgrade pip setuptools pytest-timeout" + - "python -m pip install --upgrade -e .[tests,virtualenv]" script: - "python -m pytest -v -n 8 tests/" @@ -24,7 +24,7 @@ jobs: - stage: packaging python: "3.6" install: - - "python -m pip install --upgrade pip" + - "python -m pip install --upgrade pip setuptools" - "python -m pip install --upgrade check-manifest readme-renderer" script: - "python setup.py check -m -r -s" @@ -38,8 +38,7 @@ jobs: - stage: coverage python: "3.6" install: - - "python -m pip install --upgrade pip" - - "python -m pip install --upgrade -e .[tests]" - - "python -m pip install --upgrade pytest-timeout pytest-xdist pytest-cov" + - "python -m pip install --upgrade pip setuptools pytest-timeout pytest-cov pytest-xdist" + - "python -m pip install --upgrade -e .[tests,virtualenv]" script: - "pytest -n auto --timeout 300 --cov=passa --cov-report=term-missing --cov-report=xml --cov-report=html tests" diff --git a/appveyor.yml b/appveyor.yml index 3216d26..a9ecca6 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -5,8 +5,8 @@ branches: install: - "SET PATH=C:\\Python36-x64;%PATH%" - "python --version" - - "python -m pip install --upgrade pip" - - "python -m pip install --upgrade -e .[pack,tests]" + - "python -m pip install --upgrade pip setuptools pytest-timeout pytest-xdist" + - "python -m pip install --upgrade -e .[pack,tests,virtualenv]" build_script: - "python -m invoke pack" diff --git a/tests/actions/test_remove.py b/tests/actions/test_remove.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/actions/test_sync.py b/tests/actions/test_sync.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/conftest.py b/tests/conftest.py index 0ba621b..ec02ca9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -53,10 +53,10 @@ def tmpvenv(virtualenv): @pytest.fixture(scope="function") -def project(project_directory, tmpvenv): +def project(project_directory, tmpvenv, tmpdir): import pkg_resources passa_dist = pkg_resources.get_distribution(pkg_resources.Requirement('passa')) - # passa_dist = tmpvenv.initial_working_set.by_key["passa"] resolved = tmpvenv.resolve_dist(passa_dist, tmpvenv.base_working_set) - with tmpvenv.activated(extra_dists=list(resolved)): + with vistir.contextmanagers.temp_environ(), tmpvenv.activated(extra_dists=list(resolved)): + os.environ["PACKAGEBUILDER_CACHE_DIR"] = tmpdir.strpath yield _Project(project_directory, tmpvenv) diff --git a/tox.ini b/tox.ini index d436e33..6428cef 100644 --- a/tox.ini +++ b/tox.ini @@ -8,9 +8,13 @@ setenv = LC_ALL = en_US.UTF-8 deps = coverage + setuptools + pytest + pytest-timeout + pytest-sugar -e .[tests,virtualenv] commands = coverage run --parallel -m pytest --timeout 300 [] -install_command = python -m pip install {opts} {packages} +install_command = python -m pip install --upgrade {opts} {packages} usedevelop = True [testenv:coverage-report] @@ -23,7 +27,6 @@ commands = [testenv:docs] deps = -r{toxinidir}/docs/requirements.txt - -e .[tests,virtualenv] commands = sphinx-build -d {envtmpdir}/doctrees -b html docs docs/build/html sphinx-build -d {envtmpdir}/doctrees -b man docs docs/build/man From 57bac3648ddec6d6566415d359339bd8a3cf9838 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 30 Sep 2018 16:54:16 -0400 Subject: [PATCH 12/34] Add deprecation warning ignores to test runners Signed-off-by: Dan Ryan --- setup.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.cfg b/setup.cfg index ace8c27..194cc04 100644 --- a/setup.cfg +++ b/setup.cfg @@ -102,6 +102,9 @@ strict = true addopts = -ra testpaths = tests/ norecursedirs = .* build dist news tasks docs +filterwarnings = + ignore::DeprecationWarning + ignore::PendingDeprecationWarning [build-system] requires = ["setuptools", "wheel"] From 06f1af8aba8a9807a4c4884ec33554d1f6673088 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 30 Sep 2018 16:55:49 -0400 Subject: [PATCH 13/34] Add project fixtures, venv support Signed-off-by: Dan Ryan --- src/passa/actions/add.py | 2 +- src/passa/actions/install.py | 1 + src/passa/actions/lock.py | 3 +- src/passa/actions/remove.py | 1 + src/passa/cli/options.py | 2 +- src/passa/internals/_pip.py | 163 +++++++++++++++++++++++----- src/passa/internals/_pip_shims.py | 9 ++ src/passa/internals/dependencies.py | 11 +- src/passa/models/synchronizers.py | 12 +- tests/conftest.py | 77 +++++++++---- 10 files changed, 219 insertions(+), 62 deletions(-) diff --git a/src/passa/actions/add.py b/src/passa/actions/add.py index 3efc4dc..49e4bce 100644 --- a/src/passa/actions/add.py +++ b/src/passa/actions/add.py @@ -48,7 +48,7 @@ def add_packages(packages=[], editables=[], project=None, dev=False, sync=False, syncer = Synchronizer( project, default=default, develop=develop, - clean_unneeded=clean + clean_unneeded=clean, venv=getattr(project, "venv", None) ) success = sync(syncer) if not success: diff --git a/src/passa/actions/install.py b/src/passa/actions/install.py index 1728dae..9872584 100644 --- a/src/passa/actions/install.py +++ b/src/passa/actions/install.py @@ -30,3 +30,4 @@ def install(project=None, check=True, dev=False, clean=True): return 1 print("Synchronized project at", project.root) + return 0 diff --git a/src/passa/actions/lock.py b/src/passa/actions/lock.py index 7c09469..f661c82 100644 --- a/src/passa/actions/lock.py +++ b/src/passa/actions/lock.py @@ -11,7 +11,8 @@ def lock(project=None): locker = BasicLocker(project) success = lock(locker) if not success: - return + return 1 project._l.write() print("Written to project at", project.root) + return 0 diff --git a/src/passa/actions/remove.py b/src/passa/actions/remove.py index 17ba1c7..92f6168 100644 --- a/src/passa/actions/remove.py +++ b/src/passa/actions/remove.py @@ -36,3 +36,4 @@ def remove(project=None, only="default", packages=[], clean=True, sync=False): return 1 print("Cleaned project at", project.root) + return 0 diff --git a/src/passa/cli/options.py b/src/passa/cli/options.py index 3abcadb..eafa71d 100644 --- a/src/passa/cli/options.py +++ b/src/passa/cli/options.py @@ -25,7 +25,7 @@ def __init__(self, root, *args, **kwargs): raise argparse.ArgumentError( project, "{0!r} is not a Pipfile project".format(root.as_posix()), ) - self.venv = self.get_venv(root) + self.venv = kwargs.pop("venv", self.get_venv(root)) try: super(Project, self).__init__(root.as_posix(), env_prefix=self.venv.venv_dir, *args, **kwargs) diff --git a/src/passa/internals/_pip.py b/src/passa/internals/_pip.py index 2aa143a..da52fc6 100644 --- a/src/passa/internals/_pip.py +++ b/src/passa/internals/_pip.py @@ -9,16 +9,21 @@ import os import distlib.database +import distlib.metadata import distlib.scripts import distlib.wheel import packaging.utils import pip_shims import setuptools.dist import six +import sys +import sysconfig import vistir from ..models.caches import CACHE_DIR -from ._pip_shims import VCS_SUPPORT, build_wheel as _build_wheel, unpack_url +from ._pip_shims import ( + SETUPTOOLS_SHIM, VCS_SUPPORT, build_wheel as _build_wheel, unpack_url +) from .utils import filter_sources @@ -282,6 +287,7 @@ class NoopInstaller(object): arguments, and should be called in that order to prepare an installation operation, and to actually install things. """ + def prepare(self): pass @@ -289,39 +295,121 @@ def install(self): pass -class EditableInstaller(NoopInstaller): - """Installer to handle editable. - """ - def __init__(self, requirement): - ireq = requirement.as_ireq() - self.working_directory = ireq.setup_py_dir - self.setup_py = ireq.setup_py - - def install(self): - with vistir.cd(self.working_directory), _suppress_distutils_logs(): - # Access from Setuptools to ensure things are patched correctly. - setuptools.dist.distutils.core.run_setup( - self.setup_py, ["develop", "--no-deps"], - ) - - -class WheelInstaller(NoopInstaller): - """Installer by building a wheel. +class VenvInstaller(NoopInstaller): + """Virtualenv-capable installer""" - The wheel is built during `prepare()`, and installed in `install()`. - """ - def __init__(self, requirement, sources, paths): + def __init__(self, requirement, sources=None, paths=None, venv=None): self.ireq = requirement.as_ireq() self.sources = filter_sources(requirement, sources) self.hashes = requirement.hashes or None self.paths = paths - self.wheel = None + self.venv = venv + self.built = None + self.python = sys.executable if not self.venv else self.venv.python + self.py_version = self.venv.python_version if self.venv else sysconfig.get_python_version() + self.metadata = None + self.is_wheel = False + + @property + def src_dir(self): + build_dir = os.environ.get("PASSA_BUILD_DIR", None) + if not build_dir: + build_dir = vistir.path.create_tracked_tempdir("passa-build-dir") + return build_dir + + @property + def setup_dir(self): + if not self.built: + return self.ireq.setup_py_dir + return vistir.compat.Path(self.built.path).parent + + @property + def installation_args(self): + install_arg = "install" if not self.ireq.editable else "develop" + setup_path = self.setup_dir.joinpath("setup.py") + install_keys = ["headers", "purelib", "platlib", "scripts", "data"] + install_args = [ + self.python, "-u", "-c", SETUPTOOLS_SHIM % setup_path.as_posix(), install_arg, + "--single-version-externally-managed", "--no-deps", + "--prefix={0}".format(self.paths["prefix"]) + ] + for key in install_keys: + install_args.append("--install-{0}={1}".format(key, self.paths[key])) + return install_args + + def build_wheel(self): + self.built = build_wheel(self.ireq, self.sources, self.hashes) + self.metadata = read_wheel_metadata(self.built) + self.is_wheel = True + + def build_sdist(self): + finder = _get_finder(self.sources) + self.ireq.populate_link(finder, True, False) + self.ireq.ensure_has_source_dir(self.src_dir) + self.built = get_sdist(self.ireq) + self.metadata = read_sdist_metadata(self.built) + + def install_wheel(self): + scripts = distlib.scripts.ScriptMaker(None, None) + self.built.install(self.paths, scripts) + + def install_sdist(self): + with vistir.cd(self.setup_dir.as_posix()), _suppress_distutils_logs(): + c = vistir.misc.run(self.installation_args, return_object=True, block=True, + nospin=True) + if c.returncode != 0: + err_text = "{0!r}: {1!r}".format(c.err, c.out) + raise RuntimeError("Failed to install package: {0!r}".format(err_text)) + return def prepare(self): - self.wheel = build_wheel(self.ireq, self.sources, self.hashes) + pass def install(self): - self.wheel.install(self.paths, distlib.scripts.ScriptMaker(None, None)) + if self.venv: + with self.venv.activated(): + self._install() + else: + self._install() + + +class SdistInstaller(VenvInstaller): + """Installer for SDists""" + def __init__(self, *args, **kwargs): + super(SdistInstaller, self).__init__(*args, **kwargs) + + def prepare(self): + try: + self.build_wheel() + except WheelBuildError: + self.build_sdist() + if not self.built or not self.metadata: + raise + + def _install(self): + if self.is_wheel: + self.install_wheel() + else: + self.install_sdist() + + +class Installer(SdistInstaller): + """Installer to handle editable. + """ + def __init__(self, *args, **kwargs): + super(Installer, self).__init__(*args, **kwargs) + + @property + def src_dir(self): + build_dir = os.environ.get("PIP_SRC", None) + venv = os.environ.get("VIRTUAL_ENV", None) + if venv: + src_dir = os.path.join(venv, "src") + if os.path.exists(src_dir): + build_dir = src_dir + if not build_dir: + build_dir = vistir.path.create_tracked_tempdir("passa-build-dir") + return build_dir def _iter_egg_info_directories(root, name): @@ -389,9 +477,28 @@ def _find_egg_info(ireq): return top_egg_info -def read_sdist_metadata(ireq): +def get_sdist(ireq): egg_info_dir = _find_egg_info(ireq) if not egg_info_dir: return None - distribution = distlib.database.EggInfoDistribution(egg_info_dir) - return distribution.metadata + return distlib.database.EggInfoDistribution(egg_info_dir) + + +def read_sdist_metadata(sdist): + if not getattr(sdist, "metadata", None): + sdist = get_sdist(sdist) + if not sdist: + return None + return sdist.metadata + + +def read_wheel_metadata(wheel): + metadata = None + try: + metadata = wheel.metadata + except distlib.metadata.MetadataConflictError: + import zipfile + metadata = wheel.get_wheel_metadata( + zipfile.ZipFile(os.path.join(wheel.dirname, wheel.filename)) + ) + return metadata diff --git a/src/passa/internals/_pip_shims.py b/src/passa/internals/_pip_shims.py index b2c7b6e..243675d 100644 --- a/src/passa/internals/_pip_shims.py +++ b/src/passa/internals/_pip_shims.py @@ -59,3 +59,12 @@ def _unpack_url_pre10(*args, **kwargs): if PIP_VERSION < VERSION_10: build_wheel = _build_wheel_pre10 unpack_url = _unpack_url_pre10 + + +SETUPTOOLS_SHIM = ( + "import setuptools, tokenize;__file__=%r;" + "f=getattr(tokenize, 'open', open)(__file__);" + "code=f.read().replace('\\r\\n', '\\n');" + "f.close();" + "exec(compile(code, __file__, 'exec'))" +) diff --git a/src/passa/internals/dependencies.py b/src/passa/internals/dependencies.py index 410a5e6..b163b85 100644 --- a/src/passa/internals/dependencies.py +++ b/src/passa/internals/dependencies.py @@ -14,7 +14,9 @@ import six from ..models.caches import DependencyCache, RequiresPythonCache -from ._pip import WheelBuildError, build_wheel, read_sdist_metadata +from ._pip import ( + WheelBuildError, build_wheel, get_sdist, read_wheel_metadata, read_sdist_metadata +) from .markers import contains_extra, get_contained_extras, get_without_extra from .utils import get_pinned_version, is_pinned @@ -225,17 +227,18 @@ def _get_dependencies_from_pip(ireq, sources): """ extras = ireq.extras or () try: - wheel = build_wheel(ireq, sources) + built = build_wheel(ireq, sources) except WheelBuildError: # XXX: This depends on a side effect of `build_wheel`. This block is # reached when it fails to build an sdist, where the sdist would have # been downloaded, extracted into `ireq.source_dir`, and partially # built (hopefully containing .egg-info). - metadata = read_sdist_metadata(ireq) + built = get_sdist(ireq) + metadata = read_sdist_metadata(built) if not metadata: raise else: - metadata = wheel.metadata + metadata = read_wheel_metadata(built) requirements = _read_requirements(metadata, extras) requires_python = _read_requires_python(metadata) return requirements, requires_python diff --git a/src/passa/models/synchronizers.py b/src/passa/models/synchronizers.py index 78b3bf7..2ade9c5 100644 --- a/src/passa/models/synchronizers.py +++ b/src/passa/models/synchronizers.py @@ -8,13 +8,14 @@ import sys import sysconfig +import distlib.wheel import pkg_resources import packaging.markers import packaging.version import requirementslib -from ..internals._pip import uninstall, EditableInstaller, WheelInstaller +from ..internals._pip import uninstall, Installer def _is_installation_local(name, venv=None): @@ -191,10 +192,7 @@ def _sync(self): if markers and not packaging.markers.Marker(markers).evaluate(): continue r.markers = None - if r.editable: - installer = EditableInstaller(r, venv=self.venv) - else: - installer = WheelInstaller(r, self.sources, self.paths) + installer = Installer(r, sources=self.sources, paths=self.paths, venv=self.venv) try: installer.prepare() except Exception as e: @@ -242,13 +240,11 @@ def __repr__(self): return "<{0} @ {1!r}>".format(type(self).__name__, self._root) def print(self, packages): - message = "" if not self.sync: message = "Would clean: {0}" else: message = "Cleaned: {0}" - packages = ", ".join(sorted(set(packages))) if packages else "" - print(message.format(packages)) + print(message.format(", ".join(sorted(set(packages))))) def clean(self): groupcoll = _group_installed_names(self.packages, venv=self.project.venv) diff --git a/tests/conftest.py b/tests/conftest.py index ec02ca9..2726762 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,11 +2,16 @@ import os import pytest import passa +import passa.models.projects import passa.cli.options -import mork.virtualenv +import mork +import pkg_resources +import plette import sys import vistir +from collections import deque + DEFAULT_PIPFILE_CONTENTS = """ [[source]] @@ -20,12 +25,18 @@ """.strip() -@pytest.fixture(scope="function") -def project_directory(tmpdir_factory): - project_dir = tmpdir_factory.mktemp("passa-project") - project_dir.join("Pipfile").write(DEFAULT_PIPFILE_CONTENTS) - with vistir.contextmanagers.cd(project_dir.strpath): - yield project_dir +@pytest.fixture(scope="session") +def working_set_extension(): + dists = set() + passa_dist = pkg_resources.get_distribution(pkg_resources.Requirement('passa')) + dists.add(passa_dist) + requirements = deque(passa_dist.requires(extras=('tests', 'virtualenv'))) + while requirements: + req = requirements.popleft() + dist = pkg_resources.working_set.find(req) + dists.add(dist) + requirements.extend(dist.requires()) + return dists @pytest.fixture(scope="function") @@ -41,22 +52,50 @@ def virtualenv(tmpdir_factory): class _Project(passa.cli.options.Project): - def __init__(self, root, venv=None): - self.path = root.strpath + def __init__(self, root, venv=None, working_set_extension=[]): + self.path = root self.venv = venv - super(_Project, self).__init__(self.path) + self.working_set_extension = working_set_extension + super(_Project, self).__init__(self.path, venv=venv) + self.pipfile_instance = vistir.compat.Path(self.pipfile_location) + self.lockfile_instance = vistir.compat.Path(self.lockfile_location) + + def reload(self): + self._p = passa.models.projects.ProjectFile.read( + os.path.join(self.path, "Pipfile"), + plette.Pipfile, + ) + self._l = passa.models.projects.ProjectFile.read( + os.path.join(self.path, "Pipfile.lock"), + plette.Lockfile, + invalid_ok=True, + ) + +@pytest.fixture(scope="function") +def project_directory(tmpdir_factory): + project_dir = tmpdir_factory.mktemp("passa-project") + project_dir.join("Pipfile").write(DEFAULT_PIPFILE_CONTENTS) + with vistir.contextmanagers.cd(project_dir.strpath): + yield project_dir @pytest.fixture -def tmpvenv(virtualenv): - return mork.virtualenv.VirtualEnv(virtualenv.strpath) +def tmpvenv(virtualenv, tmpdir): + venv_srcdir = virtualenv.join("src").mkdir() + venv = mork.virtualenv.VirtualEnv(virtualenv.strpath) + venv.run(["pip", "install", "--upgrade", "mork", "setuptools"]) + with vistir.contextmanagers.temp_environ(): + os.environ["PACKAGEBUILDER_CACHE_DIR"] = tmpdir.strpath + os.environ["PIP_QUIET"] = "1" + os.environ["PIP_SRC"] = venv_srcdir.strpath + venv.is_installed = lambda x: any(d for d in venv.get_distributions() if d.project_name == x) + yield venv @pytest.fixture(scope="function") -def project(project_directory, tmpvenv, tmpdir): - import pkg_resources - passa_dist = pkg_resources.get_distribution(pkg_resources.Requirement('passa')) - resolved = tmpvenv.resolve_dist(passa_dist, tmpvenv.base_working_set) - with vistir.contextmanagers.temp_environ(), tmpvenv.activated(extra_dists=list(resolved)): - os.environ["PACKAGEBUILDER_CACHE_DIR"] = tmpdir.strpath - yield _Project(project_directory, tmpvenv) +def project(project_directory, working_set_extension, tmpvenv): + # resolved = tmpvenv.resolve_dist(passa_dist, tmpvenv.base_working_set) + with tmpvenv.activated(extra_dists=list(working_set_extension)): + project = _Project(project_directory.strpath, venv=tmpvenv, working_set_extension=working_set_extension) + project.is_installed = lambda x: any(d for d in tmpvenv.get_working_set() if d.project_name == x) + yield project From 7ed8fbe25c8731f66abcac66e7ddac685c72e35b Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 30 Sep 2018 16:56:09 -0400 Subject: [PATCH 14/34] Add clean, install and lock tests Signed-off-by: Dan Ryan --- tests/actions/test_clean.py | 4 +- tests/actions/test_install.py | 96 +++++++++++++++++++++++++++++++++++ tests/actions/test_lock.py | 53 +++++++++++++++++++ 3 files changed, 152 insertions(+), 1 deletion(-) diff --git a/tests/actions/test_clean.py b/tests/actions/test_clean.py index c56d06d..2160301 100644 --- a/tests/actions/test_clean.py +++ b/tests/actions/test_clean.py @@ -3,16 +3,18 @@ import passa.actions.clean -def test_clean_subset(project): +def test_clean(project): retcode = passa.actions.add.add_packages(["requests"], project=project) assert not retcode packages = ["requests", "chardet", "certifi", "idna"] c = project.venv.run("pip install pytz") assert c.returncode == 0 + assert project.venv.is_installed("pytz") c = project.venv.run("python -c 'import pytz'") assert c.returncode == 0 clean_retcode = passa.actions.clean.clean(project=project) assert not clean_retcode + assert not project.venv.is_installed("pytz") c = project.venv.run("python -c 'import pytz'") assert c.returncode != 0 assert all(pkg in project.lockfile.default for pkg in packages) diff --git a/tests/actions/test_install.py b/tests/actions/test_install.py index e69de29..bb205e6 100644 --- a/tests/actions/test_install.py +++ b/tests/actions/test_install.py @@ -0,0 +1,96 @@ +# -*- coding=utf-8 -*- + +import passa.actions.install +import passa.actions.add +import passa.cli.options +import passa.models.projects +import pytest + + +@pytest.mark.parametrize( + 'is_dev', (True, False) +) +def test_install_one(project, is_dev): + add_kwargs = { + "project": project, + "packages": ["pytz",], + "editables": [], + "dev": is_dev, + "sync": False, + "clean": False + } + retcode = passa.actions.add.add_packages(**add_kwargs) + assert not retcode + lockfile_section = "default" if not is_dev else "develop" + assert 'pytz' in project.lockfile._data[lockfile_section].keys() + install = passa.actions.install.install(project=project, check=True, dev=is_dev, clean=False) + assert install == 0 + assert project.venv.is_installed("pytz") or project.is_installed("pytz") + + +@pytest.mark.parametrize( + 'is_dev', (True, False) +) +def test_install_one_with_deps(project, is_dev): + add_kwargs = { + "project": project, + "packages": ["requests",], + "editables": [], + "dev": is_dev, + "sync": False, + "clean": False + } + retcode = passa.actions.add.add_packages(**add_kwargs) + assert not retcode + lockfile_section = "default" if not is_dev else "develop" + assert 'requests' in project.lockfile._data[lockfile_section].keys() + assert 'idna' in project.lockfile._data[lockfile_section].keys() + install = passa.actions.install.install(project=project, check=True, dev=is_dev, clean=False) + assert install == 0 + assert project.venv.is_installed("requests") or project.is_installed("requests") + assert project.venv.is_installed("idna") or project.is_installed("idna") + + +@pytest.mark.parametrize( + 'is_dev', (True, False) +) +def test_install_editable(project, is_dev): + add_kwargs = { + "project": project, + "packages": [], + "editables": ["git+https://github.com/sarugaku/shellingham.git@1.2.1#egg=shellingham",], + "dev": is_dev, + "sync": False, + "clean": False + } + retcode = passa.actions.add.add_packages(**add_kwargs) + assert not retcode + lockfile_section = "default" if not is_dev else "develop" + assert 'shellingham' in project.lockfile._data[lockfile_section].keys() + install = passa.actions.install.install(project=project, check=True, dev=is_dev, clean=False) + assert install == 0 + project.reload() + assert (project.venv.is_installed("shellingham") or + project.is_installed("shellingham")), list([dist.project_name for dist in project.venv.get_distributions()]) + + +@pytest.mark.parametrize( + 'is_dev', (True, False) +) +def test_install_sdist(project, is_dev): + add_kwargs = { + "project": project, + "packages": ["arrow",], + "editables": [], + "dev": is_dev, + "sync": False, + "clean": False + } + retcode = passa.actions.add.add_packages(**add_kwargs) + assert not retcode + lockfile_section = "default" if not is_dev else "develop" + assert 'arrow' in project.lockfile._data[lockfile_section].keys() + install = passa.actions.install.install(project=project, check=True, dev=is_dev, clean=False) + assert install == 0 + project.reload() + assert project.venv.is_installed("arrow") or project.is_installed("arrow") diff --git a/tests/actions/test_lock.py b/tests/actions/test_lock.py index e69de29..962493e 100644 --- a/tests/actions/test_lock.py +++ b/tests/actions/test_lock.py @@ -0,0 +1,53 @@ +# -*- coding=utf-8 -*- + +import passa.actions.lock +import passa.actions.install +import passa.cli.options +import passa.models.projects +import pytest + + +@pytest.mark.parametrize( + 'is_dev', (True, False) +) +def test_lock_one(project, is_dev): + line = "pytz" + project.add_line_to_pipfile(line, develop=is_dev) + retcode = passa.actions.lock.lock(project=project) + project.reload() + assert retcode == 0 + lockfile_section = "default" if not is_dev else "develop" + assert 'pytz' in project.lockfile._data[lockfile_section].keys() + install = passa.actions.install.install(project=project, check=True, dev=is_dev, clean=False) + assert install == 0 + + +@pytest.mark.parametrize( + 'is_dev', (True, False) +) +def test_lock_one_with_deps(project, is_dev): + line = "requests" + project.add_line_to_pipfile(line, develop=is_dev) + retcode = passa.actions.lock.lock(project=project) + project.reload() + assert retcode == 0 + lockfile_section = "default" if not is_dev else "develop" + assert 'requests' in project.lockfile._data[lockfile_section].keys() + assert 'idna' in project.lockfile._data[lockfile_section].keys() + install = passa.actions.install.install(project=project, check=True, dev=is_dev, clean=False) + assert install == 0 + + +@pytest.mark.parametrize( + 'is_dev', (True, False) +) +def test_lock_editable(project, is_dev): + line = "-e git+https://github.com/sarugaku/shellingham.git@1.2.1#egg=shellingham" + project.add_line_to_pipfile(line, develop=is_dev) + retcode = passa.actions.lock.lock(project=project) + project.reload() + assert retcode == 0 + lockfile_section = "default" if not is_dev else "develop" + assert 'shellingham' in project.lockfile._data[lockfile_section].keys(), project.lockfile._data + install = passa.actions.install.install(project=project, check=True, dev=is_dev, clean=False) + assert install == 0 From c5a516aa174286c43a87a2ef904703a15a71b794 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 30 Sep 2018 16:56:55 -0400 Subject: [PATCH 15/34] Add sync and remove tests Signed-off-by: Dan Ryan --- tests/actions/test_remove_and_sync.py | 142 ++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 tests/actions/test_remove_and_sync.py diff --git a/tests/actions/test_remove_and_sync.py b/tests/actions/test_remove_and_sync.py new file mode 100644 index 0000000..e14ec8a --- /dev/null +++ b/tests/actions/test_remove_and_sync.py @@ -0,0 +1,142 @@ +# -*- coding=utf-8 -*- + +import passa.actions.add +import passa.actions.remove +import passa.cli.options +import passa.models.projects +import pytest +import vistir + + +@pytest.mark.parametrize( + 'sync', (True, False) +) +@pytest.mark.parametrize( + 'is_dev', (True, False) +) +def test_remove_one(project, sync, is_dev): + pkg = "xlrd" + add_kwargs = { + "project": project, + "packages": [pkg,], + "editables": [], + "dev": is_dev, + "sync": sync, + "clean": False + } + retcode = passa.actions.add.add_packages(**add_kwargs) + assert not retcode + lockfile_section = "default" if not is_dev else "develop" + assert pkg in project.lockfile._data[lockfile_section].keys() + if sync: + assert project.venv.is_installed(pkg) or project.is_installed(pkg) + remove = "default" if not is_dev else "dev" + retcode = passa.actions.remove.remove(project=project, packages=[pkg,], sync=sync, only=remove) + assert not retcode + project.reload() + assert pkg not in project.lockfile._data[lockfile_section].keys() + if sync: + assert not project.venv.is_installed(pkg) + + +@pytest.mark.parametrize( + 'sync', (True, False), +) +@pytest.mark.parametrize( + 'is_dev', (True, False) +) +def test_remove_one_with_deps(project, sync, is_dev): + add_kwargs = { + "project": project, + "packages": ["requests",], + "editables": [], + "dev": is_dev, + "sync": sync, + "clean": False + } + retcode = passa.actions.add.add_packages(**add_kwargs) + assert not retcode + lockfile_section = "default" if not is_dev else "develop" + assert 'requests' in project.lockfile._data[lockfile_section].keys() + assert 'idna' in project.lockfile._data[lockfile_section].keys() + if sync: + c = vistir.misc.run(["{0}".format(project.venv.python), "-c", "import requests"], + nospin=True, block=True, return_object=True) + assert c.returncode == 0, (c.out, c.err) + assert project.venv.is_installed("requests") or project.is_installed("requests") + assert project.venv.is_installed("idna") or project.is_installed("idna") + remove = "default" if not is_dev else "dev" + retcode = passa.actions.remove.remove(project=project, packages=["requests",], sync=sync, only=remove) + assert not retcode + project.reload() + assert "requests" not in project.lockfile._data[lockfile_section].keys() + assert "idna" not in project.lockfile._data[lockfile_section].keys() + if sync: + assert not project.venv.is_installed("requests") + assert not project.venv.is_installed("idna") + + +@pytest.mark.parametrize( + 'sync', (True, False), +) +@pytest.mark.parametrize( + 'is_dev', (True, False) +) +def test_remove_editable(project, sync, is_dev): + add_kwargs = { + "project": project, + "packages": [], + "editables": ["git+https://github.com/sarugaku/shellingham.git@1.2.1#egg=shellingham",], + "dev": is_dev, + "sync": sync, + "clean": False + } + retcode = passa.actions.add.add_packages(**add_kwargs) + assert not retcode + lockfile_section = "default" if not is_dev else "develop" + assert 'shellingham' in project.lockfile._data[lockfile_section].keys() + if sync: + c = vistir.misc.run(["{0}".format(project.venv.python), "-c", "import shellingham"], + nospin=True, block=True, return_object=True) + assert c.returncode == 0, (c.out, c.err) + assert project.venv.is_installed("shellingham") or project.is_installed("shellingham") + remove = "default" if not is_dev else "dev" + retcode = passa.actions.remove.remove(project=project, packages=["shellingham",], sync=sync, only=remove) + assert not retcode + project.reload() + assert "shellingham" not in project.lockfile._data[lockfile_section].keys() + if sync: + assert not project.venv.is_installed("shellingham") + + +@pytest.mark.parametrize( + 'sync', (True, False), +) +@pytest.mark.parametrize( + 'is_dev', (True, False) +) +def test_remove_sdist(project, is_dev, sync): + add_kwargs = { + "project": project, + "packages": ["arrow"], + "editables": [], + "dev": is_dev, + "sync": sync, + "clean": False + } + retcode = passa.actions.add.add_packages(**add_kwargs) + assert not retcode + lockfile_section = "default" if not is_dev else "develop" + assert 'arrow' in project.lockfile._data[lockfile_section].keys() + if sync: + c = vistir.misc.run(["{0}".format(project.venv.python), "-c", "import arrow"], + nospin=True, block=True, return_object=True) + assert c.returncode == 0, (c.out, c.err) + assert project.venv.is_installed("arrow") or project.is_installed("arrow") + remove = "default" if not is_dev else "dev" + retcode = passa.actions.remove.remove(project=project, packages=["arrow",], sync=sync, only=remove) + assert not retcode + project.reload() + assert "arrow" not in project.lockfile._data[lockfile_section].keys() + if sync: + assert not project.venv.is_installed("arrow") From d6e2aa40559084d733714efd7096eaeee361b107 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 1 Oct 2018 01:15:38 -0400 Subject: [PATCH 16/34] Monkeypatch distlib metadata to work with version 2.1 Signed-off-by: Dan Ryan --- src/passa/internals/_pip.py | 26 ++++++-------------------- src/passa/internals/_pip_shims.py | 14 ++++++++++++++ src/passa/internals/dependencies.py | 6 +++--- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/passa/internals/_pip.py b/src/passa/internals/_pip.py index da52fc6..8146df3 100644 --- a/src/passa/internals/_pip.py +++ b/src/passa/internals/_pip.py @@ -14,7 +14,6 @@ import distlib.wheel import packaging.utils import pip_shims -import setuptools.dist import six import sys import sysconfig @@ -339,15 +338,15 @@ def installation_args(self): def build_wheel(self): self.built = build_wheel(self.ireq, self.sources, self.hashes) - self.metadata = read_wheel_metadata(self.built) + self.metadata = self.built.metadata self.is_wheel = True def build_sdist(self): finder = _get_finder(self.sources) - self.ireq.populate_link(finder, True, False) + self.ireq.populate_link(finder, False, False) self.ireq.ensure_has_source_dir(self.src_dir) self.built = get_sdist(self.ireq) - self.metadata = read_sdist_metadata(self.built) + self.metadata = read_sdist_metadata(self.ireq) def install_wheel(self): scripts = distlib.scripts.ScriptMaker(None, None) @@ -381,7 +380,7 @@ def __init__(self, *args, **kwargs): def prepare(self): try: self.build_wheel() - except WheelBuildError: + except (WheelBuildError, distlib.metadata.MetadataConflictError): self.build_sdist() if not self.built or not self.metadata: raise @@ -484,21 +483,8 @@ def get_sdist(ireq): return distlib.database.EggInfoDistribution(egg_info_dir) -def read_sdist_metadata(sdist): - if not getattr(sdist, "metadata", None): - sdist = get_sdist(sdist) +def read_sdist_metadata(ireq): + sdist = get_sdist(ireq) if not sdist: return None return sdist.metadata - - -def read_wheel_metadata(wheel): - metadata = None - try: - metadata = wheel.metadata - except distlib.metadata.MetadataConflictError: - import zipfile - metadata = wheel.get_wheel_metadata( - zipfile.ZipFile(os.path.join(wheel.dirname, wheel.filename)) - ) - return metadata diff --git a/src/passa/internals/_pip_shims.py b/src/passa/internals/_pip_shims.py index 243675d..112bbc6 100644 --- a/src/passa/internals/_pip_shims.py +++ b/src/passa/internals/_pip_shims.py @@ -11,7 +11,18 @@ from __future__ import absolute_import, unicode_literals +import distlib.metadata import pip_shims +import recursive_monkey_patch + + +class LegacyMetadata(): + def set_metadata_version(self): + metadata_version = self._fields.get("Metadata-Version") + if metadata_version == "2.1": + self._fields["Metadata-Version"] = metadata_version + else: + self._fields['Metadata-Version'] = distlib.metadata._best_version(self._fields) def _build_wheel_pre10(ireq, output_dir, finder, wheel_cache, kwargs): @@ -68,3 +79,6 @@ def _unpack_url_pre10(*args, **kwargs): "f.close();" "exec(compile(code, __file__, 'exec'))" ) + + +recursive_monkey_patch.monkey_patch(LegacyMetadata, distlib.metadata.LegacyMetadata) diff --git a/src/passa/internals/dependencies.py b/src/passa/internals/dependencies.py index b163b85..6ad5629 100644 --- a/src/passa/internals/dependencies.py +++ b/src/passa/internals/dependencies.py @@ -15,7 +15,7 @@ from ..models.caches import DependencyCache, RequiresPythonCache from ._pip import ( - WheelBuildError, build_wheel, get_sdist, read_wheel_metadata, read_sdist_metadata + WheelBuildError, build_wheel, get_sdist, read_sdist_metadata ) from .markers import contains_extra, get_contained_extras, get_without_extra from .utils import get_pinned_version, is_pinned @@ -234,11 +234,11 @@ def _get_dependencies_from_pip(ireq, sources): # been downloaded, extracted into `ireq.source_dir`, and partially # built (hopefully containing .egg-info). built = get_sdist(ireq) - metadata = read_sdist_metadata(built) + metadata = read_sdist_metadata(ireq) if not metadata: raise else: - metadata = read_wheel_metadata(built) + metadata = built.metadata requirements = _read_requirements(metadata, extras) requires_python = _read_requires_python(metadata) return requirements, requires_python From 246eccdbab5014d36581cabf873a471495de7a69 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 1 Oct 2018 17:08:33 -0400 Subject: [PATCH 17/34] Update lockfile Signed-off-by: Dan Ryan --- Pipfile.lock | 297 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 228 insertions(+), 69 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index c83e057..473a2d4 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -19,7 +19,7 @@ "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92", "sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '3.6'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '3.6'", "version": "==1.4.3" }, "attrs": { @@ -27,7 +27,7 @@ "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '3.6'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '3.6'", "version": "==18.2.0" }, "backports-shutil-get-terminal-size": { @@ -69,10 +69,10 @@ }, "distlib": { "hashes": [ - "sha256:cd502c66fc27c535bab62dc4f482e403e2369c2c05281a79cc2d4e2f42a87f20" + "sha256:57977cd7d9ea27986ec62f425630e4ddb42efe651ff80bc58ed8dbc3c7c21f19" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.2.7" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.2.8" }, "enum34": { "hashes": [ @@ -89,7 +89,7 @@ "sha256:3bb3de3582cb27071cfb514f00ed784dc444b7f96dc21e140de65fe00585c95e", "sha256:41d5b64e70507d0c3ca742d68010a76060eea8a3d863e9b5130ab11a4a91aa0e" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.0.1" }, "idna": { @@ -107,6 +107,14 @@ "markers": "python_version < '2.7' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.0.4" }, + "installer": { + "hashes": [ + "sha256:1ba23de573e9b95a8dcbd04fd026c40a64b77db0aadc48f28a844b4cb87479fe", + "sha256:f4f195c9b17ea7d2b631a758451485c6b080975349b4adebe45ef4bb022db069" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.1.1" + }, "modutil": { "hashes": [ "sha256:2c85c1666649e92e56de17c00e1e831313602d9b55e8661d39c01e39003b45f7", @@ -115,13 +123,21 @@ "markers": "python_version >= '2.6' and python_version >= '3.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.0.0" }, + "packagebuilder": { + "hashes": [ + "sha256:1e85c4e0e994322996b93cd6685c12834d30f3558889154f8e3de8fb1f3fd1e7", + "sha256:dc525d06ecd102db23ab421b879d7d27021d784ff933e33e8c411a53af5c9dbe" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.1.0" + }, "packaging": { "hashes": [ - "sha256:e9215d2d2535d3ae866c3d6efc77d5b24a0192cce0ff20e42896cc0664f889c0", - "sha256:f019b770dd64e585a99714f1fd5e01c7a8f11b45635aa953fd41c689a657375b" + "sha256:0886227f54515e592aaa2e5a553332c73962917f2831f1b0f9b9f4380a4b9807", + "sha256:f95a1e147590f204328170981833854229bb2912ac3d5f89e2a8ccd2834800c9" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==17.1" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==18.0" }, "passa": { "editable": true, @@ -157,11 +173,11 @@ }, "pyparsing": { "hashes": [ - "sha256:0832bcf47acd283788593e7a0f542407bd9550a55a8a8435214a1960e04bcb04", - "sha256:fee43f17a9c4087e7ed1605bd6df994c6173c1e977d7ade7b651292fab2bd010" + "sha256:bc6c7146b91af3f567cf6daeaec360bc07d45ffec4cf5353f4d7a208ce7ca30a", + "sha256:d29593d8ebe7b57d6967b62494f8c72b03ac0262b1eed63826c6f788b3606401" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.2.0" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.2.2" }, "requests": { "hashes": [ @@ -176,7 +192,7 @@ "sha256:90151d8963f814e17190e067b60e92fb35fd1bc46c99f8dba3d7b0d93a3dd958", "sha256:c3aeaa4e0b80843ba65a68878293e07ea52a8d0706dbba86b02dad6cd20ef2dd" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.6" }, "resolvelib": { @@ -209,7 +225,6 @@ "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" ], - "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.11.0" }, "toml": { @@ -233,7 +248,7 @@ "sha256:57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4", "sha256:a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a" ], - "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "(python_version >= '2.7' and python_version < '2.8') or (python_version >= '3.4' and python_version < '3.5') and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or (python_version >= '2.7' and python_version < '2.8') or (python_version >= '3.4' and python_version < '3.5') and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==3.6.6" }, "urllib3": { @@ -244,6 +259,14 @@ "markers": "python_version < '4' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.23" }, + "virtualenv": { + "hashes": [ + "sha256:2ce32cd126117ce2c539f0134eb89de91a8413a29baac49cbab3eb50e2026669", + "sha256:ca07b4c0b54e14a91af9f34d0919790b016923d157afda5efdde55c96718f752" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==16.0.0" + }, "vistir": { "extras": [ "spinner" @@ -257,11 +280,11 @@ }, "wheel": { "hashes": [ - "sha256:0a2e54558a0628f2145d2fc822137e322412115173e8a2ddbe1c9024338ae83c", - "sha256:80044e51ec5bbf6c894ba0bc48d26a8c20a9ba629f4ca19ea26ecfcf87685f5f" + "sha256:3970f4130b7f8bf8167ca09215c8cc2f49a87c3d46506adfc60cb08c47ab0949", + "sha256:a26bc27230baaec9039972b7cb43db94b17c13e4d66a9ff6a4d46a0344c55c9a" ], "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.31.1" + "version": "==0.32.0" }, "yaspin": { "hashes": [ @@ -294,17 +317,9 @@ "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92", "sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '3.6'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '3.6'", "version": "==1.4.3" }, - "argparse": { - "hashes": [ - "sha256:62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4", - "sha256:c31647edb69fd3d465a847ea3157d37bed1f95f19760b11a47aa91c04b666314" - ], - "markers": "python_version == '2.6'", - "version": "==1.4.0" - }, "arpeggio": { "hashes": [ "sha256:a5258b84f76661d558492fa87e42db634df143685a0e51802d59cae7daad8732", @@ -326,7 +341,7 @@ "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '3.6'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '3.6'", "version": "==18.2.0" }, "babel": { @@ -353,11 +368,27 @@ }, "black": { "hashes": [ - "sha256:22158b89c1a6b4eb333a1e65e791a3f8b998cf3b11ae094adb2570f31f769a44", - "sha256:4b475bbd528acce094c503a3d2dbc2d05a4075f6d0ef7d9e7514518e14cc5191" + "sha256:817243426042db1d36617910df579a54f1afd659adb96fc5032fcf4b36209739", + "sha256:e030a9a28f542debc08acceb273f228ac422798e5215ba2a791a6ddeaaca22a5" ], "markers": "python_version >= '3.6'", - "version": "==18.6b4" + "version": "==18.9b0" + }, + "bleach": { + "hashes": [ + "sha256:0ee95f6167129859c5dce9b1ca291ebdb5d8cd7e382ca0e237dfd0dad63f63d8", + "sha256:24754b9a7d530bf30ce7cbc805bc6cce785660b4a10ff3a43633728438c105ab" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.1.4" + }, + "cached-property": { + "hashes": [ + "sha256:3a026f1a54135677e7da5ce819b0c690f156f37976f3e30c5430740725203d7f", + "sha256:9217a59f14a5682da7c4b8829deadbfc194ac22e9908ccf7c8820234e80a1504" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.5.1" }, "cerberus": { "hashes": [ @@ -374,6 +405,43 @@ "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2018.8.24" }, + "cffi": { + "hashes": [ + "sha256:151b7eefd035c56b2b2e1eb9963c90c6302dc15fbd8c1c0a83a163ff2c7d7743", + "sha256:1553d1e99f035ace1c0544050622b7bc963374a00c467edafac50ad7bd276aef", + "sha256:1b0493c091a1898f1136e3f4f991a784437fac3673780ff9de3bcf46c80b6b50", + "sha256:2ba8a45822b7aee805ab49abfe7eec16b90587f7f26df20c71dd89e45a97076f", + "sha256:3bb6bd7266598f318063e584378b8e27c67de998a43362e8fce664c54ee52d30", + "sha256:3c85641778460581c42924384f5e68076d724ceac0f267d66c757f7535069c93", + "sha256:3eb6434197633b7748cea30bf0ba9f66727cdce45117a712b29a443943733257", + "sha256:495c5c2d43bf6cebe0178eb3e88f9c4aa48d8934aa6e3cddb865c058da76756b", + "sha256:4c91af6e967c2015729d3e69c2e51d92f9898c330d6a851bf8f121236f3defd3", + "sha256:57b2533356cb2d8fac1555815929f7f5f14d68ac77b085d2326b571310f34f6e", + "sha256:770f3782b31f50b68627e22f91cb182c48c47c02eb405fd689472aa7b7aa16dc", + "sha256:79f9b6f7c46ae1f8ded75f68cf8ad50e5729ed4d590c74840471fc2823457d04", + "sha256:7a33145e04d44ce95bcd71e522b478d282ad0eafaf34fe1ec5bbd73e662f22b6", + "sha256:857959354ae3a6fa3da6651b966d13b0a8bed6bbc87a0de7b38a549db1d2a359", + "sha256:87f37fe5130574ff76c17cab61e7d2538a16f843bb7bca8ebbc4b12de3078596", + "sha256:95d5251e4b5ca00061f9d9f3d6fe537247e145a8524ae9fd30a2f8fbce993b5b", + "sha256:9d1d3e63a4afdc29bd76ce6aa9d58c771cd1599fbba8cf5057e7860b203710dd", + "sha256:a36c5c154f9d42ec176e6e620cb0dd275744aa1d804786a71ac37dc3661a5e95", + "sha256:a6a5cb8809091ec9ac03edde9304b3ad82ad4466333432b16d78ef40e0cce0d5", + "sha256:ae5e35a2c189d397b91034642cb0eab0e346f776ec2eb44a49a459e6615d6e2e", + "sha256:b0f7d4a3df8f06cf49f9f121bead236e328074de6449866515cea4907bbc63d6", + "sha256:b75110fb114fa366b29a027d0c9be3709579602ae111ff61674d28c93606acca", + "sha256:ba5e697569f84b13640c9e193170e89c13c6244c24400fc57e88724ef610cd31", + "sha256:be2a9b390f77fd7676d80bc3cdc4f8edb940d8c198ed2d8c0be1319018c778e1", + "sha256:ca1bd81f40adc59011f58159e4aa6445fc585a32bb8ac9badf7a2c1aa23822f2", + "sha256:d5d8555d9bfc3f02385c1c37e9f998e2011f0db4f90e250e5bc0c0a85a813085", + "sha256:e55e22ac0a30023426564b1059b035973ec82186ddddbac867078435801c7801", + "sha256:e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4", + "sha256:ecbb7b01409e9b782df5ded849c178a0aa7c906cf8c5a67368047daab282b184", + "sha256:ed01918d545a38998bfa5902c7c00e0fee90e957ce036a4000a88e3fe2264917", + "sha256:edabd457cd23a02965166026fd9bfd196f4324fe6032e866d0f3bd0301cd486f", + "sha256:fdf1c1dc5bafc32bc5d08b054f94d659422b05aba244d6be4ddc1c72d9aa70fb" + ], + "version": "==1.11.5" + }, "chardet": { "hashes": [ "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", @@ -384,10 +452,44 @@ }, "click": { "hashes": [ - "sha256:29f99fc6125fbc931b758dc053b3114e55c77a6e4c6c3a2674a2dc986016381d", - "sha256:f15516df478d5a56180fbf80e68f206010e6d160fc39fa508b65e035fd75130b" - ], - "version": "==6.7" + "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", + "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7" + ], + "markers": "python_version >= '2.7' and python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==7.0" + }, + "cmarkgfm": { + "hashes": [ + "sha256:0186dccca79483e3405217993b83b914ba4559fe9a8396efc4eea56561b74061", + "sha256:1a625afc6f62da428df96ec325dc30866cc5781520cbd904ff4ec44cf018171c", + "sha256:207b7673ff4e177374c572feeae0e4ef33be620ec9171c08fd22e2b796e03e3d", + "sha256:275905bb371a99285c74931700db3f0c078e7603bed383e8cf1a09f3ee05a3de", + "sha256:50098f1c4950722521f0671e54139e0edc1837d63c990cf0f3d2c49607bb51a2", + "sha256:50ed116d0b60a07df0dc7b180c28569064b9d37d1578d4c9021cff04d725cb63", + "sha256:61a72def110eed903cd1848245897bcb80d295cd9d13944d4f9f30cba5b76655", + "sha256:64186fb75d973a06df0e6ea12879533b71f6e7ba1ab01ffee7fc3e7534758889", + "sha256:665303d34d7f14f10d7b0651082f25ebf7107f29ef3d699490cac16cdc0fc8ce", + "sha256:70b18f843aec58e4e64aadce48a897fe7c50426718b7753aaee399e72df64190", + "sha256:761ee7b04d1caee2931344ac6bfebf37102ffb203b136b676b0a71a3f0ea3c87", + "sha256:811527e9b7280b136734ed6cb6845e5fbccaeaa132ddf45f0246cbe544016957", + "sha256:987b0e157f70c72a84f3c2f9ef2d7ab0f26c08f2bf326c12c087ff9eebcb3ff5", + "sha256:9fc6a2183d0a9b0974ec7cdcdad42bd78a3be674cc3e65f87dd694419b3b0ab7", + "sha256:a3d17ee4ae739fe16f7501a52255c2e287ac817cfd88565b9859f70520afffea", + "sha256:ba5b5488719c0f2ced0aa1986376f7baff1a1653a8eb5fdfcf3f84c7ce46ef8d", + "sha256:c573ea89dd95d41b6d8cf36799c34b6d5b1eac4aed0212dee0f0a11fb7b01e8f", + "sha256:c5f1b9e8592d2c448c44e6bc0d91224b16ea5f8293908b1561de1f6d2d0658b1", + "sha256:cbe581456357d8f0674d6a590b1aaf46c11d01dd0a23af147a51a798c3818034", + "sha256:cf219bec69e601fe27e3974b7307d2f06082ab385d42752738ad2eb630a47d65", + "sha256:cf5014eb214d814a83a7a47407272d5db10b719dbeaf4d3cfe5969309d0fcf4b", + "sha256:d08bad67fa18f7e8ff738c090628ee0cbf0505d74a991c848d6d04abfe67b697", + "sha256:d6f716d7b1182bf35862b5065112f933f43dd1aa4f8097c9bcfb246f71528a34", + "sha256:e08e479102627641c7cb4ece421c6ed4124820b1758765db32201136762282d9", + "sha256:e20ac21418af0298437d29599f7851915497ce9f2866bc8e86b084d8911ee061", + "sha256:e25f53c37e319241b9a412382140dffac98ca756ba8f360ac7ab5e30cad9670a", + "sha256:e8932bddf159064f04e946fbb64693753488de21586f20e840b3be51745c8c09", + "sha256:f20900f16377f2109783ae9348d34bc80530808439591c3d3df73d5c7ef1a00c" + ], + "version": "==0.4.2" }, "colorama": { "hashes": [ @@ -436,10 +538,10 @@ }, "distlib": { "hashes": [ - "sha256:cd502c66fc27c535bab62dc4f482e403e2369c2c05281a79cc2d4e2f42a87f20" + "sha256:57977cd7d9ea27986ec62f425630e4ddb42efe651ff80bc58ed8dbc3c7c21f19" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.2.7" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.2.8" }, "docutils": { "hashes": [ @@ -447,7 +549,6 @@ "sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274", "sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.14" }, "enum34": { @@ -473,7 +574,7 @@ "sha256:3bb3de3582cb27071cfb514f00ed784dc444b7f96dc21e140de65fe00585c95e", "sha256:41d5b64e70507d0c3ca742d68010a76060eea8a3d863e9b5130ab11a4a91aa0e" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.0.1" }, "funcsigs": { @@ -484,6 +585,20 @@ "markers": "python_version < '3.0' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.0.2" }, + "future": { + "hashes": [ + "sha256:e39ced1ab767b5936646cedba8bcce582398233d6a627067d4c6a454c90cfedb" + ], + "version": "==0.16.0" + }, + "html5lib": { + "hashes": [ + "sha256:20b159aa3badc9d5ee8f5c647e5efd02ed2a66ab8d354930bd9ff139fc1dc0a3", + "sha256:66cb0dcfdbbc4f9c3ba1a63fdb511ffdbd4f513b2b6d81b80cd26ce6b3fb3736" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.0.1" + }, "idna": { "hashes": [ "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", @@ -514,6 +629,14 @@ ], "version": "==17.5.0" }, + "installer": { + "hashes": [ + "sha256:1ba23de573e9b95a8dcbd04fd026c40a64b77db0aadc48f28a844b4cb87479fe", + "sha256:f4f195c9b17ea7d2b631a758451485c6b080975349b4adebe45ef4bb022db069" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.1.1" + }, "invoke": { "hashes": [ "sha256:4f4de934b15c2276caa4fbc5a3b8a61c0eb0b234f2be1780d2b793321995c2d6", @@ -559,13 +682,21 @@ "markers": "python_version < '3.0' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1" }, + "packagebuilder": { + "hashes": [ + "sha256:1e85c4e0e994322996b93cd6685c12834d30f3558889154f8e3de8fb1f3fd1e7", + "sha256:dc525d06ecd102db23ab421b879d7d27021d784ff933e33e8c411a53af5c9dbe" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.1.0" + }, "packaging": { "hashes": [ - "sha256:e9215d2d2535d3ae866c3d6efc77d5b24a0192cce0ff20e42896cc0664f889c0", - "sha256:f019b770dd64e585a99714f1fd5e01c7a8f11b45635aa953fd41c689a657375b" + "sha256:0886227f54515e592aaa2e5a553332c73962917f2831f1b0f9b9f4380a4b9807", + "sha256:f95a1e147590f204328170981833854229bb2912ac3d5f89e2a8ccd2834800c9" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==17.1" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==18.0" }, "parver": { "hashes": [ @@ -633,29 +764,35 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.6.0" }, + "pycparser": { + "hashes": [ + "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.19" + }, "pygments": { "hashes": [ "sha256:78f3f434bcc5d6ee09020f92ba487f95ba50f1e3ef83ae96b9d5ffa1bab25c5d", "sha256:dbae1046def0efb574852fab9e90209b23f556367b5a320c0bcb871c77c3e8cc" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.2.0" }, "pyparsing": { "hashes": [ - "sha256:0832bcf47acd283788593e7a0f542407bd9550a55a8a8435214a1960e04bcb04", - "sha256:fee43f17a9c4087e7ed1605bd6df994c6173c1e977d7ade7b651292fab2bd010" + "sha256:bc6c7146b91af3f567cf6daeaec360bc07d45ffec4cf5353f4d7a208ce7ca30a", + "sha256:d29593d8ebe7b57d6967b62494f8c72b03ac0262b1eed63826c6f788b3606401" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.2.0" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.2.2" }, "pytest": { "hashes": [ - "sha256:453cbbbe5ce6db38717d282b758b917de84802af4288910c12442984bde7b823", - "sha256:a8a07f84e680482eb51e244370aaf2caa6301ef265f37c2bdefb3dd3b663f99d" + "sha256:0a72d8a9f559c006ba153e0c9b4838efd7b656cf1f993747ba7128770d6eb12c", + "sha256:95529588ff4e85114a0b0ad8e9cf0131ca47d46b28230e25366c5aba66b1d854" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==3.8.0" + "version": "==3.8.1" }, "pytest-cov": { "hashes": [ @@ -683,11 +820,11 @@ }, "pytest-xdist": { "hashes": [ - "sha256:0875deac20f6d96597036bdf63970887a6f36d28289c2f6682faf652dfea687b", - "sha256:28e25e79698b2662b648319d3971c0f9ae0e6500f88258ccb9b153c31110ba9b" + "sha256:06aa39361694c9365baaa03bec71159b59ad06c9826c6279ebba368cb3571561", + "sha256:1ef0d05c905cfa0c5442c90e9e350e65c6ada120e33a00a066ca51c89f5f869a" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.23.0" + "version": "==1.23.2" }, "pytz": { "hashes": [ @@ -697,6 +834,13 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2018.5" }, + "readme-renderer": { + "hashes": [ + "sha256:237ca8705ffea849870de41101dba41543561da05c0ae45b2f1c547efa9843d2", + "sha256:f75049a3a7afa57165551e030dd8f9882ebf688b9600535a3f7e23596651875d" + ], + "version": "==22.0" + }, "requests": { "hashes": [ "sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1", @@ -717,7 +861,7 @@ "sha256:90151d8963f814e17190e067b60e92fb35fd1bc46c99f8dba3d7b0d93a3dd958", "sha256:c3aeaa4e0b80843ba65a68878293e07ea52a8d0706dbba86b02dad6cd20ef2dd" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.6" }, "resolvelib": { @@ -750,7 +894,6 @@ "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" ], - "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.11.0" }, "snowballstemmer": { @@ -763,11 +906,11 @@ }, "sphinx": { "hashes": [ - "sha256:95acd6648902333647a0e0564abdb28a74b0a76d2333148aa35e5ed1f56d3c4b", - "sha256:c091dbdd5cc5aac6eb95d591a819fd18bccec90ffb048ec465b165a48b839b45" + "sha256:652eb8c566f18823a022bb4b6dbc868d366df332a11a0226b5bc3a798a479f17", + "sha256:d222626d8356de702431e813a05c68a35967e3d66c6cd1c2c89539bb179a7464" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.8.0" + "version": "==1.8.1" }, "sphinx-rtd-theme": { "hashes": [ @@ -816,10 +959,10 @@ }, "twine": { "hashes": [ - "sha256:08eb132bbaec40c6d25b358f546ec1dc96ebd2638a86eea68769d9e67fe2b129", - "sha256:2fd9a4d9ff0bcacf41fdc40c8cb0cfaef1f1859457c9653fd1b92237cc4e9f25" + "sha256:7d89bc6acafb31d124e6e5b295ef26ac77030bf098960c2a4c4e058335827c5c", + "sha256:fad6f1251195f7ddd1460cb76d6ea106c93adb4e56c41e0da79658e56e547d2c" ], - "version": "==1.11.0" + "version": "==1.12.1" }, "typing": { "hashes": [ @@ -827,7 +970,7 @@ "sha256:57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4", "sha256:a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a" ], - "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "(python_version >= '2.7' and python_version < '2.8') or (python_version >= '3.4' and python_version < '3.5') and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or (python_version >= '2.7' and python_version < '2.8') or (python_version >= '3.4' and python_version < '3.5') and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==3.6.6" }, "urllib3": { @@ -838,6 +981,14 @@ "markers": "python_version < '4' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.23" }, + "virtualenv": { + "hashes": [ + "sha256:2ce32cd126117ce2c539f0134eb89de91a8413a29baac49cbab3eb50e2026669", + "sha256:ca07b4c0b54e14a91af9f34d0919790b016923d157afda5efdde55c96718f752" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==16.0.0" + }, "vistir": { "extras": [ "spinner" @@ -849,13 +1000,21 @@ "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.1.6" }, + "webencodings": { + "hashes": [ + "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78", + "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.5.1" + }, "wheel": { "hashes": [ - "sha256:0a2e54558a0628f2145d2fc822137e322412115173e8a2ddbe1c9024338ae83c", - "sha256:80044e51ec5bbf6c894ba0bc48d26a8c20a9ba629f4ca19ea26ecfcf87685f5f" + "sha256:3970f4130b7f8bf8167ca09215c8cc2f49a87c3d46506adfc60cb08c47ab0949", + "sha256:a26bc27230baaec9039972b7cb43db94b17c13e4d66a9ff6a4d46a0344c55c9a" ], "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.31.1" + "version": "==0.32.0" }, "yaspin": { "hashes": [ From 1f48d03da94e7e88986ad9f1da3b1020a99da18d Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 1 Oct 2018 19:24:14 -0400 Subject: [PATCH 18/34] Update pipfile and lockfile Signed-off-by: Dan Ryan --- Pipfile | 2 +- Pipfile.lock | 31 +++++++++++++++++++++++++++---- setup.cfg | 9 +++++---- 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/Pipfile b/Pipfile index 25ed7ee..51464ab 100644 --- a/Pipfile +++ b/Pipfile @@ -1,5 +1,5 @@ [packages] -passa = { editable = true, path = '.' } +passa = { editable = true, path = '.', extras = ['virtualenv'] } [dev-packages] black = '*' diff --git a/Pipfile.lock b/Pipfile.lock index 473a2d4..4564203 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "ff353680e286436d85c9b1b00ccb7ee257cb2922a1eec0becf9713817605d92b" + "sha256": "994d50f9fd0acc91cf216cf0ffc1233ebbfd0a411b57320f44a0f4487943e546" }, "pipfile-spec": 6, "requires": {}, @@ -44,6 +44,14 @@ "markers": "python_version < '3.3' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.0.post1" }, + "cached-property": { + "hashes": [ + "sha256:3a026f1a54135677e7da5ce819b0c690f156f37976f3e30c5430740725203d7f", + "sha256:9217a59f14a5682da7c4b8829deadbfc194ac22e9908ccf7c8820234e80a1504" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.5.1" + }, "cerberus": { "hashes": [ "sha256:f5c2e048fb15ecb3c088d192164316093fcfa602a74b3386eefb2983aa7e800a" @@ -141,8 +149,8 @@ }, "passa": { "editable": true, - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "path": "." + "path": ".", + "extras": ["virtualenv"] }, "pathlib2": { "hashes": [ @@ -179,6 +187,14 @@ "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.2.2" }, + "recursive-monkey-patch": { + "hashes": [ + "sha256:546739fea5be2ea9f98b5ec44fafeb697b5cf9fdcda64a03422582ab03ee24c4", + "sha256:98922554e77f2e2c85a4f5d873a0f52efdc1b553f32444bd6c788b2ff583bf3e" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.4.0" + }, "requests": { "hashes": [ "sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1", @@ -675,6 +691,14 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==4.3.0" }, + "mork": { + "hashes": [ + "sha256:13772edb4724915cf0cfa30d31426e0565487a3b2d7883b8468718eaed8ecfc2", + "sha256:b1b41bc31603eef1b50e42e75ae2d74d7a0d9ab46ea4d0dd1ba387a451870873" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.1.4" + }, "ordereddict": { "hashes": [ "sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f" @@ -711,7 +735,6 @@ "extras": [ "tests" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "path": "." }, "pathlib2": { diff --git a/setup.cfg b/setup.cfg index 194cc04..097fa9b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,15 +37,17 @@ python_requires = >=2.7,!=3.0,!=3.1,!=3.2,!=3.3 setup_requires = setuptools>=36.2.2 install_requires = appdirs + cached-property distlib - packaging - packagebuilder installer + packagebuilder + packaging pip-shims>=0.1.2 plette[validation]>=0.2.2 + recursive-monkey-patch requests - resolvelib>=0.2.1,!=1.0.0.dev0 requirementslib>=1.1.1 + resolvelib>=0.2.1,!=1.0.0.dev0 six virtualenv vistir[spinner]>=0.1.4 @@ -57,7 +59,6 @@ pack = virtualenv = mork tests = - cached-property pytest-xdist pytest-timeout pytest-cov From 4dc19fd5ff4249db60e67eaeef634294fe6e24f2 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 1 Oct 2018 20:32:44 -0400 Subject: [PATCH 19/34] Fix packing Signed-off-by: Dan Ryan --- Pipfile.lock | 8 ++++++++ tasks/package.py | 1 - 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Pipfile.lock b/Pipfile.lock index 4564203..8eb9676 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -131,6 +131,14 @@ "markers": "python_version >= '2.6' and python_version >= '3.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.0.0" }, + "mork": { + "hashes": [ + "sha256:13772edb4724915cf0cfa30d31426e0565487a3b2d7883b8468718eaed8ecfc2", + "sha256:b1b41bc31603eef1b50e42e75ae2d74d7a0d9ab46ea4d0dd1ba387a451870873" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.1.4" + }, "packagebuilder": { "hashes": [ "sha256:1e85c4e0e994322996b93cd6685c12834d30f3558889154f8e3de8fb1f3fd1e7", diff --git a/tasks/package.py b/tasks/package.py index f229e3a..bba878b 100644 --- a/tasks/package.py +++ b/tasks/package.py @@ -22,7 +22,6 @@ 'importlib', # We only support 2.7 so this is not needed. 'modutil', # This breaks <3.7. - 'toml', # Why is requirementslib still not dropping it? 'typing', # This breaks 2.7. We'll provide a special stub for it. } From b1f2e17a799e448008af04bc108b38e0cb3671ae Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 21 Sep 2018 01:13:15 -0400 Subject: [PATCH 20/34] Fix package task Signed-off-by: Dan Ryan --- tasks/package.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tasks/package.py b/tasks/package.py index bba878b..ff8cd47 100644 --- a/tasks/package.py +++ b/tasks/package.py @@ -5,7 +5,8 @@ import distlib.scripts import distlib.wheel import invoke -import passa.internals._pip +import packagebuilder +import passa.models.caches import plette import requirementslib @@ -74,8 +75,8 @@ def pack(ctx, remove_lib=True): package.pop('editable', None) # Don't install things as editable. package.pop('markers', None) # Always install everything. r = requirementslib.Requirement.from_pipfile(name, package) - wheel = passa.internals._pip.build_wheel( - r.as_ireq(), sources, r.hashes or None, + wheel = packagebuilder._pip.build_wheel( + r.as_ireq(), sources, r.hashes or None, cache_dir=passa.models.caches.CACHE_DIR ) wheel.install(paths, maker, lib_only=True) From c47b7a6a5a27b5d8c305324f131a61289b167068 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 22 Sep 2018 19:03:14 -0400 Subject: [PATCH 21/34] Add LRU cache for cacheable function calls Signed-off-by: Dan Ryan --- src/passa/internals/markers.py | 19 ++++++++++++++++--- src/passa/internals/specifiers.py | 13 +++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/passa/internals/markers.py b/src/passa/internals/markers.py index 95efab9..c2b7fdd 100644 --- a/src/passa/internals/markers.py +++ b/src/passa/internals/markers.py @@ -4,6 +4,17 @@ from packaging.markers import Marker +try: + from functools import lru_cache +except ImportError: + from backports.functools_lru_cache import lru_cache + + +def _ensure_marker(marker): + if not isinstance(marker, Marker): + return Marker(str(marker)) + return marker + def _strip_extra(elements): """Remove the "extra == ..." operands from the list. @@ -49,7 +60,7 @@ def get_without_extra(marker): # meet the demands of a pep... -d if not marker: return None - marker = Marker(str(marker)) + marker = _ensure_marker(marker) elements = marker._markers _strip_extra(elements) if elements: @@ -68,6 +79,7 @@ def _markers_collect_extras(markers, collection): _markers_collect_extras(el, collection) +@lru_cache(maxsize=128) def get_contained_extras(marker): """Collect "extra == ..." operands from a marker. @@ -75,8 +87,8 @@ def get_contained_extras(marker): """ if not marker: return set() - marker = Marker(str(marker)) extras = set() + marker = _ensure_marker(marker) _markers_collect_extras(marker._markers, extras) return extras @@ -92,10 +104,11 @@ def _markers_contains_extra(markers): return False +@lru_cache(maxsize=128) def contains_extra(marker): """Check whehter a marker contains an "extra == ..." operand. """ if not marker: return False - marker = Marker(str(marker)) + marker = _ensure_marker(marker) return _markers_contains_extra(marker._markers) diff --git a/src/passa/internals/specifiers.py b/src/passa/internals/specifiers.py index 75afb6a..bf06ce5 100644 --- a/src/passa/internals/specifiers.py +++ b/src/passa/internals/specifiers.py @@ -9,10 +9,18 @@ from vistir.misc import dedup +try: + from functools import lru_cache +except ImportError: + from backports.functools_lru_cache import lru_cache + + +@lru_cache(maxsize=128) def _tuplize_version(version): return tuple(int(x) for x in version.split(".")) +@lru_cache(maxsize=128) def _format_version(version): return ".".join(str(i) for i in version) @@ -21,6 +29,7 @@ def _format_version(version): REPLACE_RANGES = {">": ">=", "<=": "<"} +@lru_cache(maxsize=128) def _format_pyspec(specifier): if isinstance(specifier, str): if not any(op in specifier for op in Specifier._operators.keys()): @@ -42,6 +51,7 @@ def _format_pyspec(specifier): return specifier +@lru_cache(maxsize=128) def _get_specs(specset): if isinstance(specset, Specifier): specset = str(specset) @@ -53,6 +63,7 @@ def _get_specs(specset): ] +@lru_cache(maxsize=128) def _group_by_op(specs): specs = [_get_specs(x) for x in list(specs)] flattened = [(op, version) for spec in specs for op, version in spec] @@ -61,6 +72,7 @@ def _group_by_op(specs): return grouping +@lru_cache(maxsize=128) def cleanup_pyspecs(specs, joiner="or"): specs = {_format_pyspec(spec) for spec in specs} # for != operator we want to group by version @@ -113,6 +125,7 @@ def cleanup_pyspecs(specs, joiner="or"): return results +@lru_cache(maxsize=128) def pyspec_from_markers(marker): if marker._markers[0][0] != 'python_version': return From 404b9e08aba6818d7faaa034041d59dff034e75f Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 22 Sep 2018 19:06:40 -0400 Subject: [PATCH 22/34] Drop markers from editable requirements - Also clean up python specifiers Signed-off-by: Dan Ryan --- src/passa/internals/specifiers.py | 23 ++++++++++++++--------- src/passa/models/lockers.py | 3 +++ src/passa/models/metadata.py | 5 +++-- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/passa/internals/specifiers.py b/src/passa/internals/specifiers.py index bf06ce5..fd9aec8 100644 --- a/src/passa/internals/specifiers.py +++ b/src/passa/internals/specifiers.py @@ -5,7 +5,9 @@ import itertools import operator -from packaging.specifiers import SpecifierSet, Specifier +from packaging.markers import Marker +from packaging.specifiers import Specifier, SpecifierSet + from vistir.misc import dedup @@ -17,7 +19,7 @@ @lru_cache(maxsize=128) def _tuplize_version(version): - return tuple(int(x) for x in version.split(".")) + return tuple(int(x) for x in filter(lambda i: i != "*", version.split("."))) @lru_cache(maxsize=128) @@ -35,8 +37,9 @@ def _format_pyspec(specifier): if not any(op in specifier for op in Specifier._operators.keys()): specifier = "=={0}".format(specifier) specifier = Specifier(specifier) - if specifier.operator == "==" and specifier.version.endswith(".*"): - specifier = Specifier("=={0}".format(specifier.version[:-2])) + version = specifier.version.replace(".*", "") + if ".*" in specifier.version: + specifier = Specifier("{0}{1}".format(specifier.operator, version)) try: op = REPLACE_RANGES[specifier.operator] except KeyError: @@ -53,14 +56,16 @@ def _format_pyspec(specifier): @lru_cache(maxsize=128) def _get_specs(specset): + if specset is None: + return if isinstance(specset, Specifier): specset = str(specset) if isinstance(specset, str): specset = SpecifierSet(specset.replace(".*", "")) - return [ - (spec._spec[0], _tuplize_version(spec._spec[1])) - for spec in getattr(specset, "_specs", []) - ] + result = [] + for spec in set(specset): + result.append((spec.operator, _tuplize_version(spec.version))) + return result @lru_cache(maxsize=128) @@ -78,7 +83,7 @@ def cleanup_pyspecs(specs, joiner="or"): # for != operator we want to group by version # if all are consecutive, join as a list results = set() - for op, versions in _group_by_op(specs): + for op, versions in _group_by_op(tuple(specs)): versions = [version[1] for version in versions] versions = sorted(dedup(versions)) # if we are doing an or operation, we need to use the min for >= diff --git a/src/passa/models/lockers.py b/src/passa/models/lockers.py index c25ca60..53f1cab 100644 --- a/src/passa/models/lockers.py +++ b/src/passa/models/lockers.py @@ -71,6 +71,9 @@ def _collect_derived_entries(state, traces, identifiers): extras[name].extend(requirement.extras) except KeyError: extras[name] = list(requirement.extras) + if requirement.editable and requirement.markers: + requirement.markers = set() + requirement.req.req.markers = set() entries[name] = next(iter(requirement.as_pipfile().values())) for name, ext in extras.items(): entries[name]["extras"] = ext diff --git a/src/passa/models/metadata.py b/src/passa/models/metadata.py index a949f1e..45fb223 100644 --- a/src/passa/models/metadata.py +++ b/src/passa/models/metadata.py @@ -91,12 +91,13 @@ def _build_metasets(dependencies, pythons, key, trace, all_metasets): return all_parent_metasets.append((parent, parent_metasets)) - metaset_iters = [] + metasets = set() for parent, parent_metasets in all_parent_metasets: r = dependencies[parent][key] python = pythons[key] + markers = None if r.editable else get_without_extra(r.markers) metaset = ( - get_without_extra(r.markers), + markers, packaging.specifiers.SpecifierSet(python), ) metaset_iters.append( From 0ffc28ebec99d6b7968b707ed3cfaa0d52f40e2a Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 22 Sep 2018 19:09:00 -0400 Subject: [PATCH 23/34] Add PySpec object to handle comparison and consolidation - Also add magic methods to MetaSet object - Super fast and efficient now due to set comparisons - Old method had sometimes thousands of the same pyspecs - This consolidates as it builds Signed-off-by: Dan Ryan --- src/passa/internals/specifiers.py | 163 +++++++++++++++++++++++++++++- src/passa/models/metadata.py | 56 ++++++---- 2 files changed, 197 insertions(+), 22 deletions(-) diff --git a/src/passa/internals/specifiers.py b/src/passa/internals/specifiers.py index fd9aec8..ae2025b 100644 --- a/src/passa/internals/specifiers.py +++ b/src/passa/internals/specifiers.py @@ -2,9 +2,11 @@ from __future__ import absolute_import, unicode_literals +import collections import itertools import operator +from cached_property import cached_property from packaging.markers import Marker from packaging.specifiers import Specifier, SpecifierSet @@ -44,7 +46,6 @@ def _format_pyspec(specifier): op = REPLACE_RANGES[specifier.operator] except KeyError: return specifier - version = specifier.version.replace(".*", "") curr_tuple = _tuplize_version(version) try: next_tuple = (curr_tuple[0], curr_tuple[1] + 1) @@ -152,3 +153,163 @@ def pyspec_from_markers(marker): if specset: return specset return None + + +class PySpecs(collections.Set): + def __init__(self, specs=None): + if not specs: + specs = SpecifierSet() + self.specifierset = specs + self.previous_specifierset = None + self.cleaned_tuples = set() + self.markers = None + self.clean() + + def __key(self): + return tuple(sorted(self.specifierset, key=operator.attrgetter("_spec"))) + + def __contains__(self, other): + # The current specifierset fully has every value in the supplied specifierset + if not other.as_set - self.as_set: + return True + return False + + def __eq__(self, other): + return self.__key() == other.__key() + + def __hash__(self): + return hash(self.__key()) + + def __repr__(self): + return u"PySpecs({0!r})".format(str(self.specifierset)) + + def __len__(self): + return len(self.cleaned_tuples) + + def __iter__(self): + for version in self.as_string_set: + yield version + return + + def clean(self): + if len(set(self.specifierset)) == 1: + spec = next(iter(spec for spec in self.specifierset), None) + if spec: + self.cleaned_tuples.add((spec.operator, spec.version)) + else: + self.cleaned_tuples = cleanup_pyspecs(self.specifierset) + self.specifierset = self.as_specset + + def add(self, other): + new_pyspec = PySpecs(self.specifierset) + new_pyspec.specifierset &= other.specifierset + return new_pyspec + + @cached_property + @lru_cache(maxsize=128) + def as_specset(self): + specs = set() + for spec in self.cleaned_tuples: + op, value = spec + if op in ('in', 'not in'): + new_op = '!=' if op == 'not in' else '==' + for val in value.split(","): + specs.add(Specifier("{0}{1}".format(new_op, val))) + else: + specs.add(Specifier("{0}{1}".format(op, value))) + specifierset = SpecifierSet() + specifierset._specs = frozenset(specs) + return specifierset + + @cached_property + @lru_cache(maxsize=128) + def as_set(self): + return set(self.specifierset) + + @cached_property + @lru_cache(maxsize=128) + def as_string_set(self): + returnval = set() + if len(self.cleaned_tuples) == 1: + val = next(iter(spec for spec in self.cleaned_tuples), None) + if val: + returnval.add("python_version {0[0]} '{0[1]}'".format(val)) + return returnval + return set( + "python_version {0[0]} '{0[1]}'".format(s) + for s in sorted(self.cleaned_tuples) + ) + + @cached_property + @lru_cache(maxsize=128) + def marker_set(self): + markerset = {Marker(spec) for spec in self.as_string_set} + return markerset + + @cached_property + @lru_cache(maxsize=128) + def marker_string(self): + marker_string = " and ".join(sorted(str(m) for m in self.as_string_set)) + if not marker_string: + return "" + return marker_string + + @cached_property + @lru_cache(maxsize=128) + def as_markers(self): + print("generating marker using string: %s" % self.marker_string) + if not self.marker_string: + return "" + marker = Marker(self.marker_string) + return marker + + @lru_cache(maxsize=128) + def __str__(self): + string_repr = u"{0}".format(str(self.marker_string)) + print("converting to string: %s" % string_repr) + return string_repr + + def __bool__(self): + return bool(self.specifierset) + + def __nonzero__(self): # Python 2. + return self.__bool__() + + @lru_cache(maxsize=128) + def __or__(self, specset): + if not isinstance(specset, PySpecs): + specset = PySpecs(specset) + if str(self) == str(specset): + return self + combined_set = self.as_set | specset.as_set + new_specset = SpecifierSet() + new_specset._specs = frozenset(combined_set) + new_pyspec = PySpecs(new_specset) + return new_pyspec + + @classmethod + @lru_cache(maxsize=128) + def from_marker(cls, marker): + if marker._markers[0][0] != 'python_version': + return + op = marker._markers[0][1].value + version = marker._markers[0][2].value + specset = set() + if op == "in": + specset.update( + Specifier("=={0}".format(v.strip())) + for v in version.split(",") + ) + elif op == "not in": + specset.update( + Specifier("!={0}".format(v.strip())) + for v in version.split(",") + ) + else: + specset.add(Specifier("".join([op, version]))) + if specset: + specifierset = SpecifierSet() + specifierset._specs = frozenset(specset) + newset = cls(specifierset) + return newset + return None diff --git a/src/passa/models/metadata.py b/src/passa/models/metadata.py index 45fb223..e626678 100644 --- a/src/passa/models/metadata.py +++ b/src/passa/models/metadata.py @@ -11,7 +11,7 @@ import vistir.misc from ..internals.markers import get_without_extra -from ..internals.specifiers import cleanup_pyspecs, pyspec_from_markers +from ..internals.specifiers import PySpecs def dedup_markers(s): @@ -28,24 +28,39 @@ class MetaSet(object): """ def __init__(self): self.markerset = frozenset() - self.pyspecset = packaging.specifiers.SpecifierSet() + self.pyspecset = PySpecs() def __repr__(self): return "MetaSet(markerset={0!r}, pyspecset={1!r})".format( ",".join(sorted(self.markerset)), str(self.pyspecset), ) + def __key(self): + return (tuple(self.markerset), hash(self.pyspecset)) + + def __hash__(self): + return hash(self.__key()) + + def __eq__(self, other): + return self.__key() == other.__key() + + def __len__(self): + return len(self.markerset) + len(self.pyspecset) + + def __iter__(self): + return itertools.chain(self.markerset, self.pyspecset) + def __str__(self): - pyspecs = set() + pyspecs = PySpecs() markerset = set() for m in self.markerset: - marker_specs = pyspec_from_markers(packaging.markers.Marker(m)) + marker_specs = PySpecs.from_marker(packaging.markers.Marker(m)) if marker_specs: pyspecs.add(marker_specs) else: markerset.add(m) if pyspecs: - self.pyspecset._specs &= pyspecs + self.pyspecset.add(pyspecs) self.markerset = frozenset(markerset) return " and ".join(dedup_markers(itertools.chain( # Make sure to always use the same quotes so we can dedup properly. @@ -54,8 +69,7 @@ def __str__(self): for ms in (str(m).replace('"', "'") for m in self.markerset) ), ( - "python_version {0[0]} '{0[1]}'".format(spec) - for spec in cleanup_pyspecs(self.pyspecset) + "{0}".format(str(spec)) for spec in self.pyspecset ), ))) @@ -68,16 +82,17 @@ def __nonzero__(self): # Python 2. def __or__(self, pair): marker, specset = pair markerset = set(self.markerset) + specset = PySpecs(specset) if marker: - marker_specs = pyspec_from_markers(marker) + marker_specs = PySpecs.from_marker(marker) if not marker_specs: markerset.add(str(marker)) else: - specset._specs &= marker_specs + specset.add(marker_specs) metaset = MetaSet() metaset.markerset = frozenset(markerset) # TODO: Implement some logic to clean up dups like '3.0.*' and '3.0'. - metaset.pyspecset &= self.pyspecset & specset + metaset.pyspecset = self.pyspecset | specset return metaset @@ -100,11 +115,10 @@ def _build_metasets(dependencies, pythons, key, trace, all_metasets): markers, packaging.specifiers.SpecifierSet(python), ) - metaset_iters.append( - parent_metaset | metaset - for parent_metaset in parent_metasets - ) - return list(itertools.chain.from_iterable(metaset_iters)) + for parent_metaset in parent_metasets: + child_metaset = parent_metaset | metaset + metasets.add(child_metaset) + return list(metasets) def _calculate_metasets_mapping(dependencies, pythons, traces): @@ -118,7 +132,7 @@ def _calculate_metasets_mapping(dependencies, pythons, traces): metasets = _build_metasets( dependencies, pythons, key, trace, all_metasets, ) - if metasets is None: + if metasets is None or len(metasets) == 0: continue new_metasets[key] = metasets if not new_metasets: @@ -136,11 +150,11 @@ def _format_metasets(metasets): return None # This extra str(Marker()) call helps simplify the expression. - return str(packaging.markers.Marker(" or ".join( - "{0}".format(s) if " and " in s else s - for s in dedup_markers(str(metaset) for metaset in metasets - if metaset) - ))) + _metasets = (dedup_markers(str(metaset) for metaset in metasets if metaset)) + metaset_string = " or ".join(meta for meta in list(_metasets)) + if not metaset_string: + return metaset_string + return str(packaging.markers.Marker(metaset_string)) def set_metadata(candidates, traces, dependencies, pythons): From 2345480e0d610affbff9ae798b9e740150cbd933 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 23 Sep 2018 00:25:40 -0400 Subject: [PATCH 24/34] Cleanup Signed-off-by: Dan Ryan Test fixes Signed-off-by: Dan Ryan Fix specifier dedup logic Signed-off-by: Dan Ryan Fix import Signed-off-by: Dan Ryan Fix tox Signed-off-by: Dan Ryan --- src/passa/internals/markers.py | 5 +---- src/passa/internals/specifiers.py | 17 ++++++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/passa/internals/markers.py b/src/passa/internals/markers.py index c2b7fdd..3793e96 100644 --- a/src/passa/internals/markers.py +++ b/src/passa/internals/markers.py @@ -54,10 +54,7 @@ def get_without_extra(marker): This could return `None` if the `extra == ...` part is the only one in the input marker. """ - # TODO: Why is this very deep in the internals? Why is a better solution - # implementing it yourself when someone is already maintaining a codebase - # for this? It's literally a grammar implementation that is required to - # meet the demands of a pep... -d + if not marker: return None marker = _ensure_marker(marker) diff --git a/src/passa/internals/specifiers.py b/src/passa/internals/specifiers.py index ae2025b..a7c413f 100644 --- a/src/passa/internals/specifiers.py +++ b/src/passa/internals/specifiers.py @@ -3,6 +3,7 @@ from __future__ import absolute_import, unicode_literals import collections +import collections.abc import itertools import operator @@ -155,7 +156,7 @@ def pyspec_from_markers(marker): return None -class PySpecs(collections.Set): +class PySpecs(collections.abc.Set): def __init__(self, specs=None): if not specs: specs = SpecifierSet() @@ -249,15 +250,14 @@ def marker_set(self): @cached_property @lru_cache(maxsize=128) def marker_string(self): - marker_string = " and ".join(sorted(str(m) for m in self.as_string_set)) + marker_string = " and ".join(sorted(str(m) for m in self.marker_set)) if not marker_string: return "" - return marker_string + return str(Marker(marker_string)) @cached_property @lru_cache(maxsize=128) def as_markers(self): - print("generating marker using string: %s" % self.marker_string) if not self.marker_string: return "" marker = Marker(self.marker_string) @@ -265,8 +265,7 @@ def as_markers(self): @lru_cache(maxsize=128) def __str__(self): - string_repr = u"{0}".format(str(self.marker_string)) - print("converting to string: %s" % string_repr) + string_repr = "{0}".format(str(self.marker_string)) return string_repr def __bool__(self): @@ -301,9 +300,13 @@ def from_marker(cls, marker): for v in version.split(",") ) elif op == "not in": + versions = version.split(",") + bad_versions = ["3.0", "3.1", "3.2", "3.3"] + if len(versions) >= 2 and any(v in versions for v in bad_versions): + versions = bad_versions specset.update( Specifier("!={0}".format(v.strip())) - for v in version.split(",") + for v in bad_versions ) else: specset.add(Specifier("".join([op, version]))) From 82d3c70d9175cb5ce2c7e5b348fd2900dd9e840e Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 23 Sep 2018 13:53:44 -0400 Subject: [PATCH 25/34] Fix virtualenv usage and Set inheritance Signed-off-by: Dan Ryan Add set operations and comparison methods to metasets Signed-off-by: Dan Ryan Add abstract extraction methods for marker cleanup Signed-off-by: Dan Ryan Add intersection method for PySpecs and smarter unions and creation Signed-off-by: Dan Ryan Simplify and cleanup metadata implementation Signed-off-by: Dan Ryan --- Pipfile.lock | 67 +++++------- src/passa/internals/markers.py | 169 +++++++++++++++++++++++++++--- src/passa/internals/specifiers.py | 102 ++++++++++++++---- src/passa/models/metadata.py | 93 ++++++++++------ 4 files changed, 325 insertions(+), 106 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 8eb9676..edbc80a 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -19,7 +19,6 @@ "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92", "sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '3.6'", "version": "==1.4.3" }, "attrs": { @@ -30,6 +29,12 @@ "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '3.6'", "version": "==18.2.0" }, + "backports-functools-lru-cache": { + "hashes": [ + "sha256:f0b0e4eba956de51238e17573b7087e852dfe9854afd2e9c873f73fc0ca0a6dd" + ], + "version": "==1.5" + }, "backports-shutil-get-terminal-size": { "hashes": [ "sha256:0975ba55054c15e346944b38956a4c9cbee9009391e41b86c68990effb8c1f64" @@ -49,7 +54,6 @@ "sha256:3a026f1a54135677e7da5ce819b0c690f156f37976f3e30c5430740725203d7f", "sha256:9217a59f14a5682da7c4b8829deadbfc194ac22e9908ccf7c8820234e80a1504" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.5.1" }, "cerberus": { @@ -79,7 +83,7 @@ "hashes": [ "sha256:57977cd7d9ea27986ec62f425630e4ddb42efe651ff80bc58ed8dbc3c7c21f19" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.2.8" }, "enum34": { @@ -89,7 +93,7 @@ "sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79", "sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1" ], - "markers": "python_version >= '2.6' and python_version >= '2.7' and python_version < '2.8' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.6" }, "first": { @@ -97,7 +101,6 @@ "sha256:3bb3de3582cb27071cfb514f00ed784dc444b7f96dc21e140de65fe00585c95e", "sha256:41d5b64e70507d0c3ca742d68010a76060eea8a3d863e9b5130ab11a4a91aa0e" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.0.1" }, "idna": { @@ -128,7 +131,7 @@ "sha256:2c85c1666649e92e56de17c00e1e831313602d9b55e8661d39c01e39003b45f7", "sha256:cc3dad264e36ed359fdd67c4588959d2996bd0402ad9c9d974ca906821537218" ], - "markers": "python_version >= '2.6' and python_version >= '3.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.0.0" }, "mork": { @@ -165,7 +168,7 @@ "sha256:8eb170f8d0d61825e09a95b38be068299ddeda82f35e96c3301a8a5e7604cb83", "sha256:d1aa2a11ba7b8f7b21ab852b1fb5afb277e1bb99d5dfc663380b5015c0d80c5a" ], - "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.3.2" }, "pip-shims": { @@ -216,7 +219,6 @@ "sha256:90151d8963f814e17190e067b60e92fb35fd1bc46c99f8dba3d7b0d93a3dd958", "sha256:c3aeaa4e0b80843ba65a68878293e07ea52a8d0706dbba86b02dad6cd20ef2dd" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.6" }, "resolvelib": { @@ -224,7 +226,6 @@ "sha256:6c4c6690b0bdd78bcc002e1a5d1b6abbde58c694a6ea1838f165b20d2c943db7", "sha256:8734e53271ef98f38a2c99324d5e7905bc00c97dc3fc5bb7d83c82a979e71c04" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.2.2" }, "scandir": { @@ -241,7 +242,7 @@ "sha256:c9009c527929f6e25604aec39b0a43c3f831d2947d89d6caaab22f057b7055c8", "sha256:f5c71e29b4e2af7ccdc03a020c626ede51da471173b4a6ad1e904f2b2e04b4bd" ], - "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.9.0" }, "six": { @@ -272,7 +273,7 @@ "sha256:57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4", "sha256:a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a" ], - "markers": "(python_version >= '2.7' and python_version < '2.8') or (python_version >= '3.4' and python_version < '3.5') and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or (python_version >= '2.7' and python_version < '2.8') or (python_version >= '3.4' and python_version < '3.5') and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==3.6.6" }, "urllib3": { @@ -288,7 +289,7 @@ "sha256:2ce32cd126117ce2c539f0134eb89de91a8413a29baac49cbab3eb50e2026669", "sha256:ca07b4c0b54e14a91af9f34d0919790b016923d157afda5efdde55c96718f752" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==16.0.0" }, "vistir": { @@ -333,7 +334,6 @@ "sha256:37228cda29411948b422fae072f57e31d3396d2ee1c9783775980ee9c9990af6", "sha256:58587dd4dc3daefad0487f6d9ae32b4542b185e1c36db6993290e7c41ca2b47c" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.5" }, "appdirs": { @@ -341,7 +341,6 @@ "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92", "sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '3.6'", "version": "==1.4.3" }, "arpeggio": { @@ -357,7 +356,6 @@ "sha256:0312ad34fcad8fac3704d441f7b317e50af620823353ec657a53e981f92920c0", "sha256:ec9ae8adaae229e4f8446952d204a3e4b5fdd2d099f9be3aaf556120135fb3ee" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.2.1" }, "attrs": { @@ -376,6 +374,12 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.6.0" }, + "backports-functools-lru-cache": { + "hashes": [ + "sha256:f0b0e4eba956de51238e17573b7087e852dfe9854afd2e9c873f73fc0ca0a6dd" + ], + "version": "==1.5" + }, "backports-shutil-get-terminal-size": { "hashes": [ "sha256:0975ba55054c15e346944b38956a4c9cbee9009391e41b86c68990effb8c1f64" @@ -411,7 +415,6 @@ "sha256:3a026f1a54135677e7da5ce819b0c690f156f37976f3e30c5430740725203d7f", "sha256:9217a59f14a5682da7c4b8829deadbfc194ac22e9908ccf7c8820234e80a1504" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.5.1" }, "cerberus": { @@ -520,7 +523,7 @@ "sha256:463f8483208e921368c9f306094eb6f725c6ca42b0f97e313cb5d5512459feda", "sha256:48eb22f4f8461b1df5734a074b57042430fb06e1d61bd1e11b078c0fe6d7a1f1" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and sys_platform == 'win32'", + "markers": "sys_platform == 'win32'", "version": "==0.3.9" }, "coverage": { @@ -557,14 +560,13 @@ "sha256:e05cb4d9aad6233d67e0541caa7e511fa4047ed7750ec2510d466e806e0255d6", "sha256:f3f501f345f24383c0000395b26b726e46758b71393267aeae0bd36f8b3ade80" ], - "markers": "python_version < '4' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==4.5.1" }, "distlib": { "hashes": [ "sha256:57977cd7d9ea27986ec62f425630e4ddb42efe651ff80bc58ed8dbc3c7c21f19" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.2.8" }, "docutils": { @@ -582,7 +584,7 @@ "sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79", "sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1" ], - "markers": "python_version >= '2.6' and python_version >= '2.7' and python_version < '2.8' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.6" }, "execnet": { @@ -590,7 +592,6 @@ "sha256:a7a84d5fa07a089186a329528f127c9d73b9de57f1a1131b82bb5320ee651f6a", "sha256:fc155a6b553c66c838d1a22dba1dc9f5f505c43285a878c6f74a79c024750b83" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.5.0" }, "first": { @@ -598,7 +599,6 @@ "sha256:3bb3de3582cb27071cfb514f00ed784dc444b7f96dc21e140de65fe00585c95e", "sha256:41d5b64e70507d0c3ca742d68010a76060eea8a3d863e9b5130ab11a4a91aa0e" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.0.1" }, "funcsigs": { @@ -606,7 +606,7 @@ "sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca", "sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50" ], - "markers": "python_version < '3.0' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.0'", "version": "==1.0.2" }, "future": { @@ -687,7 +687,7 @@ "sha256:2c85c1666649e92e56de17c00e1e831313602d9b55e8661d39c01e39003b45f7", "sha256:cc3dad264e36ed359fdd67c4588959d2996bd0402ad9c9d974ca906821537218" ], - "markers": "python_version >= '2.6' and python_version >= '3.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.0.0" }, "more-itertools": { @@ -696,7 +696,6 @@ "sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e", "sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==4.3.0" }, "mork": { @@ -711,7 +710,7 @@ "hashes": [ "sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f" ], - "markers": "python_version < '3.0' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.0'", "version": "==1.1" }, "packagebuilder": { @@ -750,7 +749,7 @@ "sha256:8eb170f8d0d61825e09a95b38be068299ddeda82f35e96c3301a8a5e7604cb83", "sha256:d1aa2a11ba7b8f7b21ab852b1fb5afb277e1bb99d5dfc663380b5015c0d80c5a" ], - "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.3.2" }, "pip-shims": { @@ -784,7 +783,6 @@ "sha256:6e3836e39f4d36ae72840833db137f7b7d35105079aee6ec4a62d9f80d594dd1", "sha256:95eb8364a4708392bae89035f45341871286a333f749c3141c20573d2b3876e1" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.7.1" }, "py": { @@ -792,7 +790,6 @@ "sha256:06a30435d058473046be836d3fc4f27167fd84c45b99704f2fb5509ef61f9af1", "sha256:50402e9d1c9005d759426988a492e0edaadb7f4e68bcddfea586bc7432d009c6" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.6.0" }, "pycparser": { @@ -822,7 +819,6 @@ "sha256:0a72d8a9f559c006ba153e0c9b4838efd7b656cf1f993747ba7128770d6eb12c", "sha256:95529588ff4e85114a0b0ad8e9cf0131ca47d46b28230e25366c5aba66b1d854" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==3.8.1" }, "pytest-cov": { @@ -830,7 +826,6 @@ "sha256:513c425e931a0344944f84ea47f3956be0e416d95acbd897a44970c8d926d5d7", "sha256:e360f048b7dae3f2f2a9a4d067b2dd6b6a015d384d1577c994a43f3f7cbad762" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.6.0" }, "pytest-forked": { @@ -838,7 +833,6 @@ "sha256:e4500cd0509ec4a26535f7d4112a8cc0f17d3a41c29ffd4eab479d2a55b30805", "sha256:f275cb48a73fc61a6710726348e1da6d68a978f0ec0c54ece5a5fae5977e5a08" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.2" }, "pytest-timeout": { @@ -846,7 +840,6 @@ "sha256:1117fc0536e1638862917efbdc0895e6b62fa61e6cf4f39bb655686af7af9627", "sha256:b050a05da96a9992e90e884bc19b4790678b40c25471d2b77015b388417e1fa8" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.3.2" }, "pytest-xdist": { @@ -892,7 +885,6 @@ "sha256:90151d8963f814e17190e067b60e92fb35fd1bc46c99f8dba3d7b0d93a3dd958", "sha256:c3aeaa4e0b80843ba65a68878293e07ea52a8d0706dbba86b02dad6cd20ef2dd" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.6" }, "resolvelib": { @@ -900,7 +892,6 @@ "sha256:6c4c6690b0bdd78bcc002e1a5d1b6abbde58c694a6ea1838f165b20d2c943db7", "sha256:8734e53271ef98f38a2c99324d5e7905bc00c97dc3fc5bb7d83c82a979e71c04" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.2.2" }, "scandir": { @@ -917,7 +908,7 @@ "sha256:c9009c527929f6e25604aec39b0a43c3f831d2947d89d6caaab22f057b7055c8", "sha256:f5c71e29b4e2af7ccdc03a020c626ede51da471173b4a6ad1e904f2b2e04b4bd" ], - "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.9.0" }, "six": { @@ -1001,7 +992,7 @@ "sha256:57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4", "sha256:a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a" ], - "markers": "(python_version >= '2.7' and python_version < '2.8') or (python_version >= '3.4' and python_version < '3.5') and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or (python_version >= '2.7' and python_version < '2.8') or (python_version >= '3.4' and python_version < '3.5') and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==3.6.6" }, "urllib3": { @@ -1017,7 +1008,7 @@ "sha256:2ce32cd126117ce2c539f0134eb89de91a8413a29baac49cbab3eb50e2026669", "sha256:ca07b4c0b54e14a91af9f34d0919790b016923d157afda5efdde55c96718f752" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==16.0.0" }, "vistir": { diff --git a/src/passa/internals/markers.py b/src/passa/internals/markers.py index 3793e96..08b4322 100644 --- a/src/passa/internals/markers.py +++ b/src/passa/internals/markers.py @@ -3,6 +3,12 @@ from __future__ import absolute_import, unicode_literals from packaging.markers import Marker +from .specifiers import PySpecs, gen_marker +import six +import distlib.markers + +six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) +from six.moves import Mapping, reduce try: from functools import lru_cache @@ -17,24 +23,36 @@ def _ensure_marker(marker): def _strip_extra(elements): - """Remove the "extra == ..." operands from the list. + """Remove the "extra == ..." operands from the list.""" + + return _strip_marker_elem("extra", elements) + + +def _strip_pyversion(elements): + return _strip_marker_elem("python_version", elements) + + +def _strip_marker_elem(elem_name, elements): + """Remove the supplied element from the marker. This is not a comprehensive implementation, but relies on an important - characteristic of metadata generation: The "extra == ..." operand is always + characteristic of metadata generation: The element's operand is always associated with an "and" operator. This means that we can simply remove the operand and the "and" operator associated with it. """ + extra_indexes = [] + preceding_operators = ["and"] if elem_name == "extra" else ["and", "or"] for i, element in enumerate(elements): if isinstance(element, list): - cancelled = _strip_extra(element) + cancelled = _strip_marker_elem(elem_name, element) if cancelled: extra_indexes.append(i) - elif isinstance(element, tuple) and element[0].value == "extra": + elif isinstance(element, tuple) and element[0].value == elem_name: extra_indexes.append(i) for i in reversed(extra_indexes): del elements[i] - if i > 0 and elements[i - 1] == "and": + if i > 0 and elements[i - 1] in preceding_operators: # Remove the "and" before it. del elements[i - 1] elif elements: @@ -45,6 +63,19 @@ def _strip_extra(elements): return (not elements) +def _get_stripped_marker(marker, strip_func): + """Build a new marker which is cleaned according to `strip_func`""" + + if not marker: + return None + marker = _ensure_marker(marker) + elements = marker._markers + strip_func(elements) + if elements: + return marker + return None + + def get_without_extra(marker): """Build a new marker without the `extra == ...` part. @@ -55,14 +86,17 @@ def get_without_extra(marker): input marker. """ - if not marker: - return None - marker = _ensure_marker(marker) - elements = marker._markers - _strip_extra(elements) - if elements: - return marker - return None + return _get_stripped_marker(marker, _strip_extra) + + +def get_without_pyversion(marker): + """Built a new marker without the `python_version` part. + + This could return `None` if the `python_version` section is the only section in the + marker. + """ + + return _get_stripped_marker(marker, _strip_pyversion) def _markers_collect_extras(markers, collection): @@ -76,6 +110,23 @@ def _markers_collect_extras(markers, collection): _markers_collect_extras(el, collection) +def _markers_collect_pyversions(markers, collection): + local_collection = [] + marker_format_str = "{0}" + for i, el in enumerate(reversed(markers)): + if (isinstance(el, tuple) and + el[0].value == "python_version"): + new_marker = str(gen_marker(el)) + local_collection.append(marker_format_str.format(new_marker)) + elif isinstance(el, six.string_types): + local_collection.append(el) + elif isinstance(el, list): + _markers_collect_pyversions(el, local_collection) + if local_collection: + local_collection = "{0}".format(" ".join(local_collection)) + collection.append(local_collection) + + @lru_cache(maxsize=128) def get_contained_extras(marker): """Collect "extra == ..." operands from a marker. @@ -90,13 +141,54 @@ def get_contained_extras(marker): return extras +def get_contained_pyversions(marker): + """Collect all `python_version` operands from a marker. + + Returns a set of :class:`~passa.internals.specifiers.PySpecs` instances. + """ + + collection = [] + if not marker: + return set() + marker = _ensure_marker(marker) + # Collect the (Variable, Op, Value) tuples and string joiners from the marker + _markers_collect_pyversions(marker._markers, collection) + marker_str = " ".join(collection) + if not marker_str: + return set() + # Use the distlib dictionary parser to create a dictionary 'trie' which is a bit + # easier to reason about + marker_dict = distlib.markers.parse_marker(marker_str)[0] + version_set = set() + pyversions = parse_marker_dict(marker_dict) + if isinstance(pyversions, set): + version_set.update(pyversions) + else: + version_set.add(pyversions) + # Each distinct element in the set was separated by an "and" operator in the marker + # So we will need to reduce them with an intersection here rather than a union + # in order to find the boundaries + versions = reduce(lambda x, y: x & y, version_set) + if not versions: + return PySpecs() + return versions + + def _markers_contains_extra(markers): # Optimization: the marker element is usually appended at the end. + return _markers_contains_key(markers, "extra") + + +def _markers_contains_pyversion(markers): + return _markers_contains_key(markers, "python_version") + + +def _markers_contains_key(markers, key): for element in reversed(markers): - if isinstance(element, tuple) and element[0].value == "extra": + if isinstance(element, tuple) and element[0].value == key: return True elif isinstance(element, list): - if _markers_contains_extra(element): + if _markers_contains_key(element, key): return True return False @@ -109,3 +201,50 @@ def contains_extra(marker): return False marker = _ensure_marker(marker) return _markers_contains_extra(marker._markers) + + +@lru_cache(maxsize=128) +def contains_pyversion(marker): + """Check whether a marker contains a python_version operand. + """ + + if not marker: + return False + marker = _ensure_marker(marker) + return _markers_contains_pyversion(marker._markers) + + +def parse_marker_dict(marker_dict): + op = marker_dict["op"] + lhs = marker_dict["lhs"] + rhs = marker_dict["rhs"] + # This is where the spec sets for each side land if we have an "or" operator + sides = set() + # And if we hit the end of the parse tree we use this format string to make a marker + format_string = "{lhs} {op} {rhs}" + # Essentially we will iterate over each side of the parsed marker if either one is + # A mapping instance (i.e. a dictionary) and recursively parse and reduce the specset + # Union the "and" specs, intersect the "or"s to find the most appropriate range + if any(issubclass(type(side), Mapping) for side in (lhs, rhs)): + for side in (lhs, rhs): + specs = PySpecs() + if issubclass(type(side), Mapping): + specs.add(parse_marker_dict(side)) + else: + # This is the easiest way to go from a string to a PySpec instance + specs.add(PySpecs.from_marker(Marker(side))) + sides.add(specs) + if op == "and": + # When we are "and"-ing things together, it probably makes the most sense + # to reduce them here into a single PySpec instance + if not sides: + sides = [lhs, rhs] + sides = reduce(lambda x, y: x | y, sides) + return PySpecs.from_marker(Marker(str(sides))) + # Actually when we "or" things as well we can also just turn them into a reduced + # set using this logic now + return reduce(lambda x, y: x & y, sides) + else: + # At the tip of the tree we are dealing with strings all around and they just need + # to be smashed together + return PySpecs.from_marker(Marker(format_string.format(**marker_dict))) diff --git a/src/passa/internals/specifiers.py b/src/passa/internals/specifiers.py index a7c413f..3f2ae8c 100644 --- a/src/passa/internals/specifiers.py +++ b/src/passa/internals/specifiers.py @@ -2,11 +2,11 @@ from __future__ import absolute_import, unicode_literals -import collections -import collections.abc import itertools import operator +import six + from cached_property import cached_property from packaging.markers import Marker from packaging.specifiers import Specifier, SpecifierSet @@ -14,6 +14,9 @@ from vistir.misc import dedup +six.add_move(six.MovedAttribute("Set", "collections", "collections.abc")) +from six.moves import reduce, Set + try: from functools import lru_cache except ImportError: @@ -66,7 +69,15 @@ def _get_specs(specset): specset = SpecifierSet(specset.replace(".*", "")) result = [] for spec in set(specset): - result.append((spec.operator, _tuplize_version(spec.version))) + version = spec.version + op = spec.operator + if op in ("in", "not in"): + versions = version.split(",") + op = "==" if op == "in" else "!=" + for ver in versions: + result.append((op, _tuplize_version(ver.strip()))) + else: + result.append((spec.operator, _tuplize_version(spec.version))) return result @@ -74,7 +85,7 @@ def _get_specs(specset): def _group_by_op(specs): specs = [_get_specs(x) for x in list(specs)] flattened = [(op, version) for spec in specs for op, version in spec] - specs = sorted(flattened, key=operator.itemgetter(1)) + specs = sorted(flattened) grouping = itertools.groupby(specs, key=operator.itemgetter(0)) return grouping @@ -156,7 +167,14 @@ def pyspec_from_markers(marker): return None -class PySpecs(collections.abc.Set): +def gen_marker(mkr): + m = Marker("python_version == '1'") + m._markers.pop() + m._markers.append(mkr) + return m + + +class PySpecs(Set): def __init__(self, specs=None): if not specs: specs = SpecifierSet() @@ -198,15 +216,26 @@ def clean(self): if spec: self.cleaned_tuples.add((spec.operator, spec.version)) else: - self.cleaned_tuples = cleanup_pyspecs(self.specifierset) + self.cleaned_tuples = cleanup_pyspecs(self.specifierset, joiner="and") self.specifierset = self.as_specset def add(self, other): - new_pyspec = PySpecs(self.specifierset) - new_pyspec.specifierset &= other.specifierset - return new_pyspec - - @cached_property + if not isinstance(other, self.__class__): + if isinstance(other, SpecifierSet): + other = PySpecs(other) + else: + raise TypeError("Cannot add type {0!r} to PySpecs".format(type(other))) + new_specifierset = SpecifierSet() + new_specifierset &= self.as_specset + try: + new_specifierset &= other.as_specset + except AttributeError: + pass + new_pyspec = PySpecs(new_specifierset) + self.specifierset = new_pyspec.specifierset + self.cleaned_tuples = new_pyspec.cleaned_tuples + + @property @lru_cache(maxsize=128) def as_specset(self): specs = set() @@ -222,12 +251,12 @@ def as_specset(self): specifierset._specs = frozenset(specs) return specifierset - @cached_property + @property @lru_cache(maxsize=128) def as_set(self): return set(self.specifierset) - @cached_property + @property @lru_cache(maxsize=128) def as_string_set(self): returnval = set() @@ -241,13 +270,13 @@ def as_string_set(self): for s in sorted(self.cleaned_tuples) ) - @cached_property + @property @lru_cache(maxsize=128) def marker_set(self): markerset = {Marker(spec) for spec in self.as_string_set} return markerset - @cached_property + @property @lru_cache(maxsize=128) def marker_string(self): marker_string = " and ".join(sorted(str(m) for m in self.marker_set)) @@ -255,7 +284,7 @@ def marker_string(self): return "" return str(Marker(marker_string)) - @cached_property + @property @lru_cache(maxsize=128) def as_markers(self): if not self.marker_string: @@ -274,6 +303,29 @@ def __bool__(self): def __nonzero__(self): # Python 2. return self.__bool__() + @lru_cache(maxsize=128) + def __and__(self, other): + if not isinstance(other, PySpecs): + specset = PySpecs(other) + if self == other: + return self + new_specset = SpecifierSet() + diff_specset = SpecifierSet() + intersection = set(self.as_specset) & set(other.as_specset) + diff_specset._specs = frozenset(set(self.as_specset) ^ set(other.as_specset)) + tuples = cleanup_pyspecs(diff_specset, joiner="or") + new_marker_str = " and ".join( + "python_version {0} '{1}'".format(op, val) + for op, val in tuples + ) + specset = set() + if new_marker_str: + marker = Marker(new_marker_str) + specset = set(PySpecs.from_marker(marker).as_specset) + specset = frozenset(specset | intersection) + new_specset._specs = specset + return PySpecs(new_specset) + @lru_cache(maxsize=128) def __or__(self, specset): if not isinstance(specset, PySpecs): @@ -287,9 +339,19 @@ def __or__(self, specset): return new_pyspec @classmethod - @lru_cache(maxsize=128) def from_marker(cls, marker): - if marker._markers[0][0] != 'python_version': + if not marker: + return PySpecs() + if len(marker._markers) > 1: + specs = PySpecs() + markers = sorted([ + el for el in marker._markers + if isinstance(el, tuple) + ], key=lambda x: x[2].value) + for mkr in markers: + specs.add(cls.from_marker(gen_marker(mkr))) + return specs + if marker._markers[0][0].value != 'python_version': return op = marker._markers[0][1].value version = marker._markers[0][2].value @@ -300,13 +362,13 @@ def from_marker(cls, marker): for v in version.split(",") ) elif op == "not in": - versions = version.split(",") + versions = [v.strip() for v in version.split(",")] bad_versions = ["3.0", "3.1", "3.2", "3.3"] if len(versions) >= 2 and any(v in versions for v in bad_versions): versions = bad_versions specset.update( Specifier("!={0}".format(v.strip())) - for v in bad_versions + for v in sorted(bad_versions) ) else: specset.add(Specifier("".join([op, version]))) diff --git a/src/passa/models/metadata.py b/src/passa/models/metadata.py index e626678..27990a8 100644 --- a/src/passa/models/metadata.py +++ b/src/passa/models/metadata.py @@ -4,13 +4,18 @@ import copy import itertools +import operator import packaging.markers import packaging.specifiers import vistir import vistir.misc -from ..internals.markers import get_without_extra +from six.moves import reduce + +from ..internals.markers import ( + get_without_extra, get_without_pyversion, get_contained_pyversions +) from ..internals.specifiers import PySpecs @@ -36,7 +41,7 @@ def __repr__(self): ) def __key(self): - return (tuple(self.markerset), hash(self.pyspecset)) + return (tuple(self.markerset), hash(tuple(self.pyspecset))) def __hash__(self): return hash(self.__key()) @@ -50,27 +55,30 @@ def __len__(self): def __iter__(self): return itertools.chain(self.markerset, self.pyspecset) + def __lt__(self, other): + return operator.lt(self.__key(), other.__key()) + + def __le__(self, other): + return operator.le(self.__key(), other.__key()) + + def __ge__(self, other): + return operator.ge(self.__key(), other.__key()) + + def __gt__(self, other): + return operator.gt(self.__key(), other.__key()) + def __str__(self): - pyspecs = PySpecs() - markerset = set() - for m in self.markerset: - marker_specs = PySpecs.from_marker(packaging.markers.Marker(m)) - if marker_specs: - pyspecs.add(marker_specs) - else: - markerset.add(m) - if pyspecs: - self.pyspecset.add(pyspecs) - self.markerset = frozenset(markerset) + self.markerset = frozenset(filter(None, self.markerset)) return " and ".join(dedup_markers(itertools.chain( # Make sure to always use the same quotes so we can dedup properly. ( "{0}".format(ms) if " or " in ms else ms for ms in (str(m).replace('"', "'") for m in self.markerset) + if ms ), ( - "{0}".format(str(spec)) for spec in self.pyspecset - ), + "{0}".format(str(self.pyspecset)) if self.pyspecset else "", + ) ))) def __bool__(self): @@ -79,20 +87,38 @@ def __bool__(self): def __nonzero__(self): # Python 2. return self.__bool__() - def __or__(self, pair): + @classmethod + def from_tuple(cls, pair): marker, specset = pair - markerset = set(self.markerset) - specset = PySpecs(specset) + pyspecs = PySpecs(specset) + markerset = set() if marker: - marker_specs = PySpecs.from_marker(marker) - if not marker_specs: - markerset.add(str(marker)) - else: - specset.add(marker_specs) + # Returns a PySpec instance or None + marker_pyversions = get_contained_pyversions(marker) + if marker_pyversions: + pyspecs.add(marker_pyversions) + # The remainder of the marker, if there is any + cleaned_marker = get_without_pyversion(marker) + if cleaned_marker: + markerset.add(str(cleaned_marker)) + metaset = cls() + metaset.markerset = frozenset(markerset) + metaset.pyspecset = pyspecs + return metaset + + def __or__(self, other): + if not isinstance(other, type(self)): + other = self.from_tuple(other) metaset = MetaSet() + markerset = set() + specset = PySpecs() + for meta in (self, other): + if meta.markerset: + markerset |= set(meta.markerset) + if meta.pyspecset: + specset = specset | meta.pyspecset metaset.markerset = frozenset(markerset) - # TODO: Implement some logic to clean up dups like '3.0.*' and '3.0'. - metaset.pyspecset = self.pyspecset | specset + metaset.pyspecset = specset return metaset @@ -145,16 +171,17 @@ def _calculate_metasets_mapping(dependencies, pythons, traces): def _format_metasets(metasets): - # If there is an unconditional route, this needs to be unconditional. - if not metasets or not all(metasets): - return None + metasets = dedup_markers(metaset for metaset in metasets if metaset) + # If there is an unconditional route, this needs to be unconditional. + if not metasets: + return "" + combined_metaset = str(MetaSet() | reduce(lambda x, y: x | y, metasets)) + if not combined_metaset: + return "" # This extra str(Marker()) call helps simplify the expression. - _metasets = (dedup_markers(str(metaset) for metaset in metasets if metaset)) - metaset_string = " or ".join(meta for meta in list(_metasets)) - if not metaset_string: - return metaset_string - return str(packaging.markers.Marker(metaset_string)) + metaset_string = str(packaging.markers.Marker(combined_metaset)) + return metaset_string def set_metadata(candidates, traces, dependencies, pythons): From 703c8c5fbc6e2f3620d71e20f50a7c240ceb0f3e Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 26 Sep 2018 10:16:25 -0400 Subject: [PATCH 26/34] update lockfile? Signed-off-by: Dan Ryan --- Pipfile.lock | 148 ++++++++++++++++++------------ src/passa/internals/specifiers.py | 51 +++++----- 2 files changed, 116 insertions(+), 83 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index edbc80a..e1bc109 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -33,20 +33,21 @@ "hashes": [ "sha256:f0b0e4eba956de51238e17573b7087e852dfe9854afd2e9c873f73fc0ca0a6dd" ], + "markers": "python_version < '2.8' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.5" }, "backports-shutil-get-terminal-size": { "hashes": [ "sha256:0975ba55054c15e346944b38956a4c9cbee9009391e41b86c68990effb8c1f64" ], - "markers": "python_version < '3.3' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.3' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.0.0" }, "backports-weakref": { "hashes": [ "sha256:81bc9b51c0abc58edc76aefbbc68c62a787918ffe943a37947e162c3f8e19e82" ], - "markers": "python_version < '3.3' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.3' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.0.post1" }, "cached-property": { @@ -54,13 +55,14 @@ "sha256:3a026f1a54135677e7da5ce819b0c690f156f37976f3e30c5430740725203d7f", "sha256:9217a59f14a5682da7c4b8829deadbfc194ac22e9908ccf7c8820234e80a1504" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.5.1" }, "cerberus": { "hashes": [ "sha256:f5c2e048fb15ecb3c088d192164316093fcfa602a74b3386eefb2983aa7e800a" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.2" }, "certifi": { @@ -68,7 +70,7 @@ "sha256:376690d6f16d32f9d1fe8932551d80b23e9d393a8578c5633a2ed39a64861638", "sha256:456048c7e371c089d0a77a5212fb37a2c2dce1e24146e3b7e0261736aaeaa22a" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2018.8.24" }, "chardet": { @@ -76,7 +78,7 @@ "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==3.0.4" }, "distlib": { @@ -93,7 +95,7 @@ "sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79", "sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1" ], - "markers": "python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '2.8' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.6" }, "first": { @@ -101,6 +103,7 @@ "sha256:3bb3de3582cb27071cfb514f00ed784dc444b7f96dc21e140de65fe00585c95e", "sha256:41d5b64e70507d0c3ca742d68010a76060eea8a3d863e9b5130ab11a4a91aa0e" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.0.1" }, "idna": { @@ -108,14 +111,14 @@ "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.7" }, "importlib": { "hashes": [ "sha256:b6ee7066fea66e35f8d0acee24d98006de1a0a8a94a8ce6efe73a9a23c8d9826" ], - "markers": "python_version < '2.7' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '2.7' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.0.4" }, "installer": { @@ -123,7 +126,7 @@ "sha256:1ba23de573e9b95a8dcbd04fd026c40a64b77db0aadc48f28a844b4cb87479fe", "sha256:f4f195c9b17ea7d2b631a758451485c6b080975349b4adebe45ef4bb022db069" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.1.1" }, "modutil": { @@ -131,7 +134,7 @@ "sha256:2c85c1666649e92e56de17c00e1e831313602d9b55e8661d39c01e39003b45f7", "sha256:cc3dad264e36ed359fdd67c4588959d2996bd0402ad9c9d974ca906821537218" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '3.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.0.0" }, "mork": { @@ -139,15 +142,15 @@ "sha256:13772edb4724915cf0cfa30d31426e0565487a3b2d7883b8468718eaed8ecfc2", "sha256:b1b41bc31603eef1b50e42e75ae2d74d7a0d9ab46ea4d0dd1ba387a451870873" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.1.4" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.1.2" }, "packagebuilder": { "hashes": [ "sha256:1e85c4e0e994322996b93cd6685c12834d30f3558889154f8e3de8fb1f3fd1e7", "sha256:dc525d06ecd102db23ab421b879d7d27021d784ff933e33e8c411a53af5c9dbe" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.1.0" }, "packaging": { @@ -155,7 +158,7 @@ "sha256:0886227f54515e592aaa2e5a553332c73962917f2831f1b0f9b9f4380a4b9807", "sha256:f95a1e147590f204328170981833854229bb2912ac3d5f89e2a8ccd2834800c9" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==18.0" }, "passa": { @@ -168,7 +171,7 @@ "sha256:8eb170f8d0d61825e09a95b38be068299ddeda82f35e96c3301a8a5e7604cb83", "sha256:d1aa2a11ba7b8f7b21ab852b1fb5afb277e1bb99d5dfc663380b5015c0d80c5a" ], - "markers": "python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.3.2" }, "pip-shims": { @@ -176,7 +179,7 @@ "sha256:9c8a568b4a8ce4000a2982224f48a35736fca81214dfdb30dcae24287866a7e4", "sha256:ebc2bb29ddd21fa00c0cf28a5d8c725100f2f7ee98703aba237efd02e205c1c1" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.1.2" }, "plette": { @@ -187,7 +190,7 @@ "sha256:c0e3553c1e581d8423daccbd825789c6e7f29b7d9e00e5331b12e1642a1a26d3", "sha256:dde5d525cf5f0cbad4d938c83b93db17887918daf63c13eafed257c4f61b07b4" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.2.2" }, "pyparsing": { @@ -195,7 +198,7 @@ "sha256:bc6c7146b91af3f567cf6daeaec360bc07d45ffec4cf5353f4d7a208ce7ca30a", "sha256:d29593d8ebe7b57d6967b62494f8c72b03ac0262b1eed63826c6f788b3606401" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.2.2" }, "recursive-monkey-patch": { @@ -211,7 +214,7 @@ "sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1", "sha256:ec22d826a36ed72a7358ff3fe56cbd4ba69dd7a6718ffd450ff0e9df7a47ce6a" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.19.1" }, "requirementslib": { @@ -219,6 +222,7 @@ "sha256:90151d8963f814e17190e067b60e92fb35fd1bc46c99f8dba3d7b0d93a3dd958", "sha256:c3aeaa4e0b80843ba65a68878293e07ea52a8d0706dbba86b02dad6cd20ef2dd" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.6" }, "resolvelib": { @@ -226,6 +230,7 @@ "sha256:6c4c6690b0bdd78bcc002e1a5d1b6abbde58c694a6ea1838f165b20d2c943db7", "sha256:8734e53271ef98f38a2c99324d5e7905bc00c97dc3fc5bb7d83c82a979e71c04" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.2.2" }, "scandir": { @@ -242,7 +247,7 @@ "sha256:c9009c527929f6e25604aec39b0a43c3f831d2947d89d6caaab22f057b7055c8", "sha256:f5c71e29b4e2af7ccdc03a020c626ede51da471173b4a6ad1e904f2b2e04b4bd" ], - "markers": "python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.9.0" }, "six": { @@ -250,6 +255,7 @@ "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" ], + "markers": "python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.11.0" }, "toml": { @@ -257,6 +263,7 @@ "sha256:380178cde50a6a79f9d2cf6f42a62a5174febe5eea4126fe4038785f1d888d42", "sha256:a7901919d3e4f92ffba7ff40a9d697e35bbbc8a8049fe8da742f34c83606d957" ], + "markers": "python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.9.6" }, "tomlkit": { @@ -264,7 +271,7 @@ "sha256:8ab16e93162fc44d3ad83d2aa29a7140b8f7d996ae1790a73b9a7aed6fb504ac", "sha256:ca181cee7aee805d455628f7c94eb8ae814763769a93e69157f250fe4ebe1926" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.4.4" }, "typing": { @@ -273,7 +280,7 @@ "sha256:57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4", "sha256:a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a" ], - "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==3.6.6" }, "urllib3": { @@ -281,7 +288,7 @@ "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf", "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5" ], - "markers": "python_version < '4' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '4' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.23" }, "virtualenv": { @@ -289,7 +296,7 @@ "sha256:2ce32cd126117ce2c539f0134eb89de91a8413a29baac49cbab3eb50e2026669", "sha256:ca07b4c0b54e14a91af9f34d0919790b016923d157afda5efdde55c96718f752" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==16.0.0" }, "vistir": { @@ -300,7 +307,7 @@ "sha256:8a360ac20cbcc0863d6dbbe7a52e8b2c9ebf48abd6833c3813a82c70708244af", "sha256:bc6e10284792485c10585536e6aede9e38996c841cc9d2a67238cd05742c2d0b" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.1.6" }, "wheel": { @@ -308,7 +315,7 @@ "sha256:3970f4130b7f8bf8167ca09215c8cc2f49a87c3d46506adfc60cb08c47ab0949", "sha256:a26bc27230baaec9039972b7cb43db94b17c13e4d66a9ff6a4d46a0344c55c9a" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.32.0" }, "yaspin": { @@ -316,7 +323,7 @@ "sha256:36fdccc5e0637b5baa8892fe2c3d927782df7d504e9020f40eb2c1502518aa5a", "sha256:8e52bf8079a48e2a53f3dfeec9e04addb900c101d1591c85df69cf677d3237e7" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.14.0" } }, @@ -334,6 +341,7 @@ "sha256:37228cda29411948b422fae072f57e31d3396d2ee1c9783775980ee9c9990af6", "sha256:58587dd4dc3daefad0487f6d9ae32b4542b185e1c36db6993290e7c41ca2b47c" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.5" }, "appdirs": { @@ -341,6 +349,7 @@ "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92", "sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e" ], + "markers": "python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.4.3" }, "arpeggio": { @@ -356,6 +365,7 @@ "sha256:0312ad34fcad8fac3704d441f7b317e50af620823353ec657a53e981f92920c0", "sha256:ec9ae8adaae229e4f8446952d204a3e4b5fdd2d099f9be3aaf556120135fb3ee" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.2.1" }, "attrs": { @@ -363,7 +373,7 @@ "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '3.6'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==18.2.0" }, "babel": { @@ -378,20 +388,21 @@ "hashes": [ "sha256:f0b0e4eba956de51238e17573b7087e852dfe9854afd2e9c873f73fc0ca0a6dd" ], + "markers": "python_version < '2.8' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.5" }, "backports-shutil-get-terminal-size": { "hashes": [ "sha256:0975ba55054c15e346944b38956a4c9cbee9009391e41b86c68990effb8c1f64" ], - "markers": "python_version < '3.3' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.3' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.0.0" }, "backports-weakref": { "hashes": [ "sha256:81bc9b51c0abc58edc76aefbbc68c62a787918ffe943a37947e162c3f8e19e82" ], - "markers": "python_version < '3.3' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.3' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.0.post1" }, "black": { @@ -410,18 +421,26 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.1.4" }, + "bleach": { + "hashes": [ + "sha256:0ee95f6167129859c5dce9b1ca291ebdb5d8cd7e382ca0e237dfd0dad63f63d8", + "sha256:24754b9a7d530bf30ce7cbc805bc6cce785660b4a10ff3a43633728438c105ab" + ], + "version": "==2.1.4" + }, "cached-property": { "hashes": [ "sha256:3a026f1a54135677e7da5ce819b0c690f156f37976f3e30c5430740725203d7f", "sha256:9217a59f14a5682da7c4b8829deadbfc194ac22e9908ccf7c8820234e80a1504" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.5.1" }, "cerberus": { "hashes": [ "sha256:f5c2e048fb15ecb3c088d192164316093fcfa602a74b3386eefb2983aa7e800a" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.2" }, "certifi": { @@ -429,7 +448,7 @@ "sha256:376690d6f16d32f9d1fe8932551d80b23e9d393a8578c5633a2ed39a64861638", "sha256:456048c7e371c089d0a77a5212fb37a2c2dce1e24146e3b7e0261736aaeaa22a" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2018.8.24" }, "cffi": { @@ -474,7 +493,7 @@ "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==3.0.4" }, "click": { @@ -482,7 +501,7 @@ "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7" ], - "markers": "python_version >= '2.7' and python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7'", "version": "==7.0" }, "cmarkgfm": { @@ -523,7 +542,7 @@ "sha256:463f8483208e921368c9f306094eb6f725c6ca42b0f97e313cb5d5512459feda", "sha256:48eb22f4f8461b1df5734a074b57042430fb06e1d61bd1e11b078c0fe6d7a1f1" ], - "markers": "sys_platform == 'win32'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and sys_platform == 'win32'", "version": "==0.3.9" }, "coverage": { @@ -560,6 +579,7 @@ "sha256:e05cb4d9aad6233d67e0541caa7e511fa4047ed7750ec2510d466e806e0255d6", "sha256:f3f501f345f24383c0000395b26b726e46758b71393267aeae0bd36f8b3ade80" ], + "markers": "python_version < '4' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==4.5.1" }, "distlib": { @@ -584,7 +604,7 @@ "sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79", "sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1" ], - "markers": "python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '2.8' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.6" }, "execnet": { @@ -592,6 +612,7 @@ "sha256:a7a84d5fa07a089186a329528f127c9d73b9de57f1a1131b82bb5320ee651f6a", "sha256:fc155a6b553c66c838d1a22dba1dc9f5f505c43285a878c6f74a79c024750b83" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.5.0" }, "first": { @@ -599,6 +620,7 @@ "sha256:3bb3de3582cb27071cfb514f00ed784dc444b7f96dc21e140de65fe00585c95e", "sha256:41d5b64e70507d0c3ca742d68010a76060eea8a3d863e9b5130ab11a4a91aa0e" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.0.1" }, "funcsigs": { @@ -606,7 +628,7 @@ "sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca", "sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50" ], - "markers": "python_version < '3.0'", + "markers": "python_version < '3.0' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.0.2" }, "future": { @@ -628,7 +650,7 @@ "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.7" }, "imagesize": { @@ -643,7 +665,7 @@ "hashes": [ "sha256:b6ee7066fea66e35f8d0acee24d98006de1a0a8a94a8ce6efe73a9a23c8d9826" ], - "markers": "python_version < '2.7' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '2.7' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.0.4" }, "incremental": { @@ -658,7 +680,7 @@ "sha256:1ba23de573e9b95a8dcbd04fd026c40a64b77db0aadc48f28a844b4cb87479fe", "sha256:f4f195c9b17ea7d2b631a758451485c6b080975349b4adebe45ef4bb022db069" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.1.1" }, "invoke": { @@ -687,7 +709,7 @@ "sha256:2c85c1666649e92e56de17c00e1e831313602d9b55e8661d39c01e39003b45f7", "sha256:cc3dad264e36ed359fdd67c4588959d2996bd0402ad9c9d974ca906821537218" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '3.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.0.0" }, "more-itertools": { @@ -696,6 +718,7 @@ "sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e", "sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==4.3.0" }, "mork": { @@ -710,7 +733,7 @@ "hashes": [ "sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f" ], - "markers": "python_version < '3.0'", + "markers": "python_version < '3.0' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1" }, "packagebuilder": { @@ -718,7 +741,7 @@ "sha256:1e85c4e0e994322996b93cd6685c12834d30f3558889154f8e3de8fb1f3fd1e7", "sha256:dc525d06ecd102db23ab421b879d7d27021d784ff933e33e8c411a53af5c9dbe" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.1.0" }, "packaging": { @@ -726,7 +749,7 @@ "sha256:0886227f54515e592aaa2e5a553332c73962917f2831f1b0f9b9f4380a4b9807", "sha256:f95a1e147590f204328170981833854229bb2912ac3d5f89e2a8ccd2834800c9" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==18.0" }, "parver": { @@ -749,7 +772,7 @@ "sha256:8eb170f8d0d61825e09a95b38be068299ddeda82f35e96c3301a8a5e7604cb83", "sha256:d1aa2a11ba7b8f7b21ab852b1fb5afb277e1bb99d5dfc663380b5015c0d80c5a" ], - "markers": "python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.3.2" }, "pip-shims": { @@ -757,7 +780,7 @@ "sha256:9c8a568b4a8ce4000a2982224f48a35736fca81214dfdb30dcae24287866a7e4", "sha256:ebc2bb29ddd21fa00c0cf28a5d8c725100f2f7ee98703aba237efd02e205c1c1" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.1.2" }, "pkginfo": { @@ -775,7 +798,7 @@ "sha256:c0e3553c1e581d8423daccbd825789c6e7f29b7d9e00e5331b12e1642a1a26d3", "sha256:dde5d525cf5f0cbad4d938c83b93db17887918daf63c13eafed257c4f61b07b4" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.2.2" }, "pluggy": { @@ -783,6 +806,7 @@ "sha256:6e3836e39f4d36ae72840833db137f7b7d35105079aee6ec4a62d9f80d594dd1", "sha256:95eb8364a4708392bae89035f45341871286a333f749c3141c20573d2b3876e1" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.7.1" }, "py": { @@ -790,6 +814,7 @@ "sha256:06a30435d058473046be836d3fc4f27167fd84c45b99704f2fb5509ef61f9af1", "sha256:50402e9d1c9005d759426988a492e0edaadb7f4e68bcddfea586bc7432d009c6" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.6.0" }, "pycparser": { @@ -811,7 +836,7 @@ "sha256:bc6c7146b91af3f567cf6daeaec360bc07d45ffec4cf5353f4d7a208ce7ca30a", "sha256:d29593d8ebe7b57d6967b62494f8c72b03ac0262b1eed63826c6f788b3606401" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.2.2" }, "pytest": { @@ -819,6 +844,7 @@ "sha256:0a72d8a9f559c006ba153e0c9b4838efd7b656cf1f993747ba7128770d6eb12c", "sha256:95529588ff4e85114a0b0ad8e9cf0131ca47d46b28230e25366c5aba66b1d854" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==3.8.1" }, "pytest-cov": { @@ -826,6 +852,7 @@ "sha256:513c425e931a0344944f84ea47f3956be0e416d95acbd897a44970c8d926d5d7", "sha256:e360f048b7dae3f2f2a9a4d067b2dd6b6a015d384d1577c994a43f3f7cbad762" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.6.0" }, "pytest-forked": { @@ -833,6 +860,7 @@ "sha256:e4500cd0509ec4a26535f7d4112a8cc0f17d3a41c29ffd4eab479d2a55b30805", "sha256:f275cb48a73fc61a6710726348e1da6d68a978f0ec0c54ece5a5fae5977e5a08" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.2" }, "pytest-timeout": { @@ -840,6 +868,7 @@ "sha256:1117fc0536e1638862917efbdc0895e6b62fa61e6cf4f39bb655686af7af9627", "sha256:b050a05da96a9992e90e884bc19b4790678b40c25471d2b77015b388417e1fa8" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.3.2" }, "pytest-xdist": { @@ -870,7 +899,7 @@ "sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1", "sha256:ec22d826a36ed72a7358ff3fe56cbd4ba69dd7a6718ffd450ff0e9df7a47ce6a" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.19.1" }, "requests-toolbelt": { @@ -885,6 +914,7 @@ "sha256:90151d8963f814e17190e067b60e92fb35fd1bc46c99f8dba3d7b0d93a3dd958", "sha256:c3aeaa4e0b80843ba65a68878293e07ea52a8d0706dbba86b02dad6cd20ef2dd" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.6" }, "resolvelib": { @@ -892,6 +922,7 @@ "sha256:6c4c6690b0bdd78bcc002e1a5d1b6abbde58c694a6ea1838f165b20d2c943db7", "sha256:8734e53271ef98f38a2c99324d5e7905bc00c97dc3fc5bb7d83c82a979e71c04" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.2.2" }, "scandir": { @@ -908,7 +939,7 @@ "sha256:c9009c527929f6e25604aec39b0a43c3f831d2947d89d6caaab22f057b7055c8", "sha256:f5c71e29b4e2af7ccdc03a020c626ede51da471173b4a6ad1e904f2b2e04b4bd" ], - "markers": "python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.9.0" }, "six": { @@ -916,6 +947,7 @@ "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" ], + "markers": "python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.11.0" }, "snowballstemmer": { @@ -954,6 +986,7 @@ "sha256:380178cde50a6a79f9d2cf6f42a62a5174febe5eea4126fe4038785f1d888d42", "sha256:a7901919d3e4f92ffba7ff40a9d697e35bbbc8a8049fe8da742f34c83606d957" ], + "markers": "python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.9.6" }, "tomlkit": { @@ -961,7 +994,7 @@ "sha256:8ab16e93162fc44d3ad83d2aa29a7140b8f7d996ae1790a73b9a7aed6fb504ac", "sha256:ca181cee7aee805d455628f7c94eb8ae814763769a93e69157f250fe4ebe1926" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.4.4" }, "towncrier": { @@ -992,7 +1025,7 @@ "sha256:57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4", "sha256:a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a" ], - "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==3.6.6" }, "urllib3": { @@ -1000,7 +1033,7 @@ "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf", "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5" ], - "markers": "python_version < '4' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '4' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.23" }, "virtualenv": { @@ -1008,7 +1041,7 @@ "sha256:2ce32cd126117ce2c539f0134eb89de91a8413a29baac49cbab3eb50e2026669", "sha256:ca07b4c0b54e14a91af9f34d0919790b016923d157afda5efdde55c96718f752" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==16.0.0" }, "vistir": { @@ -1019,7 +1052,7 @@ "sha256:8a360ac20cbcc0863d6dbbe7a52e8b2c9ebf48abd6833c3813a82c70708244af", "sha256:bc6e10284792485c10585536e6aede9e38996c841cc9d2a67238cd05742c2d0b" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.1.6" }, "webencodings": { @@ -1027,7 +1060,6 @@ "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78", "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.5.1" }, "wheel": { @@ -1035,7 +1067,7 @@ "sha256:3970f4130b7f8bf8167ca09215c8cc2f49a87c3d46506adfc60cb08c47ab0949", "sha256:a26bc27230baaec9039972b7cb43db94b17c13e4d66a9ff6a4d46a0344c55c9a" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.32.0" }, "yaspin": { @@ -1043,7 +1075,7 @@ "sha256:36fdccc5e0637b5baa8892fe2c3d927782df7d504e9020f40eb2c1502518aa5a", "sha256:8e52bf8079a48e2a53f3dfeec9e04addb900c101d1591c85df69cf677d3237e7" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.14.0" } } diff --git a/src/passa/internals/specifiers.py b/src/passa/internals/specifiers.py index 3f2ae8c..76946cf 100644 --- a/src/passa/internals/specifiers.py +++ b/src/passa/internals/specifiers.py @@ -143,30 +143,6 @@ def cleanup_pyspecs(specs, joiner="or"): return results -@lru_cache(maxsize=128) -def pyspec_from_markers(marker): - if marker._markers[0][0] != 'python_version': - return - op = marker._markers[0][1].value - version = marker._markers[0][2].value - specset = set() - if op == "in": - specset.update( - Specifier("=={0}".format(v.strip())) - for v in version.split(",") - ) - elif op == "not in": - specset.update( - Specifier("!={0}".format(v.strip())) - for v in version.split(",") - ) - else: - specset.add(Specifier("".join([op, version]))) - if specset: - return specset - return None - - def gen_marker(mkr): m = Marker("python_version == '1'") m._markers.pop() @@ -175,6 +151,12 @@ def gen_marker(mkr): class PySpecs(Set): + + MAX_VERSIONS = { + 2: 7, + 3: 9 + } + def __init__(self, specs=None): if not specs: specs = SpecifierSet() @@ -210,6 +192,16 @@ def __iter__(self): yield version return + @classmethod + def get_versions(cls): + major_versions = list(cls.MAX_VERSIONS.keys()) + versions = ( + "{0}.{1}".format(major, minor) for major in major_versions + for minor in range(cls.MAX_VERSIONS[major] + 1) + ) + versions = (packaging.version.parse(v) for v in versions) + return versions + def clean(self): if len(set(self.specifierset)) == 1: spec = next(iter(spec for spec in self.specifierset), None) @@ -305,13 +297,22 @@ def __nonzero__(self): # Python 2. @lru_cache(maxsize=128) def __and__(self, other): + # Unintuitive perhaps, but this is for "x or y" and needs to handle the + # widest possible range encapsulated by the two using the intersection if not isinstance(other, PySpecs): specset = PySpecs(other) if self == other: return self new_specset = SpecifierSet() diff_specset = SpecifierSet() - intersection = set(self.as_specset) & set(other.as_specset) + own_versions = [v for v in self.get_versions() if v in self.specifierset] + other_versions = [v for v in self.get_versions() if v in other.specifierset] + intersection = set(own_versions) & set(other_versions) + min_included = None + max_included = None + if intersection: + max_included = max(intersection) + min_included = min(intersection) diff_specset._specs = frozenset(set(self.as_specset) ^ set(other.as_specset)) tuples = cleanup_pyspecs(diff_specset, joiner="or") new_marker_str = " and ".join( From 85de00e2af950167bf615ceca2faabbe669f8d0c Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 27 Sep 2018 01:13:42 -0400 Subject: [PATCH 27/34] Finalize marker intersection logic Signed-off-by: Dan Ryan --- src/passa/internals/specifiers.py | 174 ++++++++++++++++++++++++------ 1 file changed, 140 insertions(+), 34 deletions(-) diff --git a/src/passa/internals/specifiers.py b/src/passa/internals/specifiers.py index 76946cf..14654e8 100644 --- a/src/passa/internals/specifiers.py +++ b/src/passa/internals/specifiers.py @@ -7,16 +7,16 @@ import six -from cached_property import cached_property from packaging.markers import Marker from packaging.specifiers import Specifier, SpecifierSet +import packaging.version from vistir.misc import dedup - six.add_move(six.MovedAttribute("Set", "collections", "collections.abc")) from six.moves import reduce, Set + try: from functools import lru_cache except ImportError: @@ -55,6 +55,13 @@ def _format_pyspec(specifier): next_tuple = (curr_tuple[0], curr_tuple[1] + 1) except IndexError: next_tuple = (curr_tuple[0], 1) + if not next_tuple[1] <= PySpecs.MAX_VERSIONS[next_tuple[0]]: + if (specifier.operator == "<" + and next_tuple[1] - 1 <= PySpecs.MAX_VERSIONS[next_tuple[0]]): + op = "<=" + next_tuple = (next_tuple[0], next_tuple[1] - 1) + else: + return specifier specifier = Specifier("{0}{1}".format(op, _format_version(next_tuple))) return specifier @@ -143,6 +150,49 @@ def cleanup_pyspecs(specs, joiner="or"): return results +def fix_version_tuple(version_tuple): + op, version = version_tuple + max_allowed = PySpecs.MAX_VERSIONS[version[0]] + if op == "<" and version[1] > max_allowed and version[1] - 1 <= max_allowed: + op = "<=" + version = (version[0], version[1] - 1) + return (op, version) + + +@lru_cache(maxsize=128) +def get_versions(specset, group_by_operator=True): + specs = [_get_specs(x) for x in list(tuple(specset))] + initial_sort_key = lambda k: (k[0], k[1]) + initial_grouping_key = operator.itemgetter(0) + if not group_by_operator: + initial_grouping_key = operator.itemgetter(1) + initial_sort_key = operator.itemgetter(1) + version_tuples = sorted( + set((op, version) for spec in specs for op, version in spec), + key=initial_sort_key + ) + version_tuples = [fix_version_tuple(t) for t in version_tuples] + op_groups = [ + (grp, list(map(operator.itemgetter(1), keys))) + for grp, keys in itertools.groupby(version_tuples, key=initial_grouping_key) + ] + versions = [ + (op, packaging.version.parse(".".join(str(v) for v in val))) + for op, vals in op_groups for val in vals + ] + return versions + + +def get_next_python(version): + version_index = ALL_PYTHON_VERSIONS.index(version) + return ALL_PYTHON_VERSIONS[version_index + 1] + + +def get_previous_python(version): + version_index = ALL_PYTHON_VERSIONS.index(version) + return ALL_PYTHON_VERSIONS[version_index - 1] + + def gen_marker(mkr): m = Marker("python_version == '1'") m._markers.pop() @@ -192,16 +242,6 @@ def __iter__(self): yield version return - @classmethod - def get_versions(cls): - major_versions = list(cls.MAX_VERSIONS.keys()) - versions = ( - "{0}.{1}".format(major, minor) for major in major_versions - for minor in range(cls.MAX_VERSIONS[major] + 1) - ) - versions = (packaging.version.parse(v) for v in versions) - return versions - def clean(self): if len(set(self.specifierset)) == 1: spec = next(iter(spec for spec in self.specifierset), None) @@ -295,36 +335,89 @@ def __bool__(self): def __nonzero__(self): # Python 2. return self.__bool__() + def get_versions(self, group_by_operator=True): + return get_versions(self.specifierset, group_by_operator=group_by_operator) + + def get_versions_in_specset(self): + return set([v[1] for v in self.get_versions() if v[1] in self.specifierset]) + + def get_version_excludes(self): + return set(v[1] for v in self.get_versions() if v[1] not in self.specifierset) + + def group_specs(self, specs=None, handle_exclusions=True): + if not specs: + specs = self.get_versions(group_by_operator=handle_exclusions) + else: + specs = get_versions(specs, group_by_operator=handle_exclusions) + pyversions = enumerate(self.get_versions(group_by_operator=handle_exclusions)) + + def get_version(v): + return ALL_PYTHON_VERSIONS.index(v[1][1]) + + excludes = set() + ranges = set() + + # group the versions on their index from ALL_PYTHON_VERSIONS - their index here + # consecutive elements will share a group, e.g. + # ALL_PYTHON_VERSIONS.index(parse_version("2.7")) == 7, 3.0 == 8, 3.1 == 9 + # if 2.7 is element 1, (7 - 1) = 6, if 3.0 is element 2, (8 - 2) = 6 + # and they will share a group (i.e. they are consecutive) + for k, grp in itertools.groupby(pyversions, lambda t: get_version(t) - t[0]): + version_group = list(grp) + op = next(iter(v[1][0] for v in version_group), None) + _versions = [v[1][1] for v in version_group] + if op == "!=": + excludes.update(set(_versions)) + else: + min_ = min(_versions) + max_ = max(_versions) + if len(_versions) == 1 or str(min_) == str(max_): + ranges.add((min_,)) + else: + ranges.add((min_, max_)) + return ranges, excludes + + def create_specset_from_ranges(self, specset=None, ranges=None, excludes=None): + group_args = {"handle_exclusions": False} + if specset: + group_args["specs"] = specset + if not ranges: + ranges, _ = self.group_specs(**group_args) + if not excludes: + group_args["handle_exclusions"] = True + _, excludes = self.group_specs(**group_args) + spec_ranges = set() + for range_ in ranges: + if len(range_) == 1: + spec_ranges.add(Specifier("=={0}".format(str(next(iter(range_)))))) + else: + min_, max_ = range_ + if min_ == max_: + spec_ranges.add("<={0}").format(min_) + else: + spec_ranges.add(Specifier(">={0}".format(str(min_)))) + spec_ranges.add(Specifier("<={0}".format(str(max_)))) + for exclude in excludes: + spec_ranges.add(Specifier("!={0}".format(str(exclude)))) + new_specset = SpecifierSet() + new_specset._specs = frozenset(spec_ranges) + @lru_cache(maxsize=128) def __and__(self, other): # Unintuitive perhaps, but this is for "x or y" and needs to handle the # widest possible range encapsulated by the two using the intersection if not isinstance(other, PySpecs): - specset = PySpecs(other) + other = PySpecs(other) if self == other: return self new_specset = SpecifierSet() - diff_specset = SpecifierSet() - own_versions = [v for v in self.get_versions() if v in self.specifierset] - other_versions = [v for v in self.get_versions() if v in other.specifierset] - intersection = set(own_versions) & set(other_versions) - min_included = None - max_included = None - if intersection: - max_included = max(intersection) - min_included = min(intersection) - diff_specset._specs = frozenset(set(self.as_specset) ^ set(other.as_specset)) - tuples = cleanup_pyspecs(diff_specset, joiner="or") - new_marker_str = " and ".join( - "python_version {0} '{1}'".format(op, val) - for op, val in tuples - ) - specset = set() - if new_marker_str: - marker = Marker(new_marker_str) - specset = set(PySpecs.from_marker(marker).as_specset) - specset = frozenset(specset | intersection) - new_specset._specs = specset + own_ranges, own_excludes = self.group_specs() + other_ranges, other_excludes = other.group_specs() + # In order to do an "or" propertly we need to intersect the "good" versions + intersection = self.get_versions_in_specset() & other.get_versions_in_specset() + # And then we need to union the "bad" versions + excludes = own_excludes | other_excludes + new_specset = self.create_specset_from_ranges(ranges=intersection, excludes=excludes) return PySpecs(new_specset) @lru_cache(maxsize=128) @@ -379,3 +472,16 @@ def from_marker(cls, marker): newset = cls(specifierset) return newset return None + + +def get_all_python_versions(): + major_versions = list(PySpecs.MAX_VERSIONS.keys()) + versions = ( + "{0}.{1}".format(major, minor) for major in major_versions + for minor in range(PySpecs.MAX_VERSIONS[major] + 1) + ) + versions = (packaging.version.parse(v) for v in versions) + return versions + + +ALL_PYTHON_VERSIONS = sorted(get_all_python_versions()) From 2c7f670858f3e08d5cb6071dd121bb2edf4db9c2 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 2 Oct 2018 19:12:12 -0400 Subject: [PATCH 28/34] Clean up PySpec intersection methods Signed-off-by: Dan Ryan --- Pipfile.lock | 65 +++++++++++------------------ setup.cfg | 1 + src/passa/internals/specifiers.py | 69 ++++++++++++++++++++----------- src/passa/models/metadata.py | 6 +-- 4 files changed, 72 insertions(+), 69 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index e1bc109..4feb79d 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -19,6 +19,7 @@ "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92", "sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e" ], + "markers": "python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.4.3" }, "attrs": { @@ -26,14 +27,14 @@ "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '3.6'", + "markers": "python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==18.2.0" }, "backports-functools-lru-cache": { "hashes": [ "sha256:f0b0e4eba956de51238e17573b7087e852dfe9854afd2e9c873f73fc0ca0a6dd" ], - "markers": "python_version < '2.8' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.5" }, "backports-shutil-get-terminal-size": { @@ -143,7 +144,7 @@ "sha256:b1b41bc31603eef1b50e42e75ae2d74d7a0d9ab46ea4d0dd1ba387a451870873" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.1.2" + "version": "==0.1.4" }, "packagebuilder": { "hashes": [ @@ -163,8 +164,10 @@ }, "passa": { "editable": true, - "path": ".", - "extras": ["virtualenv"] + "extras": [ + "virtualenv" + ], + "path": "." }, "pathlib2": { "hashes": [ @@ -280,7 +283,7 @@ "sha256:57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4", "sha256:a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a" ], - "markers": "python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.5'", "version": "==3.6.6" }, "urllib3": { @@ -333,7 +336,6 @@ "sha256:674bb3bab080f598371f4443c5008cbfeb1a5e622dd312395d2d82af2c54c456", "sha256:b63b1f4dc77c074d386752ec4a8a7517600f6c0db8cd42980cae17ab7b3275d7" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.7.11" }, "apipkg": { @@ -357,7 +359,6 @@ "sha256:a5258b84f76661d558492fa87e42db634df143685a0e51802d59cae7daad8732", "sha256:dc5c0541e7cc2c6033dc0338133436abfac53655624784736e9bc8bd35e56583" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.9.0" }, "atomicwrites": { @@ -373,7 +374,7 @@ "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==18.2.0" }, "babel": { @@ -381,14 +382,13 @@ "sha256:6778d85147d5d85345c14a26aada5e478ab04e39b078b0745ee6870c2b5cf669", "sha256:8cba50f48c529ca3fa18cf81fa9403be176d374ac4d60738b839122dfaaa3d23" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.6.0" }, "backports-functools-lru-cache": { "hashes": [ "sha256:f0b0e4eba956de51238e17573b7087e852dfe9854afd2e9c873f73fc0ca0a6dd" ], - "markers": "python_version < '2.8' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.5" }, "backports-shutil-get-terminal-size": { @@ -413,14 +413,6 @@ "markers": "python_version >= '3.6'", "version": "==18.9b0" }, - "bleach": { - "hashes": [ - "sha256:0ee95f6167129859c5dce9b1ca291ebdb5d8cd7e382ca0e237dfd0dad63f63d8", - "sha256:24754b9a7d530bf30ce7cbc805bc6cce785660b4a10ff3a43633728438c105ab" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.1.4" - }, "bleach": { "hashes": [ "sha256:0ee95f6167129859c5dce9b1ca291ebdb5d8cd7e382ca0e237dfd0dad63f63d8", @@ -501,7 +493,7 @@ "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7" ], - "markers": "python_version >= '2.7'", + "markers": "python_version >= '3.6'", "version": "==7.0" }, "cmarkgfm": { @@ -642,7 +634,6 @@ "sha256:20b159aa3badc9d5ee8f5c647e5efd02ed2a66ab8d354930bd9ff139fc1dc0a3", "sha256:66cb0dcfdbbc4f9c3ba1a63fdb511ffdbd4f513b2b6d81b80cd26ce6b3fb3736" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.0.1" }, "idna": { @@ -658,7 +649,6 @@ "sha256:3f349de3eb99145973fefb7dbe38554414e5c30abd0c8e4b970a7c9d09f3a1d8", "sha256:f3832918bc3c66617f92e35f5d70729187676313caa60c187eb0f28b8fe5e3b5" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.0" }, "importlib": { @@ -721,14 +711,6 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==4.3.0" }, - "mork": { - "hashes": [ - "sha256:13772edb4724915cf0cfa30d31426e0565487a3b2d7883b8468718eaed8ecfc2", - "sha256:b1b41bc31603eef1b50e42e75ae2d74d7a0d9ab46ea4d0dd1ba387a451870873" - ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.1.4" - }, "ordereddict": { "hashes": [ "sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f" @@ -757,7 +739,6 @@ "sha256:ac4afff688d19d5e1876bb68d4bccc1a1b6a5cc8bd6a646939a14d366695ba15", "sha256:f025fba8f88a9c776971df6d62b6cf7f37d1108f84c163bda91e157d7d527075" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.1.1" }, "passa": { @@ -821,7 +802,6 @@ "hashes": [ "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.19" }, "pygments": { @@ -841,11 +821,11 @@ }, "pytest": { "hashes": [ - "sha256:0a72d8a9f559c006ba153e0c9b4838efd7b656cf1f993747ba7128770d6eb12c", - "sha256:95529588ff4e85114a0b0ad8e9cf0131ca47d46b28230e25366c5aba66b1d854" + "sha256:7e258ee50338f4e46957f9e09a0f10fb1c2d05493fa901d113a8dafd0790de4e", + "sha256:9332147e9af2dcf46cd7ceb14d5acadb6564744ddff1fe8c17f0ce60ece7d9a2" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==3.8.1" + "version": "==3.8.2" }, "pytest-cov": { "hashes": [ @@ -884,7 +864,6 @@ "sha256:a061aa0a9e06881eb8b3b2b43f05b9439d6583c206d0a6c340ff72a7b6669053", "sha256:ffb9ef1de172603304d9d2819af6f5ece76f2e85ec10692a524dd876e72bf277" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2018.5" }, "readme-renderer": { @@ -894,6 +873,14 @@ ], "version": "==22.0" }, + "recursive-monkey-patch": { + "hashes": [ + "sha256:546739fea5be2ea9f98b5ec44fafeb697b5cf9fdcda64a03422582ab03ee24c4", + "sha256:98922554e77f2e2c85a4f5d873a0f52efdc1b553f32444bd6c788b2ff583bf3e" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.4.0" + }, "requests": { "hashes": [ "sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1", @@ -955,7 +942,6 @@ "sha256:919f26a68b2c17a7634da993d91339e288964f93c274f1343e3bbbe2096e1128", "sha256:9f3bcd3c401c3e862ec0ebe6d2c069ebc012ce142cce209c098ccb5b09136e89" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.2.1" }, "sphinx": { @@ -963,7 +949,6 @@ "sha256:652eb8c566f18823a022bb4b6dbc868d366df332a11a0226b5bc3a798a479f17", "sha256:d222626d8356de702431e813a05c68a35967e3d66c6cd1c2c89539bb179a7464" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.8.1" }, "sphinx-rtd-theme": { @@ -978,7 +963,6 @@ "sha256:68ca7ff70785cbe1e7bccc71a48b5b6d965d79ca50629606c7861a21b206d9dd", "sha256:9de47f375baf1ea07cdb3436ff39d7a9c76042c10a769c52353ec46e4e8fc3b9" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.0" }, "toml": { @@ -1009,7 +993,6 @@ "sha256:18f1818ce951aeb9ea162ae1098b43f583f7d057b34d706f66939353d1208889", "sha256:df02c0650160986bac0218bb07952245fc6960d23654648b5d5526ad5a4128c9" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1'", "version": "==4.26.0" }, "twine": { @@ -1025,7 +1008,7 @@ "sha256:57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4", "sha256:a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a" ], - "markers": "python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.5'", "version": "==3.6.6" }, "urllib3": { diff --git a/setup.cfg b/setup.cfg index 097fa9b..ce974e4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,6 +37,7 @@ python_requires = >=2.7,!=3.0,!=3.1,!=3.2,!=3.3 setup_requires = setuptools>=36.2.2 install_requires = appdirs + backports-functools-lru-cache;python_version <= "3.4" cached-property distlib installer diff --git a/src/passa/internals/specifiers.py b/src/passa/internals/specifiers.py index 14654e8..907f250 100644 --- a/src/passa/internals/specifiers.py +++ b/src/passa/internals/specifiers.py @@ -183,16 +183,6 @@ def get_versions(specset, group_by_operator=True): return versions -def get_next_python(version): - version_index = ALL_PYTHON_VERSIONS.index(version) - return ALL_PYTHON_VERSIONS[version_index + 1] - - -def get_previous_python(version): - version_index = ALL_PYTHON_VERSIONS.index(version) - return ALL_PYTHON_VERSIONS[version_index - 1] - - def gen_marker(mkr): m = Marker("python_version == '1'") m._markers.pop() @@ -342,14 +332,24 @@ def get_versions_in_specset(self): return set([v[1] for v in self.get_versions() if v[1] in self.specifierset]) def get_version_excludes(self): - return set(v[1] for v in self.get_versions() if v[1] not in self.specifierset) + return set([v[1] for v in self.get_versions() if v[1] not in self.specifierset]) + + def get_version_includes(self): + return set([v for v in ALL_PYTHON_VERSIONS if v in self.specifierset]) + + def get_specset_from_versions(self, versions, include=True): + _specset = SpecifierSet() + op = "==" if include else "!=" + specs = set([Specifier("{0}{1}".format(op, v)) for v in versions]) + _specset._specs = frozenset(specs) + return _specset def group_specs(self, specs=None, handle_exclusions=True): if not specs: specs = self.get_versions(group_by_operator=handle_exclusions) else: specs = get_versions(specs, group_by_operator=handle_exclusions) - pyversions = enumerate(self.get_versions(group_by_operator=handle_exclusions)) + pyversions = enumerate(specs) def get_version(v): return ALL_PYTHON_VERSIONS.index(v[1][1]) @@ -378,7 +378,23 @@ def get_version(v): return ranges, excludes def create_specset_from_ranges(self, specset=None, ranges=None, excludes=None): + """This method takes a specifier set and simplifies it down to some range sets. + + The goal is to consume a list of matching individual version specifiers in + "==" notation (accompanied by a set of excluded versions, that is a set() of + Version objects) and produce a SpecifierSet with a min and max range and the + appropriate excludes (i.e. the simplified set). + + :param specset: A specifierset with the enumerated versions + :param ranges: The ranges to use as inputs (or the specset will be generated from it) + :param excludes: A set of Version objects to exclude in the specifierset + :return: A specifierset with the desired ranges + """ + group_args = {"handle_exclusions": False} + if ranges and not specset and isinstance(ranges, SpecifierSet): + group_args["specs"] = ranges + ranges, _ = self.group_specs(**group_args) if specset: group_args["specs"] = specset if not ranges: @@ -387,20 +403,22 @@ def create_specset_from_ranges(self, specset=None, ranges=None, excludes=None): group_args["handle_exclusions"] = True _, excludes = self.group_specs(**group_args) spec_ranges = set() - for range_ in ranges: - if len(range_) == 1: - spec_ranges.add(Specifier("=={0}".format(str(next(iter(range_)))))) - else: - min_, max_ = range_ - if min_ == max_: - spec_ranges.add("<={0}").format(min_) - else: - spec_ranges.add(Specifier(">={0}".format(str(min_)))) - spec_ranges.add(Specifier("<={0}".format(str(max_)))) + if len(ranges) == 1: + spec_ranges.add(Specifier("=={0}".format(str(next(iter(ranges[0])))))) + else: + min_version = min([r[0] for r in ranges]) + rhs_versions = [ + r[1] for r in ranges if isinstance(r, tuple) and len(r) > 1 + ] + max_version = max(rhs_versions) if rhs_versions else None + spec_ranges.add(Specifier(">={0}".format(str(min_version)))) + if max_version and max_version != ALL_PYTHON_VERSIONS[-1]: + spec_ranges.add(Specifier("<={0}".format(str(max_version)))) for exclude in excludes: spec_ranges.add(Specifier("!={0}".format(str(exclude)))) new_specset = SpecifierSet() new_specset._specs = frozenset(spec_ranges) + return new_specset @lru_cache(maxsize=128) def __and__(self, other): @@ -414,10 +432,11 @@ def __and__(self, other): own_ranges, own_excludes = self.group_specs() other_ranges, other_excludes = other.group_specs() # In order to do an "or" propertly we need to intersect the "good" versions - intersection = self.get_versions_in_specset() & other.get_versions_in_specset() + intersection = self.get_version_includes() | other.get_version_includes() + intersection_specset = self.get_specset_from_versions(intersection) # And then we need to union the "bad" versions - excludes = own_excludes | other_excludes - new_specset = self.create_specset_from_ranges(ranges=intersection, excludes=excludes) + excludes = self.get_version_excludes() | other.get_version_excludes() + new_specset = self.create_specset_from_ranges(ranges=intersection_specset, excludes=excludes) return PySpecs(new_specset) @lru_cache(maxsize=128) diff --git a/src/passa/models/metadata.py b/src/passa/models/metadata.py index 27990a8..e3cde70 100644 --- a/src/passa/models/metadata.py +++ b/src/passa/models/metadata.py @@ -69,7 +69,7 @@ def __gt__(self, other): def __str__(self): self.markerset = frozenset(filter(None, self.markerset)) - return " and ".join(dedup_markers(itertools.chain( + return " and ".join([mkr_part for mkr_part in dedup_markers(itertools.chain( # Make sure to always use the same quotes so we can dedup properly. ( "{0}".format(ms) if " or " in ms else ms @@ -78,8 +78,8 @@ def __str__(self): ), ( "{0}".format(str(self.pyspecset)) if self.pyspecset else "", - ) - ))) + ))) if mkr_part + ]) def __bool__(self): return bool(self.markerset or self.pyspecset) From 3c9a86c066cc1c3f2ac719a5cfc58e5f3321944c Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 2 Oct 2018 21:41:29 -0400 Subject: [PATCH 29/34] Force upgrades Signed-off-by: Dan Ryan --- .travis.yml | 10 +++++----- tox.ini | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3acd3a4..b13fa9d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,8 +7,8 @@ matrix: fast_finish: true install: - - "python -m pip install --upgrade pip setuptools pytest-timeout" - - "python -m pip install --upgrade -e .[tests,virtualenv]" + - "python -m pip install --upgrade --upgrade-strategy=eager pip setuptools pytest pytest-timeout pytest-cov pytest-xdist" + - "python -m pip install --upgrade --upgrade-strategy=eager -e .[tests,virtualenv]" script: - "python -m pytest -v -n 8 tests/" @@ -31,14 +31,14 @@ jobs: - stage: packing python: "3.6" install: - - "python -m pip install --upgrade -e .[pack]" + - "python -m pip install --upgrade --upgrade-strategy=eager -e .[pack]" script: - "invoke pack" - "python2.7 pack/passa.zip --help" - stage: coverage python: "3.6" install: - - "python -m pip install --upgrade pip setuptools pytest-timeout pytest-cov pytest-xdist" - - "python -m pip install --upgrade -e .[tests,virtualenv]" + - "python -m pip install --upgrade --upgrade-strategy=eager pip setuptools pytest-timeout pytest-cov pytest-xdist" + - "python -m pip install --upgrade --upgrade-strategy=eager -e .[tests,virtualenv]" script: - "pytest -n auto --timeout 300 --cov=passa --cov-report=term-missing --cov-report=xml --cov-report=html tests" diff --git a/tox.ini b/tox.ini index 6428cef..64e1959 100644 --- a/tox.ini +++ b/tox.ini @@ -14,7 +14,7 @@ deps = pytest-sugar -e .[tests,virtualenv] commands = coverage run --parallel -m pytest --timeout 300 [] -install_command = python -m pip install --upgrade {opts} {packages} +install_command = python -m pip install --upgrade --upgrade-strategy=eager {opts} {packages} usedevelop = True [testenv:coverage-report] From d26f4855194f570f7eaedfa47988e7a3c9c4e1cb Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 2 Oct 2018 22:48:56 -0400 Subject: [PATCH 30/34] Add assertion for debugging travis Signed-off-by: Dan Ryan --- tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/conftest.py b/tests/conftest.py index 2726762..7c32c48 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -34,6 +34,7 @@ def working_set_extension(): while requirements: req = requirements.popleft() dist = pkg_resources.working_set.find(req) + assert dist, req dists.add(dist) requirements.extend(dist.requires()) return dists From 4d72b7aace05c11599548f9f2609691e4cc1f6aa Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 2 Oct 2018 22:55:50 -0400 Subject: [PATCH 31/34] Fix lru cache install Signed-off-by: Dan Ryan --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index ce974e4..51bc08d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,7 @@ python_requires = >=2.7,!=3.0,!=3.1,!=3.2,!=3.3 setup_requires = setuptools>=36.2.2 install_requires = appdirs - backports-functools-lru-cache;python_version <= "3.4" + backports.functools_lru_cache; python_version <= "3.4" cached-property distlib installer From 18e650192242e2a862d671fcc9b4c08862bf8771 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 3 Oct 2018 22:11:41 -0400 Subject: [PATCH 32/34] Fix specifier math Signed-off-by: Dan Ryan --- Pipfile | 1 - Pipfile.lock | 23 ++++++++--------------- src/passa/internals/markers.py | 2 ++ src/passa/internals/specifiers.py | 21 ++++++++++++--------- src/passa/models/projects.py | 5 +++-- 5 files changed, 25 insertions(+), 27 deletions(-) diff --git a/Pipfile b/Pipfile index 51464ab..7f3e67a 100644 --- a/Pipfile +++ b/Pipfile @@ -18,7 +18,6 @@ passa-remove = 'python -m passa.cli.remove' passa-upgrade = 'python -m passa.cli.upgrade' passa-lock = 'python -m passa.cli.lock' passa-freeze = 'python -m passa.cli.freeze' - black = 'black src/passa/ --exclude "/(\.git|\.hg|\.mypy_cache|\.tox|\.venv|_build|buck-out|build|dist)/"' build = 'inv build' changelog = 'towncrier' diff --git a/Pipfile.lock b/Pipfile.lock index 4feb79d..c540bfb 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -283,7 +283,7 @@ "sha256:57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4", "sha256:a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a" ], - "markers": "python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.5'", + "markers": "python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==3.6.6" }, "urllib3": { @@ -333,10 +333,10 @@ "develop": { "alabaster": { "hashes": [ - "sha256:674bb3bab080f598371f4443c5008cbfeb1a5e622dd312395d2d82af2c54c456", - "sha256:b63b1f4dc77c074d386752ec4a8a7517600f6c0db8cd42980cae17ab7b3275d7" + "sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359", + "sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02" ], - "version": "==0.7.11" + "version": "==0.7.12" }, "apipkg": { "hashes": [ @@ -415,10 +415,10 @@ }, "bleach": { "hashes": [ - "sha256:0ee95f6167129859c5dce9b1ca291ebdb5d8cd7e382ca0e237dfd0dad63f63d8", - "sha256:24754b9a7d530bf30ce7cbc805bc6cce785660b4a10ff3a43633728438c105ab" + "sha256:9c471c0dd9c820f6bf4ee5ca3e348ceccefbc1475d9a40c397ed5d04e0b42c54", + "sha256:b407b2612b37e6cdc6704f84cec18c1f140b78e6c625652a844e89d6b9855f6b" ], - "version": "==2.1.4" + "version": "==3.0.0" }, "cached-property": { "hashes": [ @@ -629,13 +629,6 @@ ], "version": "==0.16.0" }, - "html5lib": { - "hashes": [ - "sha256:20b159aa3badc9d5ee8f5c647e5efd02ed2a66ab8d354930bd9ff139fc1dc0a3", - "sha256:66cb0dcfdbbc4f9c3ba1a63fdb511ffdbd4f513b2b6d81b80cd26ce6b3fb3736" - ], - "version": "==1.0.1" - }, "idna": { "hashes": [ "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", @@ -1008,7 +1001,7 @@ "sha256:57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4", "sha256:a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a" ], - "markers": "python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.5'", + "markers": "python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==3.6.6" }, "urllib3": { diff --git a/src/passa/internals/markers.py b/src/passa/internals/markers.py index 08b4322..9c1a2fb 100644 --- a/src/passa/internals/markers.py +++ b/src/passa/internals/markers.py @@ -240,6 +240,8 @@ def parse_marker_dict(marker_dict): if not sides: sides = [lhs, rhs] sides = reduce(lambda x, y: x | y, sides) + if not sides: + return sides return PySpecs.from_marker(Marker(str(sides))) # Actually when we "or" things as well we can also just turn them into a reduced # set using this logic now diff --git a/src/passa/internals/specifiers.py b/src/passa/internals/specifiers.py index 907f250..40433b8 100644 --- a/src/passa/internals/specifiers.py +++ b/src/passa/internals/specifiers.py @@ -152,6 +152,9 @@ def cleanup_pyspecs(specs, joiner="or"): def fix_version_tuple(version_tuple): op, version = version_tuple + max_major = max(PySpecs.MAX_VERSIONS.keys()) + if version[0] > max_major: + return (op, (max_major, PySpecs.MAX_VERSIONS[max_major])) max_allowed = PySpecs.MAX_VERSIONS[version[0]] if op == "<" and version[1] > max_allowed and version[1] - 1 <= max_allowed: op = "<=" @@ -332,7 +335,10 @@ def get_versions_in_specset(self): return set([v[1] for v in self.get_versions() if v[1] in self.specifierset]) def get_version_excludes(self): - return set([v[1] for v in self.get_versions() if v[1] not in self.specifierset]) + return set([ + v[1] for v in self.get_versions() + if v[0] == "!=" and v[1] not in self.specifierset + ]) def get_version_includes(self): return set([v for v in ALL_PYTHON_VERSIONS if v in self.specifierset]) @@ -403,7 +409,7 @@ def create_specset_from_ranges(self, specset=None, ranges=None, excludes=None): group_args["handle_exclusions"] = True _, excludes = self.group_specs(**group_args) spec_ranges = set() - if len(ranges) == 1: + if len(ranges) == 1 and not isinstance(next(iter(ranges)), tuple): spec_ranges.add(Specifier("=={0}".format(str(next(iter(ranges[0])))))) else: min_version = min([r[0] for r in ranges]) @@ -429,13 +435,11 @@ def __and__(self, other): if self == other: return self new_specset = SpecifierSet() - own_ranges, own_excludes = self.group_specs() - other_ranges, other_excludes = other.group_specs() # In order to do an "or" propertly we need to intersect the "good" versions intersection = self.get_version_includes() | other.get_version_includes() intersection_specset = self.get_specset_from_versions(intersection) # And then we need to union the "bad" versions - excludes = self.get_version_excludes() | other.get_version_excludes() + excludes = self.get_version_excludes() & other.get_version_excludes() new_specset = self.create_specset_from_ranges(ranges=intersection_specset, excludes=excludes) return PySpecs(new_specset) @@ -445,10 +449,9 @@ def __or__(self, specset): specset = PySpecs(specset) if str(self) == str(specset): return self - combined_set = self.as_set | specset.as_set - new_specset = SpecifierSet() - new_specset._specs = frozenset(combined_set) - new_pyspec = PySpecs(new_specset) + combined_set = self.specifierset & specset.specifierset + # new_specset._specs = frozenset(combined_set) + new_pyspec = PySpecs(combined_set) return new_pyspec @classmethod diff --git a/src/passa/models/projects.py b/src/passa/models/projects.py index 7ff6f31..058a5cc 100644 --- a/src/passa/models/projects.py +++ b/src/passa/models/projects.py @@ -9,11 +9,12 @@ import attr import packaging.markers import packaging.utils -import plette -import plette.models import six import tomlkit +import plette +import plette.models + SectionDifference = collections.namedtuple("SectionDifference", [ "inthis", "inthat", From fc759b3ffb6b500cd0d1aba51daa1f9cc930d9ec Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 8 Apr 2019 00:43:37 -0400 Subject: [PATCH 33/34] Fix mork invocations Signed-off-by: Dan Ryan --- src/passa/cli/options.py | 2 +- src/passa/internals/markers.py | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/passa/cli/options.py b/src/passa/cli/options.py index eafa71d..2b044cb 100644 --- a/src/passa/cli/options.py +++ b/src/passa/cli/options.py @@ -27,7 +27,7 @@ def __init__(self, root, *args, **kwargs): ) self.venv = kwargs.pop("venv", self.get_venv(root)) try: - super(Project, self).__init__(root.as_posix(), env_prefix=self.venv.venv_dir, + super(Project, self).__init__(root.as_posix(), env_prefix=self.venv.prefix, *args, **kwargs) except tomlkit.exceptions.ParseError as e: raise argparse.ArgumentError( diff --git a/src/passa/internals/markers.py b/src/passa/internals/markers.py index 9c1a2fb..3b2ce0e 100644 --- a/src/passa/internals/markers.py +++ b/src/passa/internals/markers.py @@ -114,17 +114,14 @@ def _markers_collect_pyversions(markers, collection): local_collection = [] marker_format_str = "{0}" for i, el in enumerate(reversed(markers)): - if (isinstance(el, tuple) and - el[0].value == "python_version"): + if isinstance(el, tuple) and el[0].value == "python_version": new_marker = str(gen_marker(el)) local_collection.append(marker_format_str.format(new_marker)) - elif isinstance(el, six.string_types): - local_collection.append(el) elif isinstance(el, list): _markers_collect_pyversions(el, local_collection) if local_collection: - local_collection = "{0}".format(" ".join(local_collection)) - collection.append(local_collection) + # local_collection = "{0}".format(" ".join(local_collection)) + collection.extend(local_collection) @lru_cache(maxsize=128) From 802474118c67c7b9cfd01b49acaea75a37b10ac7 Mon Sep 17 00:00:00 2001 From: frostming Date: Mon, 30 Sep 2019 09:36:59 +0800 Subject: [PATCH 34/34] merge branch 'add-tests-frost' --- .editorconfig | 5 +- .github/workflows/ci.yml | 71 ++ .gitignore | 4 +- .gitmodules | 3 + .travis.yml | 44 - MANIFEST.in | 1 + Pipfile | 11 +- Pipfile.lock | 964 ++++++++++-------- appveyor.yml | 22 - news/66.bugfix.rst | 1 + news/66.feature.rst | 6 + setup.cfg | 14 +- src/passa/actions/add.py | 3 +- src/passa/actions/clean.py | 8 +- src/passa/actions/remove.py | 6 +- src/passa/actions/sync.py | 2 +- src/passa/cli/add.py | 3 +- src/passa/cli/install.py | 2 +- src/passa/cli/options.py | 19 +- src/passa/cli/remove.py | 2 +- src/passa/cli/upgrade.py | 2 +- src/passa/internals/_pip.py | 256 +++-- src/passa/internals/_pip_shims.py | 6 +- src/passa/internals/dependencies.py | 39 +- src/passa/internals/markers.py | 8 +- src/passa/internals/specifiers.py | 4 +- src/passa/internals/utils.py | 48 +- src/passa/models/caches.py | 52 +- src/passa/models/environments.py | 452 ++++++++ src/passa/models/lockers.py | 14 +- src/passa/models/metadata.py | 8 +- src/passa/models/projects.py | 12 +- src/passa/models/providers.py | 4 +- src/passa/models/synchronizers.py | 283 +++-- src/passa/operations/sync.py | 10 +- tasks/admin.py | 8 + tests/__init__.py | 5 + tests/actions/test_add.py | 31 +- tests/actions/test_clean.py | 12 +- tests/actions/test_freeze.py | 9 +- tests/actions/test_init.py | 24 +- tests/actions/test_install.py | 76 +- tests/actions/test_lock.py | 107 +- tests/actions/test_remove_and_sync.py | 124 +-- tests/conftest.py | 150 ++- .../git/github.com/testing/demo.git/demo.py | 1 + .../git/github.com/testing/demo.git/setup.py | 14 + .../github.com/testing/no_dep.git/no_dep.py | 1 + .../github.com/testing/no_dep.git/setup.py | 9 + tests/pypi | 1 + tests/pytest-pypi/DESCRIPTION.rst | 5 + tests/pytest-pypi/MANIFEST.in | 4 + tests/pytest-pypi/README.md | 4 + tests/pytest-pypi/pytest_pypi/__init__.py | 14 + tests/pytest-pypi/pytest_pypi/app.py | 227 +++++ tests/pytest-pypi/pytest_pypi/certs.py | 22 + .../pytest-pypi/pytest_pypi/certs/cacert.pem | 63 ++ tests/pytest-pypi/pytest_pypi/certs/cert.pem | 73 ++ tests/pytest-pypi/pytest_pypi/certs/key.pem | 28 + tests/pytest-pypi/pytest_pypi/plugin.py | 43 + tests/pytest-pypi/pytest_pypi/serve.py | 134 +++ .../pytest_pypi/templates/artifact.html | 14 + .../pytest_pypi/templates/artifacts.html | 13 + .../pytest_pypi/templates/package.html | 14 + .../pytest_pypi/templates/package_pypi.html | 4 + .../pytest_pypi/templates/simple.html | 13 + tests/pytest-pypi/pytest_pypi/version.py | 1 + tests/pytest-pypi/runtests.sh | 3 + tests/pytest-pypi/setup.cfg | 5 + tests/pytest-pypi/setup.py | 106 ++ tests/pytest-pypi/tox.ini | 10 + tests/unit/__init__.py | 0 tests/{ => unit}/test_markers.py | 0 tests/{ => unit}/test_specifiers.py | 0 tox.ini | 9 +- 75 files changed, 2675 insertions(+), 1095 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .gitmodules delete mode 100644 .travis.yml delete mode 100644 appveyor.yml create mode 100644 news/66.bugfix.rst create mode 100644 news/66.feature.rst create mode 100644 src/passa/models/environments.py create mode 100644 tests/__init__.py create mode 100644 tests/fixtures/git/github.com/testing/demo.git/demo.py create mode 100644 tests/fixtures/git/github.com/testing/demo.git/setup.py create mode 100644 tests/fixtures/git/github.com/testing/no_dep.git/no_dep.py create mode 100644 tests/fixtures/git/github.com/testing/no_dep.git/setup.py create mode 160000 tests/pypi create mode 100644 tests/pytest-pypi/DESCRIPTION.rst create mode 100644 tests/pytest-pypi/MANIFEST.in create mode 100644 tests/pytest-pypi/README.md create mode 100644 tests/pytest-pypi/pytest_pypi/__init__.py create mode 100644 tests/pytest-pypi/pytest_pypi/app.py create mode 100644 tests/pytest-pypi/pytest_pypi/certs.py create mode 100644 tests/pytest-pypi/pytest_pypi/certs/cacert.pem create mode 100644 tests/pytest-pypi/pytest_pypi/certs/cert.pem create mode 100644 tests/pytest-pypi/pytest_pypi/certs/key.pem create mode 100644 tests/pytest-pypi/pytest_pypi/plugin.py create mode 100644 tests/pytest-pypi/pytest_pypi/serve.py create mode 100644 tests/pytest-pypi/pytest_pypi/templates/artifact.html create mode 100644 tests/pytest-pypi/pytest_pypi/templates/artifacts.html create mode 100644 tests/pytest-pypi/pytest_pypi/templates/package.html create mode 100644 tests/pytest-pypi/pytest_pypi/templates/package_pypi.html create mode 100644 tests/pytest-pypi/pytest_pypi/templates/simple.html create mode 100644 tests/pytest-pypi/pytest_pypi/version.py create mode 100644 tests/pytest-pypi/runtests.sh create mode 100644 tests/pytest-pypi/setup.cfg create mode 100644 tests/pytest-pypi/setup.py create mode 100644 tests/pytest-pypi/tox.ini create mode 100644 tests/unit/__init__.py rename tests/{ => unit}/test_markers.py (100%) rename tests/{ => unit}/test_specifiers.py (100%) diff --git a/.editorconfig b/.editorconfig index c3821e8..fd09030 100644 --- a/.editorconfig +++ b/.editorconfig @@ -11,10 +11,7 @@ insert_final_newline = true [*.md] trim_trailing_whitespace = false -[*.toml] -indent_size = 2 - -[*.yaml] +[*.{toml,yaml,yml}] indent_size = 2 # Makefiles always use tabs for indentation diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..280d827 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,71 @@ +name: Continuous Integration and Deployment + +on: [push] + +jobs: + build: + + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + max-parallel: 6 + matrix: + python-version: [2.7, 3.5, 3.6, 3.7] + os: [ubuntu-latest, macOS-latest, windows-latest] + + steps: + - uses: actions/checkout@v1 + with: + submodules: true + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Setup build and test environment + run: | + python -m pip install --upgrade "pip<19.2.0" setuptools wheel + - name: Build Python Package + run: | + python -m pip install -e ".[tests,virtualenv]" + python -m pip install -e tests/pytest-pypi + # Temporarily pin tomlkit to lower version dueto sdispater/tomlkit/issues/56. + python -m pip install --upgrade "tomlkit<=0.5.3" + - name: Lint with flake8 + run: | + python -m pip install flake8 + flake8 --show-source src/ tests/ + - name: Test with pytest + run: | + pytest -n auto --cov=passa --cov-report=xml + - name: Report code coverage + env: + CODECOV_TOKEN: ${{secrets.CODECOV_TOKEN}} + run: | + pip install codecov + coverage report + codecov + + pack: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: Set up Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Install Requirements + run: | + python -m pip install --upgrade "pip<19.2.0" check-manifest invoke + pip install -e ".[pack]" + - name: Check MANIFEST + run: | + check-manifest + - name: Packaging + run: | + invoke pack + python pack/passa.zip --help + - name: Upload packed result + uses: actions/upload-artifact@v1 + with: + name: packed + path: pack/passa.zip diff --git a/.gitignore b/.gitignore index 741f1b4..86c6ae9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ -/build -/dist +build/ +dist/ /docs/_build /pack htmlcov/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..69edd3a --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "tests/pypi"] + path = tests/pypi + url = https://github.com/sarugaku/pipenv-test-artifacts.git diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index b13fa9d..0000000 --- a/.travis.yml +++ /dev/null @@ -1,44 +0,0 @@ -language: python -sudo: false -cache: pip -dist: trusty - -matrix: - fast_finish: true - -install: - - "python -m pip install --upgrade --upgrade-strategy=eager pip setuptools pytest pytest-timeout pytest-cov pytest-xdist" - - "python -m pip install --upgrade --upgrade-strategy=eager -e .[tests,virtualenv]" -script: - - "python -m pytest -v -n 8 tests/" - -jobs: - include: - - python: "3.7" - dist: xenial - sudo: required - - python: "3.6" - - python: "2.7" - - python: "3.5" - - python: "3.4" - - stage: packaging - python: "3.6" - install: - - "python -m pip install --upgrade pip setuptools" - - "python -m pip install --upgrade check-manifest readme-renderer" - script: - - "python setup.py check -m -r -s" - - stage: packing - python: "3.6" - install: - - "python -m pip install --upgrade --upgrade-strategy=eager -e .[pack]" - script: - - "invoke pack" - - "python2.7 pack/passa.zip --help" - - stage: coverage - python: "3.6" - install: - - "python -m pip install --upgrade --upgrade-strategy=eager pip setuptools pytest-timeout pytest-cov pytest-xdist" - - "python -m pip install --upgrade --upgrade-strategy=eager -e .[tests,virtualenv]" - script: - - "pytest -n auto --timeout 300 --cov=passa --cov-report=term-missing --cov-report=xml --cov-report=html tests" diff --git a/MANIFEST.in b/MANIFEST.in index 5625dbc..e4b835a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -12,6 +12,7 @@ recursive-include docs Makefile *.rst *.py *.bat recursive-exclude docs requirements*.txt prune .github +prune .gitmodules prune docs/build prune news prune tasks diff --git a/Pipfile b/Pipfile index f1eabeb..9a60cd7 100644 --- a/Pipfile +++ b/Pipfile @@ -1,6 +1,7 @@ [packages] -passa = { editable = true, path = '.', extras = ['virtualenv'] } - +passa = {editable = true,path = '.'} +tomlkit = "<=0.5.3" +yaspin = {file="https://github.com/sarugaku/yaspin/releases/download/v0.15.0post1/yaspin-0.15.0.post1-py2.py3-none-any.whl"} # Override sdist-only dependency via TOMLkit to fix build. (sarugaku/passa#61) [packages.functools32] file = """\ @@ -12,12 +13,13 @@ markers = "python_version < '3.0'" black = '*' invoke = '*' parver = '*' -passa = { editable = true, path = '.', extras = ['tests'] } +passa = {editable = true,path = '.',extras = ['tests']} sphinx = '*' sphinx-rtd-theme = '*' towncrier = '*' twine = '*' wheel = '*' +coverage = "<5.0" [scripts] passa-add = 'python -m passa.cli.add' @@ -33,3 +35,6 @@ draft = 'towncrier --draft' release = 'inv release' tests = "pytest -v tests" upload = 'inv upload' + +[pipenv] +allow_prereleases = true diff --git a/Pipfile.lock b/Pipfile.lock index f3869a3..8defa93 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "98ad4ec51e7ea8861ce2e31ac6d39a134251d84a2a7b47f2053e905900312639" + "sha256": "c378a05a5b239a430bd12e291ff6554b13d6405d688e6dd4941dd6e60a524b53" }, "pipfile-spec": 6, "requires": {}, @@ -19,36 +19,39 @@ "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92", "sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e" ], - "markers": "python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '3.6'", "version": "==1.4.3" }, "attrs": { "hashes": [ - "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", - "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" + "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", + "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" ], - "markers": "python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==18.2.0" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==19.1.0" }, "backports-functools-lru-cache": { "hashes": [ + "sha256:9d98697f088eb1b0fa451391f91afb5e3ebde16bbdb272819fd091151fda4f1a", "sha256:f0b0e4eba956de51238e17573b7087e852dfe9854afd2e9c873f73fc0ca0a6dd" ], - "markers": "python_version <= '3.4' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version == '2.7' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.5" }, "backports-shutil-get-terminal-size": { "hashes": [ - "sha256:0975ba55054c15e346944b38956a4c9cbee9009391e41b86c68990effb8c1f64" + "sha256:0975ba55054c15e346944b38956a4c9cbee9009391e41b86c68990effb8c1f64", + "sha256:713e7a8228ae80341c70586d1cc0a8caa5207346927e23d09dcbcaf18eadec80" ], - "markers": "python_version < '3.3' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version == '2.7' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version == '2.7' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.0.0" }, "backports-weakref": { "hashes": [ - "sha256:81bc9b51c0abc58edc76aefbbc68c62a787918ffe943a37947e162c3f8e19e82" + "sha256:81bc9b51c0abc58edc76aefbbc68c62a787918ffe943a37947e162c3f8e19e82", + "sha256:bc4170a29915f8b22c9e7c4939701859650f2eb84184aee80da329ac0b9825c2" ], - "markers": "python_version < '3.3' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version == '2.7' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version == '2.7' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.0.post1" }, "cached-property": { @@ -56,30 +59,30 @@ "sha256:3a026f1a54135677e7da5ce819b0c690f156f37976f3e30c5430740725203d7f", "sha256:9217a59f14a5682da7c4b8829deadbfc194ac22e9908ccf7c8820234e80a1504" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.5.1" }, "cerberus": { "hashes": [ - "sha256:f5c2e048fb15ecb3c088d192164316093fcfa602a74b3386eefb2983aa7e800a" + "sha256:0be48fc0dc84f83202a5309c0aa17cd5393e70731a1698a50d118b762fbe6875" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.2" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.3.1" }, "certifi": { "hashes": [ - "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7", - "sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033" + "sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50", + "sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2018.11.29" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' or python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' or python_version >= '2.7' and python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==2019.9.11" }, "chardet": { "hashes": [ "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' or python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' or python_version >= '2.7' and python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==3.0.4" }, "colorama": { @@ -87,22 +90,31 @@ "sha256:05eed71e2e327246ad6b38c540c4a3117230b19679b875190486ddd2d721422d", "sha256:f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' and sys_platform == 'win32' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and sys_platform == 'win32'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' and sys_platform == 'win32' or python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3' and sys_platform == 'win32' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and sys_platform == 'win32'", "version": "==0.4.1" }, - "cursor": { + "configparser": { "hashes": [ - "sha256:8ee9fe5b925e1001f6ae6c017e93682583d2b4d1ef7130a26cfcdf1651c0032c" + "sha256:254c1d9c79f60c45dfde850850883d5aaa7f19a23f13561243a050d5a7c3fe4c", + "sha256:c7d282687a5308319bf3d2e7706e575c635b0a470342641c93bea0ea3b5331df" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.2.0" + "markers": "python_version < '3' and python_version < '3.8' and python_version >= '2.6' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3' and python_version < '3.8' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==4.0.2" + }, + "contextlib2": { + "hashes": [ + "sha256:7197aa736777caac513dbd800944c209a49765bf1979b12b037dce0277077ed3", + "sha256:9d2c67f18c1f9b6db1b46317f7f784aa82789d2ee5dea5d9c0f0f2a764eb862e" + ], + "markers": "python_version < '3' and python_version < '3.8' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3' and python_version < '3.8' and python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3' and python_version < '3.8' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.6.0" }, "distlib": { "hashes": [ - "sha256:57977cd7d9ea27986ec62f425630e4ddb42efe651ff80bc58ed8dbc3c7c21f19" + "sha256:ecb3d0e4f71d0fa7f38db6bcc276c7c9a1c6638a516d726495934a553eb3fbe0" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.2.8" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.2.9.post0" }, "enum34": { "hashes": [ @@ -116,23 +128,23 @@ }, "first": { "hashes": [ - "sha256:3bb3de3582cb27071cfb514f00ed784dc444b7f96dc21e140de65fe00585c95e", - "sha256:41d5b64e70507d0c3ca742d68010a76060eea8a3d863e9b5130ab11a4a91aa0e" + "sha256:8d8e46e115ea8ac652c76123c0865e3ff18372aef6f03c22809ceefcea9dec86", + "sha256:ff285b08c55f8c97ce4ea7012743af2495c9f1291785f163722bd36f6af6d3bf" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.0.1" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.0.2" }, "functools32": { "file": "https://github.com/sarugaku/functools32/releases/download/3.2.3-2/functools32-3.2.3.post2-py2.py3-none-any.whl", - "markers": "python_version < '3.0' or python_version < '3.0' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.0' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'" + "markers": "python_version >= '2.6' and python_version >= '2.7' and python_version < '2.8' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version < '2.8' or python_version >= '2.7' and python_version >= '2.7' and python_version < '2.8' and python_version not in '3.0, 3.1, 3.2, 3.3'" }, "idna": { "hashes": [ - "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", - "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16" + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.7" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' or python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' or python_version >= '2.7' and python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==2.8" }, "importlib": { "hashes": [ @@ -141,45 +153,53 @@ "markers": "python_version < '2.7' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.0.4" }, + "importlib-metadata": { + "hashes": [ + "sha256:aa18d7378b00b40847790e7c27e11673d7fed219354109d0e7b9e5b25dc3ad26", + "sha256:d5f18a79777f3aa179c145737780282e27b508fc8fd688cb17c7a813e8bd39af" + ], + "markers": "python_version < '3.8' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.8' and python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.8' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.23" + }, "installer": { "hashes": [ "sha256:1ba23de573e9b95a8dcbd04fd026c40a64b77db0aadc48f28a844b4cb87479fe", "sha256:f4f195c9b17ea7d2b631a758451485c6b080975349b4adebe45ef4bb022db069" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.1.1" }, - "modutil": { + "more-itertools": { "hashes": [ - "sha256:2c85c1666649e92e56de17c00e1e831313602d9b55e8661d39c01e39003b45f7", - "sha256:cc3dad264e36ed359fdd67c4588959d2996bd0402ad9c9d974ca906821537218" + "sha256:409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832", + "sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4" ], - "markers": "python_version >= '3.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.0.0" + "markers": "python_version < '3.8' and python_version >= '2.6' and python_version >= '3.4' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.8' and python_version >= '2.7' and python_version >= '3.4' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.6' and python_version >= '3.4' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version >= '3.4' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==7.2.0" }, - "mork": { + "orderedmultidict": { "hashes": [ - "sha256:13772edb4724915cf0cfa30d31426e0565487a3b2d7883b8468718eaed8ecfc2", - "sha256:b1b41bc31603eef1b50e42e75ae2d74d7a0d9ab46ea4d0dd1ba387a451870873" + "sha256:04070bbb5e87291cc9bfa51df413677faf2141c73c61d2a5f7b26bea3cd882ad", + "sha256:43c839a17ee3cdd62234c47deca1a8508a3f2ca1d0678a3bf791c87cf84adbf3" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.1.4" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.0.1" }, "packagebuilder": { "hashes": [ "sha256:1e85c4e0e994322996b93cd6685c12834d30f3558889154f8e3de8fb1f3fd1e7", "sha256:dc525d06ecd102db23ab421b879d7d27021d784ff933e33e8c411a53af5c9dbe" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.1.0" }, "packaging": { "hashes": [ - "sha256:0886227f54515e592aaa2e5a553332c73962917f2831f1b0f9b9f4380a4b9807", - "sha256:f95a1e147590f204328170981833854229bb2912ac3d5f89e2a8ccd2834800c9" + "sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47", + "sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==18.0" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==19.2" }, "passa": { "editable": true, @@ -190,27 +210,27 @@ }, "pathlib2": { "hashes": [ - "sha256:25199318e8cc3c25dcb45cbe084cc061051336d5a9ea2a12448d3d8cb748f742", - "sha256:5887121d7f7df3603bca2f710e7219f3eca0eb69e0b7cc6e0a022e155ac931a7" + "sha256:2156525d6576d21c4dcaddfa427fae887ef89a7a9de5cbfe0728b3aafa78427e", + "sha256:446014523bb9be5c28128c4d2a10ad6bb60769e78bd85658fe44a450674e0ef8" ], - "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.3.3" + "markers": "python_version < '3.6' and python_version < '3.8' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.6' and python_version < '3.8' and python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.6' and python_version < '3.8' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.6' and python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.6' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.3.4" }, "pep517": { "hashes": [ - "sha256:cc663a438fdfe2e88d8d3c5ef2203ac858de34e31b6609b1fc505d611490a926", - "sha256:f79bb08fb064dfc5b141204bfeb56a4141a6d504677fab4723036a464fc25cc1" + "sha256:273345f4538306f6e4056d8bbced566e186ab4defc188cb3be3e413b5d255912", + "sha256:dde535e9a42de94f4cd941dbaa6feb0a4b5143ffd3906efea091c3826cb7d33d" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.3" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.6.0" }, "pip-shims": { "hashes": [ - "sha256:3bc24ec050a6b9eea35419467237e4f47eaf806dadc9999bf887355c377edea7", - "sha256:edb4cf3c509eab2f36b55c1ac1a59a4c485ccd537cc87934d74950880f641256" + "sha256:0162d846bd60c7b1feb4e1336541a3e661eb6a1eff4b9ea0d759780f8e0a2936", + "sha256:d73372b9fa3a10e73f057fced70d39bb82df16a1ee3ded027fed0bb8d7c0ff97" ], "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.3.2" + "version": "==0.3.3" }, "plette": { "extras": [ @@ -225,34 +245,43 @@ }, "pyparsing": { "hashes": [ - "sha256:40856e74d4987de5d01761a22d1621ae1c7f8774585acae358aa5c5936c6c90b", - "sha256:f353aab21fd474459d97b709e527b5571314ee5f067441dc9f88e33eecd96592" + "sha256:6f98a7b9397e206d78cc01df10131398f1c8b8510a2f4d97d9abd82e1aacdd80", + "sha256:d9338df12903bbf5d65a0e4e87c2161968b10d2e489652bb47001d82a9b028b4" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.3.0" + "markers": "python_version >= '2.6' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.4.2" }, "pytoml": { "hashes": [ - "sha256:ca2d0cb127c938b8b76a9a0d0f855cf930c1d50cc3a0af6d3595b566519a1013" + "sha256:57a21e6347049f73bfb62011ff34cd72774c031b9828cb628a752225136dfc33", + "sha256:8eecf7c8d0adcff3b375b09fe403407aa9b645c499e5ab8cac670ac4a35f61e7" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.1.21" + }, + "recursive-monkey-patch": { + "hashes": [ + "sha256:546739fea5be2ea9f98b5ec44fafeb697b5cf9fdcda64a03422582ab03ee24c4", + "sha256:98922554e77f2e2c85a4f5d873a0f52efdc1b553f32444bd6c788b2ff583bf3e" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.1.20" + "version": "==0.4.0" }, "requests": { "hashes": [ - "sha256:65b3a120e4329e33c9889db89c80976c5272f56ea92d3e74da8a463992e3ff54", - "sha256:ea881206e59f41dbd0bd445437d792e43906703fff75ca8ff43ccdb11f33f263" + "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", + "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.20.1" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' or python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' or python_version >= '2.7' and python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==2.22.0" }, "requirementslib": { "hashes": [ - "sha256:c2c00c7bd3bd4984c97d10cd4d143efbe33b5ed9e55961bea30ca7a9a4927289", - "sha256:dc6b692e8dee03d6e90c29db1e337b0bf8152cce84a57f0fb4765e596afde4e0" + "sha256:50731ac1052473e4c7df59a44a1f3aa20f32e687110bc05d73c3b4109eebc23d", + "sha256:8b594ab8b6280ee97cffd68fc766333345de150124d5b76061dd575c3a21fe5a" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.3.3" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.5.3" }, "resolvelib": { "hashes": [ @@ -264,27 +293,28 @@ }, "scandir": { "hashes": [ - "sha256:04b8adb105f2ed313a7c2ef0f1cf7aff4871aa7a1883fa4d8c44b5551ab052d6", - "sha256:1444134990356c81d12f30e4b311379acfbbcd03e0bab591de2696a3b126d58e", - "sha256:1b5c314e39f596875e5a95dd81af03730b338c277c54a454226978d5ba95dbb6", - "sha256:346619f72eb0ddc4cf355ceffd225fa52506c92a2ff05318cfabd02a144e7c4e", - "sha256:44975e209c4827fc18a3486f257154d34ec6eaec0f90fef0cca1caa482db7064", - "sha256:61859fd7e40b8c71e609c202db5b0c1dbec0d5c7f1449dec2245575bdc866792", - "sha256:a5e232a0bf188362fa00123cc0bb842d363a292de7126126df5527b6a369586a", - "sha256:c14701409f311e7a9b7ec8e337f0815baf7ac95776cc78b419a1e6d49889a383", - "sha256:c7708f29d843fc2764310732e41f0ce27feadde453261859ec0fca7865dfc41b", - "sha256:c9009c527929f6e25604aec39b0a43c3f831d2947d89d6caaab22f057b7055c8", - "sha256:f5c71e29b4e2af7ccdc03a020c626ede51da471173b4a6ad1e904f2b2e04b4bd" - ], - "markers": "python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.9.0" + "sha256:2586c94e907d99617887daed6c1d102b5ca28f1085f90446554abf1faf73123e", + "sha256:2ae41f43797ca0c11591c0c35f2f5875fa99f8797cb1a1fd440497ec0ae4b022", + "sha256:2b8e3888b11abb2217a32af0766bc06b65cc4a928d8727828ee68af5a967fa6f", + "sha256:2c712840c2e2ee8dfaf36034080108d30060d759c7b73a01a52251cc8989f11f", + "sha256:4d4631f6062e658e9007ab3149a9b914f3548cb38bfb021c64f39a025ce578ae", + "sha256:67f15b6f83e6507fdc6fca22fedf6ef8b334b399ca27c6b568cbfaa82a364173", + "sha256:7d2d7a06a252764061a020407b997dd036f7bd6a175a5ba2b345f0a357f0b3f4", + "sha256:8c5922863e44ffc00c5c693190648daa6d15e7c1207ed02d6f46a8dcc2869d32", + "sha256:92c85ac42f41ffdc35b6da57ed991575bdbe69db895507af88b9f499b701c188", + "sha256:b24086f2375c4a094a6b51e78b4cf7ca16c721dcee2eddd7aa6494b42d6d519d", + "sha256:cb925555f43060a1745d0a321cca94bcea927c50114b623d73179189a4e100ac" + ], + "markers": "python_version < '3.5' and python_version < '3.6' and python_version < '3.8' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version < '3.6' and python_version < '3.8' and python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version < '3.6' and python_version < '3.8' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version < '3.6' and python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version < '3.6' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.10.0" }, "six": { "hashes": [ - "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", - "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" ], - "version": "==1.11.0" + "markers": "python_version < '3.0' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.6' and python_version < '3.8' and python_version >= '2.6' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.6' and python_version < '3.8' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.6' and python_version >= '2.6' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.6' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.6' and python_version >= '3.6' and python_version not in '3.0, 3.1' or python_version >= '2.6' and python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.12.0" }, "tomlkit": { "hashes": [ @@ -296,20 +326,20 @@ }, "typing": { "hashes": [ - "sha256:4027c5f6127a6267a435201981ba156de91ad0d1d98e9ddc2aa173453453492d", - "sha256:57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4", - "sha256:a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a" + "sha256:91dfe6f3f706ee8cc32d38edbbf304e9b7583fb37108fef38229617f8b3eba23", + "sha256:c8cabb5ab8945cd2f54917be357d134db9cc1eb039e59d1606dc1e60cb1d9d36", + "sha256:f38d83c5a7a7086543a0f649564d661859c5146a85775ab90c0d2f93ffaa9714" ], - "markers": "python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==3.6.6" + "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==3.7.4.1" }, "urllib3": { "hashes": [ - "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", - "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" + "sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398", + "sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86" ], - "markers": "python_version < '4' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '4' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.24.1" + "markers": "python_version < '4' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' or python_version < '4' and python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' or python_version < '4' and python_version >= '2.7' and python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' or python_version < '4' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==1.25.6" }, "virtualenv": { "hashes": [ @@ -324,27 +354,30 @@ "spinner" ], "hashes": [ - "sha256:3a1020fb7be000b268af96641ced9ead844b1f75840c41e20e473647688fc630", - "sha256:6d2005ad670f77bd9c9b5415c4e2a4a20dce5b0cf0e0d11598eb463b2e0ebe44" + "sha256:2166e3148a67c438c9e3edbba0cde153d42dec6e3bf5d8f4624feb27686c0990", + "sha256:3a0529b4b6c2e842fd19b5ceaa95b6c9201321314825c110406d4af3331a0709" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.2.5" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.4.3" }, "wheel": { "hashes": [ - "sha256:029703bf514e16c8271c3821806a1c171220cc5bdd325cbf4e7da1e056a01db6", - "sha256:1e53cdb3f808d5ccd0df57f964263752aa74ea7359526d3da6c02114ec1e1d44" + "sha256:10c9da68765315ed98850f8e048347c3eb06dd81822dc2ab1d4fde9dc9702646", + "sha256:f4da1763d3becf2e2cd92a14a7c920f0f00eca30fdde9ea992c836685b9faf28" ], "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.32.3" + "version": "==0.33.6" }, "yaspin": { + "file": "https://github.com/sarugaku/yaspin/releases/download/v0.15.0post1/yaspin-0.15.0.post1-py2.py3-none-any.whl" + }, + "zipp": { "hashes": [ - "sha256:36fdccc5e0637b5baa8892fe2c3d927782df7d504e9020f40eb2c1502518aa5a", - "sha256:8e52bf8079a48e2a53f3dfeec9e04addb900c101d1591c85df69cf677d3237e7" + "sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e", + "sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.14.0" + "markers": "python_version < '3.8' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.8' and python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.8' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.6.0" } }, "develop": { @@ -353,7 +386,7 @@ "sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359", "sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '3.5'", "version": "==0.7.12" }, "apipkg": { @@ -369,7 +402,7 @@ "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92", "sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e" ], - "markers": "python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '3.6'", "version": "==1.4.3" }, "arpeggio": { @@ -381,85 +414,90 @@ }, "atomicwrites": { "hashes": [ - "sha256:0312ad34fcad8fac3704d441f7b317e50af620823353ec657a53e981f92920c0", - "sha256:ec9ae8adaae229e4f8446952d204a3e4b5fdd2d099f9be3aaf556120135fb3ee" + "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", + "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.2.1" + "markers": "python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.3.0" }, "attrs": { "hashes": [ - "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", - "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" + "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", + "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" ], - "markers": "python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==18.2.0" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==19.1.0" }, "babel": { "hashes": [ - "sha256:6778d85147d5d85345c14a26aada5e478ab04e39b078b0745ee6870c2b5cf669", - "sha256:8cba50f48c529ca3fa18cf81fa9403be176d374ac4d60738b839122dfaaa3d23" + "sha256:af92e6106cb7c55286b25b38ad7695f8b4efb36a90ba483d7f7a6628c46158ab", + "sha256:e86135ae101e31e2c8ec20a4e0c5220f4eed12487d5cf3f78be7e98d3a57fc28" ], - "version": "==2.6.0" + "markers": "python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.7.0" }, "backports-functools-lru-cache": { "hashes": [ + "sha256:9d98697f088eb1b0fa451391f91afb5e3ebde16bbdb272819fd091151fda4f1a", "sha256:f0b0e4eba956de51238e17573b7087e852dfe9854afd2e9c873f73fc0ca0a6dd" ], - "markers": "python_version <= '3.4' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version == '2.7' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.5" }, "backports-shutil-get-terminal-size": { "hashes": [ - "sha256:0975ba55054c15e346944b38956a4c9cbee9009391e41b86c68990effb8c1f64" + "sha256:0975ba55054c15e346944b38956a4c9cbee9009391e41b86c68990effb8c1f64", + "sha256:713e7a8228ae80341c70586d1cc0a8caa5207346927e23d09dcbcaf18eadec80" ], - "markers": "python_version < '3.3' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version == '2.7' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version == '2.7' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.0.0" }, "backports-weakref": { "hashes": [ - "sha256:81bc9b51c0abc58edc76aefbbc68c62a787918ffe943a37947e162c3f8e19e82" + "sha256:81bc9b51c0abc58edc76aefbbc68c62a787918ffe943a37947e162c3f8e19e82", + "sha256:bc4170a29915f8b22c9e7c4939701859650f2eb84184aee80da329ac0b9825c2" ], - "markers": "python_version < '3.3' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version == '2.7' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version == '2.7' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.0.post1" }, "black": { "hashes": [ - "sha256:817243426042db1d36617910df579a54f1afd659adb96fc5032fcf4b36209739", - "sha256:e030a9a28f542debc08acceb273f228ac422798e5215ba2a791a6ddeaaca22a5" + "sha256:09a9dcb7c46ed496a9850b76e4e825d6049ecd38b611f1224857a79bd985a8cf", + "sha256:68950ffd4d9169716bcb8719a56c07a2f4485354fec061cdd5910aa07369731c" ], "markers": "python_version >= '3.6'", - "version": "==18.9b0" + "version": "==19.3b0" }, "bleach": { "hashes": [ - "sha256:9c471c0dd9c820f6bf4ee5ca3e348ceccefbc1475d9a40c397ed5d04e0b42c54", - "sha256:b407b2612b37e6cdc6704f84cec18c1f140b78e6c625652a844e89d6b9855f6b" + "sha256:213336e49e102af26d9cde77dd2d0397afabc5a6bf2fed985dc35b5d1e285a16", + "sha256:3fdf7f77adcf649c9911387df51254b813185e32b2c6619f690b593a617e19fa" ], - "version": "==3.0.0" + "markers": "python_version >= '2.7' and python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==3.1.0" }, "cached-property": { "hashes": [ "sha256:3a026f1a54135677e7da5ce819b0c690f156f37976f3e30c5430740725203d7f", "sha256:9217a59f14a5682da7c4b8829deadbfc194ac22e9908ccf7c8820234e80a1504" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.5.1" }, "cerberus": { "hashes": [ - "sha256:f5c2e048fb15ecb3c088d192164316093fcfa602a74b3386eefb2983aa7e800a" + "sha256:0be48fc0dc84f83202a5309c0aa17cd5393e70731a1698a50d118b762fbe6875" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.2" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.3.1" }, "certifi": { "hashes": [ - "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7", - "sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033" + "sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50", + "sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2018.11.29" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' or python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' or python_version >= '2.7' and python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==2019.9.11" }, "cffi": { "hashes": [ @@ -503,7 +541,7 @@ "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' or python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' or python_version >= '2.7' and python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==3.0.4" }, "click": { @@ -514,105 +552,89 @@ "markers": "python_version >= '3.6'", "version": "==7.0" }, - "cmarkgfm": { - "hashes": [ - "sha256:0186dccca79483e3405217993b83b914ba4559fe9a8396efc4eea56561b74061", - "sha256:1a625afc6f62da428df96ec325dc30866cc5781520cbd904ff4ec44cf018171c", - "sha256:207b7673ff4e177374c572feeae0e4ef33be620ec9171c08fd22e2b796e03e3d", - "sha256:275905bb371a99285c74931700db3f0c078e7603bed383e8cf1a09f3ee05a3de", - "sha256:50098f1c4950722521f0671e54139e0edc1837d63c990cf0f3d2c49607bb51a2", - "sha256:50ed116d0b60a07df0dc7b180c28569064b9d37d1578d4c9021cff04d725cb63", - "sha256:61a72def110eed903cd1848245897bcb80d295cd9d13944d4f9f30cba5b76655", - "sha256:64186fb75d973a06df0e6ea12879533b71f6e7ba1ab01ffee7fc3e7534758889", - "sha256:665303d34d7f14f10d7b0651082f25ebf7107f29ef3d699490cac16cdc0fc8ce", - "sha256:70b18f843aec58e4e64aadce48a897fe7c50426718b7753aaee399e72df64190", - "sha256:761ee7b04d1caee2931344ac6bfebf37102ffb203b136b676b0a71a3f0ea3c87", - "sha256:811527e9b7280b136734ed6cb6845e5fbccaeaa132ddf45f0246cbe544016957", - "sha256:987b0e157f70c72a84f3c2f9ef2d7ab0f26c08f2bf326c12c087ff9eebcb3ff5", - "sha256:9fc6a2183d0a9b0974ec7cdcdad42bd78a3be674cc3e65f87dd694419b3b0ab7", - "sha256:a3d17ee4ae739fe16f7501a52255c2e287ac817cfd88565b9859f70520afffea", - "sha256:ba5b5488719c0f2ced0aa1986376f7baff1a1653a8eb5fdfcf3f84c7ce46ef8d", - "sha256:c573ea89dd95d41b6d8cf36799c34b6d5b1eac4aed0212dee0f0a11fb7b01e8f", - "sha256:c5f1b9e8592d2c448c44e6bc0d91224b16ea5f8293908b1561de1f6d2d0658b1", - "sha256:cbe581456357d8f0674d6a590b1aaf46c11d01dd0a23af147a51a798c3818034", - "sha256:cf219bec69e601fe27e3974b7307d2f06082ab385d42752738ad2eb630a47d65", - "sha256:cf5014eb214d814a83a7a47407272d5db10b719dbeaf4d3cfe5969309d0fcf4b", - "sha256:d08bad67fa18f7e8ff738c090628ee0cbf0505d74a991c848d6d04abfe67b697", - "sha256:d6f716d7b1182bf35862b5065112f933f43dd1aa4f8097c9bcfb246f71528a34", - "sha256:e08e479102627641c7cb4ece421c6ed4124820b1758765db32201136762282d9", - "sha256:e20ac21418af0298437d29599f7851915497ce9f2866bc8e86b084d8911ee061", - "sha256:e25f53c37e319241b9a412382140dffac98ca756ba8f360ac7ab5e30cad9670a", - "sha256:e8932bddf159064f04e946fbb64693753488de21586f20e840b3be51745c8c09", - "sha256:f20900f16377f2109783ae9348d34bc80530808439591c3d3df73d5c7ef1a00c" - ], - "version": "==0.4.2" + "click-default-group": { + "hashes": [ + "sha256:d9560e8e8dfa44b3562fbc9425042a0fd6d21956fcc2db0077f63f34253ab904" + ], + "version": "==1.2.2" }, "colorama": { "hashes": [ "sha256:05eed71e2e327246ad6b38c540c4a3117230b19679b875190486ddd2d721422d", "sha256:f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' and sys_platform == 'win32' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and sys_platform == 'win32'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' and sys_platform == 'win32' or python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3' and sys_platform == 'win32' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and sys_platform == 'win32'", "version": "==0.4.1" }, - "coverage": { + "configparser": { "hashes": [ - "sha256:09e47c529ff77bf042ecfe858fb55c3e3eb97aac2c87f0349ab5a7efd6b3939f", - "sha256:0a1f9b0eb3aa15c990c328535655847b3420231af299386cfe5efc98f9c250fe", - "sha256:0cc941b37b8c2ececfed341444a456912e740ecf515d560de58b9a76562d966d", - "sha256:10e8af18d1315de936d67775d3a814cc81d0747a1a0312d84e27ae5610e313b0", - "sha256:1b4276550b86caa60606bd3572b52769860a81a70754a54acc8ba789ce74d607", - "sha256:1e8a2627c48266c7b813975335cfdea58c706fe36f607c97d9392e61502dc79d", - "sha256:2b224052bfd801beb7478b03e8a66f3f25ea56ea488922e98903914ac9ac930b", - "sha256:447c450a093766744ab53bf1e7063ec82866f27bcb4f4c907da25ad293bba7e3", - "sha256:46101fc20c6f6568561cdd15a54018bb42980954b79aa46da8ae6f008066a30e", - "sha256:4710dc676bb4b779c4361b54eb308bc84d64a2fa3d78e5f7228921eccce5d815", - "sha256:510986f9a280cd05189b42eee2b69fecdf5bf9651d4cd315ea21d24a964a3c36", - "sha256:5535dda5739257effef56e49a1c51c71f1d37a6e5607bb25a5eee507c59580d1", - "sha256:5a7524042014642b39b1fcae85fb37556c200e64ec90824ae9ecf7b667ccfc14", - "sha256:5f55028169ef85e1fa8e4b8b1b91c0b3b0fa3297c4fb22990d46ff01d22c2d6c", - "sha256:6694d5573e7790a0e8d3d177d7a416ca5f5c150742ee703f3c18df76260de794", - "sha256:6831e1ac20ac52634da606b658b0b2712d26984999c9d93f0c6e59fe62ca741b", - "sha256:77f0d9fa5e10d03aa4528436e33423bfa3718b86c646615f04616294c935f840", - "sha256:828ad813c7cdc2e71dcf141912c685bfe4b548c0e6d9540db6418b807c345ddd", - "sha256:85a06c61598b14b015d4df233d249cd5abfa61084ef5b9f64a48e997fd829a82", - "sha256:8cb4febad0f0b26c6f62e1628f2053954ad2c555d67660f28dfb1b0496711952", - "sha256:a5c58664b23b248b16b96253880b2868fb34358911400a7ba39d7f6399935389", - "sha256:aaa0f296e503cda4bc07566f592cd7a28779d433f3a23c48082af425d6d5a78f", - "sha256:ab235d9fe64833f12d1334d29b558aacedfbca2356dfb9691f2d0d38a8a7bfb4", - "sha256:b3b0c8f660fae65eac74fbf003f3103769b90012ae7a460863010539bb7a80da", - "sha256:bab8e6d510d2ea0f1d14f12642e3f35cefa47a9b2e4c7cea1852b52bc9c49647", - "sha256:c45297bbdbc8bb79b02cf41417d63352b70bcb76f1bbb1ee7d47b3e89e42f95d", - "sha256:d19bca47c8a01b92640c614a9147b081a1974f69168ecd494687c827109e8f42", - "sha256:d64b4340a0c488a9e79b66ec9f9d77d02b99b772c8b8afd46c1294c1d39ca478", - "sha256:da969da069a82bbb5300b59161d8d7c8d423bc4ccd3b410a9b4d8932aeefc14b", - "sha256:ed02c7539705696ecb7dc9d476d861f3904a8d2b7e894bd418994920935d36bb", - "sha256:ee5b8abc35b549012e03a7b1e86c09491457dba6c94112a2482b18589cc2bdb9" - ], - "markers": "python_version < '4' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==4.5.2" - }, - "cursor": { - "hashes": [ - "sha256:8ee9fe5b925e1001f6ae6c017e93682583d2b4d1ef7130a26cfcdf1651c0032c" + "sha256:254c1d9c79f60c45dfde850850883d5aaa7f19a23f13561243a050d5a7c3fe4c", + "sha256:c7d282687a5308319bf3d2e7706e575c635b0a470342641c93bea0ea3b5331df" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.2.0" + "markers": "python_version < '3' and python_version < '3.8' and python_version >= '2.6' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3' and python_version < '3.8' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==4.0.2" + }, + "contextlib2": { + "hashes": [ + "sha256:7197aa736777caac513dbd800944c209a49765bf1979b12b037dce0277077ed3", + "sha256:9d2c67f18c1f9b6db1b46317f7f784aa82789d2ee5dea5d9c0f0f2a764eb862e" + ], + "markers": "python_version < '3' and python_version < '3.8' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3' and python_version < '3.8' and python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3' and python_version < '3.8' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.6.0" + }, + "coverage": { + "hashes": [ + "sha256:08907593569fe59baca0bf152c43f3863201efb6113ecb38ce7e97ce339805a6", + "sha256:0be0f1ed45fc0c185cfd4ecc19a1d6532d72f86a2bac9de7e24541febad72650", + "sha256:141f08ed3c4b1847015e2cd62ec06d35e67a3ac185c26f7635f4406b90afa9c5", + "sha256:19e4df788a0581238e9390c85a7a09af39c7b539b29f25c89209e6c3e371270d", + "sha256:23cc09ed395b03424d1ae30dcc292615c1372bfba7141eb85e11e50efaa6b351", + "sha256:245388cda02af78276b479f299bbf3783ef0a6a6273037d7c60dc73b8d8d7755", + "sha256:331cb5115673a20fb131dadd22f5bcaf7677ef758741312bee4937d71a14b2ef", + "sha256:386e2e4090f0bc5df274e720105c342263423e77ee8826002dcffe0c9533dbca", + "sha256:3a794ce50daee01c74a494919d5ebdc23d58873747fa0e288318728533a3e1ca", + "sha256:60851187677b24c6085248f0a0b9b98d49cba7ecc7ec60ba6b9d2e5574ac1ee9", + "sha256:63a9a5fc43b58735f65ed63d2cf43508f462dc49857da70b8980ad78d41d52fc", + "sha256:6b62544bb68106e3f00b21c8930e83e584fdca005d4fffd29bb39fb3ffa03cb5", + "sha256:6ba744056423ef8d450cf627289166da65903885272055fb4b5e113137cfa14f", + "sha256:7494b0b0274c5072bddbfd5b4a6c6f18fbbe1ab1d22a41e99cd2d00c8f96ecfe", + "sha256:826f32b9547c8091679ff292a82aca9c7b9650f9fda3e2ca6bf2ac905b7ce888", + "sha256:93715dffbcd0678057f947f496484e906bf9509f5c1c38fc9ba3922893cda5f5", + "sha256:9a334d6c83dfeadae576b4d633a71620d40d1c379129d587faa42ee3e2a85cce", + "sha256:af7ed8a8aa6957aac47b4268631fa1df984643f07ef00acd374e456364b373f5", + "sha256:bf0a7aed7f5521c7ca67febd57db473af4762b9622254291fbcbb8cd0ba5e33e", + "sha256:bf1ef9eb901113a9805287e090452c05547578eaab1b62e4ad456fcc049a9b7e", + "sha256:c0afd27bc0e307a1ffc04ca5ec010a290e49e3afbe841c5cafc5c5a80ecd81c9", + "sha256:dd579709a87092c6dbee09d1b7cfa81831040705ffa12a1b248935274aee0437", + "sha256:df6712284b2e44a065097846488f66840445eb987eb81b3cc6e4149e7b6982e1", + "sha256:e07d9f1a23e9e93ab5c62902833bf3e4b1f65502927379148b6622686223125c", + "sha256:e2ede7c1d45e65e209d6093b762e98e8318ddeff95317d07a27a2140b80cfd24", + "sha256:e4ef9c164eb55123c62411f5936b5c2e521b12356037b6e1c2617cef45523d47", + "sha256:eca2b7343524e7ba246cab8ff00cab47a2d6d54ada3b02772e908a45675722e2", + "sha256:eee64c616adeff7db37cc37da4180a3a5b6177f5c46b187894e633f088fb5b28", + "sha256:ef824cad1f980d27f26166f86856efe11eff9912c4fed97d3804820d43fa550c", + "sha256:efc89291bd5a08855829a3c522df16d856455297cf35ae827a37edac45f466a7", + "sha256:fa964bae817babece5aa2e8c1af841bebb6d0b9add8e637548809d040443fee0", + "sha256:ff37757e068ae606659c28c3bd0d923f9d29a85de79bf25b2b34b148473b5025" + ], + "markers": "python_version < '4' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2' or python_version < '4' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==4.5.4" }, "distlib": { "hashes": [ - "sha256:57977cd7d9ea27986ec62f425630e4ddb42efe651ff80bc58ed8dbc3c7c21f19" + "sha256:ecb3d0e4f71d0fa7f38db6bcc276c7c9a1c6638a516d726495934a553eb3fbe0" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.2.8" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.2.9.post0" }, "docutils": { "hashes": [ - "sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6", - "sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274", - "sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6" + "sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0", + "sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827", + "sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99" ], - "version": "==0.14" + "markers": "python_version >= '2.6' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2' or python_version >= '2.6' and python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2'", + "version": "==0.15.2" }, "enum34": { "hashes": [ @@ -626,26 +648,26 @@ }, "execnet": { "hashes": [ - "sha256:a7a84d5fa07a089186a329528f127c9d73b9de57f1a1131b82bb5320ee651f6a", - "sha256:fc155a6b553c66c838d1a22dba1dc9f5f505c43285a878c6f74a79c024750b83" + "sha256:cacb9df31c9680ec5f95553976c4da484d407e85e41c83cb812aa014f0eddc50", + "sha256:d4efd397930c46415f62f8a31388d6be4f27a91d7550eb79bc64a756e0056547" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.5.0" + "version": "==1.7.1" }, "first": { "hashes": [ - "sha256:3bb3de3582cb27071cfb514f00ed784dc444b7f96dc21e140de65fe00585c95e", - "sha256:41d5b64e70507d0c3ca742d68010a76060eea8a3d863e9b5130ab11a4a91aa0e" + "sha256:8d8e46e115ea8ac652c76123c0865e3ff18372aef6f03c22809ceefcea9dec86", + "sha256:ff285b08c55f8c97ce4ea7012743af2495c9f1291785f163722bd36f6af6d3bf" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.0.1" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.0.2" }, "funcsigs": { "hashes": [ "sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca", "sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50" ], - "markers": "python_version < '3.0' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '3.0' and python_version < '3.3' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.0.2" }, "future": { @@ -656,21 +678,22 @@ }, "functools32": { "file": "https://github.com/sarugaku/functools32/releases/download/3.2.3-2/functools32-3.2.3.post2-py2.py3-none-any.whl", - "markers": "python_version < '3.0' or python_version < '3.0' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.0' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'" + "markers": "python_version >= '2.6' and python_version >= '2.7' and python_version < '2.8' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version < '2.8' or python_version >= '2.7' and python_version >= '2.7' and python_version < '2.8' and python_version not in '3.0, 3.1, 3.2, 3.3'" }, "idna": { "hashes": [ - "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", - "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16" + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.7" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' or python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' or python_version >= '2.7' and python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==2.8" }, "imagesize": { "hashes": [ "sha256:3f349de3eb99145973fefb7dbe38554414e5c30abd0c8e4b970a7c9d09f3a1d8", "sha256:f3832918bc3c66617f92e35f5d70729187676313caa60c187eb0f28b8fe5e3b5" ], + "markers": "python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.0" }, "importlib": { @@ -680,6 +703,14 @@ "markers": "python_version < '2.7' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.0.4" }, + "importlib-metadata": { + "hashes": [ + "sha256:aa18d7378b00b40847790e7c27e11673d7fed219354109d0e7b9e5b25dc3ad26", + "sha256:d5f18a79777f3aa179c145737780282e27b508fc8fd688cb17c7a813e8bd39af" + ], + "markers": "python_version < '3.8' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.8' and python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.8' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.23" + }, "incremental": { "hashes": [ "sha256:717e12246dddf231a349175f48d74d93e2897244939173b01974ab6661406b9f", @@ -692,73 +723,96 @@ "sha256:1ba23de573e9b95a8dcbd04fd026c40a64b77db0aadc48f28a844b4cb87479fe", "sha256:f4f195c9b17ea7d2b631a758451485c6b080975349b4adebe45ef4bb022db069" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.1.1" }, "invoke": { "hashes": [ - "sha256:4f4de934b15c2276caa4fbc5a3b8a61c0eb0b234f2be1780d2b793321995c2d6", - "sha256:dc492f8f17a0746e92081aec3f86ae0b4750bf41607ea2ad87e5a7b5705121b7", - "sha256:eb6f9262d4d25b40330fb21d1e99bf0f85011ccc3526980f8a3eaedd4b43892e" + "sha256:c52274d2e8a6d64ef0d61093e1983268ea1fc0cd13facb9448c4ef0c9a7ac7da", + "sha256:f4ec8a134c0122ea042c8912529f87652445d9f4de590b353d23f95bfa1f0efd", + "sha256:fc803a5c9052f15e63310aa81a43498d7c55542beb18564db88a9d75a176fa44" ], - "version": "==1.2.0" + "version": "==1.3.0" }, "jinja2": { "hashes": [ - "sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd", - "sha256:f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4" + "sha256:065c4f02ebe7f7cf559e49ee5a95fb800a9e4528727aec6f24402a5374c65013", + "sha256:14dd6caf1527abb21f08f86c784eac40853ba93edb79552aa1e4b8aef1b61c7b" ], - "version": "==2.10" + "version": "==2.10.1" }, "markupsafe": { "hashes": [ - "sha256:048ef924c1623740e70204aa7143ec592504045ae4429b59c30054cb31e3c432", - "sha256:130f844e7f5bdd8e9f3f42e7102ef1d49b2e6fdf0d7526df3f87281a532d8c8b", - "sha256:19f637c2ac5ae9da8bfd98cef74d64b7e1bb8a63038a3505cd182c3fac5eb4d9", - "sha256:1b8a7a87ad1b92bd887568ce54b23565f3fd7018c4180136e1cf412b405a47af", - "sha256:1c25694ca680b6919de53a4bb3bdd0602beafc63ff001fea2f2fc16ec3a11834", - "sha256:1f19ef5d3908110e1e891deefb5586aae1b49a7440db952454b4e281b41620cd", - "sha256:1fa6058938190ebe8290e5cae6c351e14e7bb44505c4a7624555ce57fbbeba0d", - "sha256:31cbb1359e8c25f9f48e156e59e2eaad51cd5242c05ed18a8de6dbe85184e4b7", - "sha256:3e835d8841ae7863f64e40e19477f7eb398674da6a47f09871673742531e6f4b", - "sha256:4e97332c9ce444b0c2c38dd22ddc61c743eb208d916e4265a2a3b575bdccb1d3", - "sha256:525396ee324ee2da82919f2ee9c9e73b012f23e7640131dd1b53a90206a0f09c", - "sha256:52b07fbc32032c21ad4ab060fec137b76eb804c4b9a1c7c7dc562549306afad2", - "sha256:52ccb45e77a1085ec5461cde794e1aa037df79f473cbc69b974e73940655c8d7", - "sha256:5c3fbebd7de20ce93103cb3183b47671f2885307df4a17a0ad56a1dd51273d36", - "sha256:5e5851969aea17660e55f6a3be00037a25b96a9b44d2083651812c99d53b14d1", - "sha256:5edfa27b2d3eefa2210fb2f5d539fbed81722b49f083b2c6566455eb7422fd7e", - "sha256:7d263e5770efddf465a9e31b78362d84d015cc894ca2c131901a4445eaa61ee1", - "sha256:83381342bfc22b3c8c06f2dd93a505413888694302de25add756254beee8449c", - "sha256:857eebb2c1dc60e4219ec8e98dfa19553dae33608237e107db9c6078b1167856", - "sha256:98e439297f78fca3a6169fd330fbe88d78b3bb72f967ad9961bcac0d7fdd1550", - "sha256:bf54103892a83c64db58125b3f2a43df6d2cb2d28889f14c78519394feb41492", - "sha256:d9ac82be533394d341b41d78aca7ed0e0f4ba5a2231602e2f05aa87f25c51672", - "sha256:e982fe07ede9fada6ff6705af70514a52beb1b2c3d25d4e873e82114cf3c5401", - "sha256:edce2ea7f3dfc981c4ddc97add8a61381d9642dc3273737e756517cc03e84dd6", - "sha256:efdc45ef1afc238db84cb4963aa689c0408912a0239b0721cb172b4016eb31d6", - "sha256:f137c02498f8b935892d5c0172560d7ab54bc45039de8805075e19079c639a9c", - "sha256:f82e347a72f955b7017a39708a3667f106e6ad4d10b25f237396a7115d8ed5fd", - "sha256:fb7c206e01ad85ce57feeaaa0bf784b97fa3cad0d4a5737bc5295785f5c613a1" - ], - "version": "==1.0" - }, - "modutil": { - "hashes": [ - "sha256:2c85c1666649e92e56de17c00e1e831313602d9b55e8661d39c01e39003b45f7", - "sha256:cc3dad264e36ed359fdd67c4588959d2996bd0402ad9c9d974ca906821537218" - ], - "markers": "python_version >= '3.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.0.0" + "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", + "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161", + "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235", + "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5", + "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff", + "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b", + "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1", + "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e", + "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183", + "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66", + "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1", + "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1", + "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e", + "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b", + "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905", + "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735", + "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d", + "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e", + "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d", + "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c", + "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21", + "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2", + "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5", + "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b", + "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6", + "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f", + "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f", + "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7" + ], + "markers": "python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.1.1" + }, + "mock": { + "hashes": [ + "sha256:83657d894c90d5681d62155c82bda9c1187827525880eda8ff5df4ec813437c3", + "sha256:d157e52d4e5b938c550f39eb2fd15610db062441a9c2747d3dbfa9298211d0f8" + ], + "markers": "python_version < '3.0' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==3.0.5" }, "more-itertools": { "hashes": [ - "sha256:c187a73da93e7a8acc0001572aebc7e3c69daf7bf6881a2cea10650bd4420092", - "sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e", - "sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d" + "sha256:409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832", + "sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==4.3.0" + "markers": "python_version < '3.8' and python_version >= '2.6' and python_version >= '3.4' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.8' and python_version >= '2.7' and python_version >= '3.4' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.6' and python_version >= '3.4' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version >= '3.4' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==7.2.0" + }, + "ordereddict": { + "hashes": [ + "sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f" + ], + "markers": "python_version < '3.0' and python_version < '3.3' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.1" + }, + "orderedmultidict": { + "hashes": [ + "sha256:04070bbb5e87291cc9bfa51df413677faf2141c73c61d2a5f7b26bea3cd882ad", + "sha256:43c839a17ee3cdd62234c47deca1a8508a3f2ca1d0678a3bf791c87cf84adbf3" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.0.1" + }, + "packagebuilder": { + "hashes": [ + "sha256:1e85c4e0e994322996b93cd6685c12834d30f3558889154f8e3de8fb1f3fd1e7", + "sha256:dc525d06ecd102db23ab421b879d7d27021d784ff933e33e8c411a53af5c9dbe" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.1.0" }, "ordereddict": { "hashes": [ @@ -777,19 +831,19 @@ }, "packaging": { "hashes": [ - "sha256:0886227f54515e592aaa2e5a553332c73962917f2831f1b0f9b9f4380a4b9807", - "sha256:f95a1e147590f204328170981833854229bb2912ac3d5f89e2a8ccd2834800c9" + "sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47", + "sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==18.0" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==19.2" }, "parver": { "hashes": [ - "sha256:b8b2976fd8a73a0515465b2a265fd9b20cc25a6dc88bc1154fd5f60f10dad4db", - "sha256:d9ae08a2629105fdb83e4971ae8a04f1de5a3803d1dd928f6e181aeadb398180" + "sha256:1b37a691af145a3a193eff269d53ba5b2ab16dfbb65d47d85360755919f5fe4b", + "sha256:72d056b8f8883ac90eef5554a9c8a47fac39d3b66479f3d2c8d5bc21b849cdba" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.2.0" + "version": "==0.2.1" }, "passa": { "editable": true, @@ -800,34 +854,35 @@ }, "pathlib2": { "hashes": [ - "sha256:25199318e8cc3c25dcb45cbe084cc061051336d5a9ea2a12448d3d8cb748f742", - "sha256:5887121d7f7df3603bca2f710e7219f3eca0eb69e0b7cc6e0a022e155ac931a7" + "sha256:2156525d6576d21c4dcaddfa427fae887ef89a7a9de5cbfe0728b3aafa78427e", + "sha256:446014523bb9be5c28128c4d2a10ad6bb60769e78bd85658fe44a450674e0ef8" ], - "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.3.3" + "markers": "python_version < '3.6' and python_version < '3.8' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.6' and python_version < '3.8' and python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.6' and python_version < '3.8' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.6' and python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.6' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.3.4" }, "pep517": { "hashes": [ - "sha256:cc663a438fdfe2e88d8d3c5ef2203ac858de34e31b6609b1fc505d611490a926", - "sha256:f79bb08fb064dfc5b141204bfeb56a4141a6d504677fab4723036a464fc25cc1" + "sha256:273345f4538306f6e4056d8bbced566e186ab4defc188cb3be3e413b5d255912", + "sha256:dde535e9a42de94f4cd941dbaa6feb0a4b5143ffd3906efea091c3826cb7d33d" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.3" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.6.0" }, "pip-shims": { "hashes": [ - "sha256:3bc24ec050a6b9eea35419467237e4f47eaf806dadc9999bf887355c377edea7", - "sha256:edb4cf3c509eab2f36b55c1ac1a59a4c485ccd537cc87934d74950880f641256" + "sha256:0162d846bd60c7b1feb4e1336541a3e661eb6a1eff4b9ea0d759780f8e0a2936", + "sha256:d73372b9fa3a10e73f057fced70d39bb82df16a1ee3ded027fed0bb8d7c0ff97" ], "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.3.2" + "version": "==0.3.3" }, "pkginfo": { "hashes": [ - "sha256:5878d542a4b3f237e359926384f1dde4e099c9f5525d236b1840cf704fa8d474", - "sha256:a39076cb3eb34c333a0dd390b568e9e1e881c7bf2cc0aee12120636816f55aee" + "sha256:7424f2c8511c186cd5424bbf31045b77435b37a8d604990b79d4e70d741148bb", + "sha256:a6d9e40ca61ad3ebd0b72fbadd4fba16e4c0e4df0428c041e01e06eb6ee71f32" ], - "version": "==1.4.2" + "markers": "python_version >= '3.6'", + "version": "==1.5.0.1" }, "plette": { "extras": [ @@ -842,19 +897,19 @@ }, "pluggy": { "hashes": [ - "sha256:447ba94990e8014ee25ec853339faf7b0fc8050cdc3289d4d71f7f410fb90095", - "sha256:bde19360a8ec4dfd8a20dcb811780a30998101f078fc7ded6162f0076f50508f" + "sha256:0db4b7601aae1d35b4a033282da476845aa19185c1e6964b25cf324b5e4ec3e6", + "sha256:fa5fa1622fa6dd5c030e9cad086fa19ef6a0cf6d7a2d12318e10cb49d6d68f34" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.8.0" + "markers": "python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.13.0" }, "py": { "hashes": [ - "sha256:bf92637198836372b520efcba9e020c330123be8ce527e535d185ed4b6f45694", - "sha256:e76826342cefe3c3d5f7e8ee4316b80d1dd8a300781612ddbc765c17ba25a6c6" + "sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa", + "sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.7.0" + "markers": "python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.8.0" }, "pycparser": { "hashes": [ @@ -864,42 +919,51 @@ }, "pygments": { "hashes": [ - "sha256:6301ecb0997a52d2d31385e62d0a4a4cf18d2f2da7054a5ddad5c366cd39cee7", - "sha256:82666aac15622bd7bb685a4ee7f6625dd716da3ef7473620c192c0168aae64fc" + "sha256:71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127", + "sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297" ], - "version": "==2.3.0" + "markers": "python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' or python_version >= '2.7' and python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==2.4.2" }, "pyparsing": { "hashes": [ - "sha256:40856e74d4987de5d01761a22d1621ae1c7f8774585acae358aa5c5936c6c90b", - "sha256:f353aab21fd474459d97b709e527b5571314ee5f067441dc9f88e33eecd96592" + "sha256:6f98a7b9397e206d78cc01df10131398f1c8b8510a2f4d97d9abd82e1aacdd80", + "sha256:d9338df12903bbf5d65a0e4e87c2161968b10d2e489652bb47001d82a9b028b4" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.3.0" + "markers": "python_version >= '2.6' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.4.2" }, "pytest": { "hashes": [ - "sha256:1d131cc532be0023ef8ae265e2a779938d0619bb6c2510f52987ffcba7fa1ee4", - "sha256:ca4761407f1acc85ffd1609f464ca20bb71a767803505bd4127d0e45c5a50e23" + "sha256:813b99704b22c7d377bbd756ebe56c35252bb710937b46f207100e843440b3c2", + "sha256:cc6620b96bc667a0c8d4fa592a8c9c94178a1bd6cc799dbb057dfd9286d31a31" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==4.0.1" + "markers": "python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==5.1.3" }, "pytest-cov": { "hashes": [ - "sha256:513c425e931a0344944f84ea47f3956be0e416d95acbd897a44970c8d926d5d7", - "sha256:e360f048b7dae3f2f2a9a4d067b2dd6b6a015d384d1577c994a43f3f7cbad762" + "sha256:2b097cde81a302e1047331b48cadacf23577e431b61e9c6f49a1170bbe3d3da6", + "sha256:e00ea4fdde970725482f1f35630d12f074e121a23801aabf2ae154ec6bdd343a" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.6.0" + "version": "==2.7.1" }, "pytest-forked": { "hashes": [ - "sha256:e4500cd0509ec4a26535f7d4112a8cc0f17d3a41c29ffd4eab479d2a55b30805", - "sha256:f275cb48a73fc61a6710726348e1da6d68a978f0ec0c54ece5a5fae5977e5a08" + "sha256:5fe33fbd07d7b1302c95310803a5e5726a4ff7f19d5a542b7ce57c76fed8135f", + "sha256:d352aaced2ebd54d42a65825722cb433004b4446ab5d2044851d9cc7a00c9e38" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.2" + "version": "==1.0.2" + }, + "pytest-mock": { + "hashes": [ + "sha256:43ce4e9dd5074993e7c021bb1c22cbb5363e612a2b5a76bc6d956775b10758b7", + "sha256:5bf5771b1db93beac965a7347dc81c675ec4090cb841e49d9d34637a25c30568" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.10.4" }, "pytest-timeout": { "hashes": [ @@ -911,32 +975,34 @@ }, "pytest-xdist": { "hashes": [ - "sha256:5e8b68466c057f0f37e36909612f8838e518ce703c8da31f85e47c7dea8acc93", - "sha256:909bb938bdb21e68a28a8d58c16a112b30da088407b678633efb01067e3923de" + "sha256:3489d91516d7847db5eaecff7a2e623dba68984835dbe6cedb05ae126c4fb17f", + "sha256:501795cb99e567746f30fe78850533d4cd500c93794128e6ab9988e92a17b1f8" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.24.1" + "version": "==1.29.0" }, "pytoml": { "hashes": [ - "sha256:ca2d0cb127c938b8b76a9a0d0f855cf930c1d50cc3a0af6d3595b566519a1013" + "sha256:57a21e6347049f73bfb62011ff34cd72774c031b9828cb628a752225136dfc33", + "sha256:8eecf7c8d0adcff3b375b09fe403407aa9b645c499e5ab8cac670ac4a35f61e7" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.1.20" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.1.21" }, "pytz": { "hashes": [ - "sha256:31cb35c89bd7d333cd32c5f278fca91b523b0834369e757f4c5641ea252236ca", - "sha256:8e0f8568c118d3077b46be7d654cc8167fa916092e28320cde048e54bfc9f1e6" + "sha256:26c0b32e437e54a18161324a2fca3c4b9846b74a8dccddd843113109e1116b32", + "sha256:c894d57500a4cd2d5c71114aaab77dbab5eabd9022308ce5ac9bb93a60a6f0c7" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2018.7" + "markers": "python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2019.2" }, "readme-renderer": { "hashes": [ "sha256:bb16f55b259f27f75f640acf5e00cf897845a8b3e4731b5c1a436e4b8529202f", "sha256:c8532b79afc0375a85f10433eca157d6b50f7d6990f337fa498c96cd4bfc203d" ], + "markers": "python_version >= '3.6'", "version": "==24.0" }, "recursive-monkey-patch": { @@ -949,26 +1015,27 @@ }, "requests": { "hashes": [ - "sha256:65b3a120e4329e33c9889db89c80976c5272f56ea92d3e74da8a463992e3ff54", - "sha256:ea881206e59f41dbd0bd445437d792e43906703fff75ca8ff43ccdb11f33f263" + "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", + "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.20.1" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' or python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' or python_version >= '2.7' and python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==2.22.0" }, "requests-toolbelt": { "hashes": [ - "sha256:42c9c170abc2cacb78b8ab23ac957945c7716249206f90874651971a4acff237", - "sha256:f6a531936c6fa4c6cfce1b9c10d5c4f498d16528d2a54a22ca00011205a187b5" + "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f", + "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0" ], - "version": "==0.8.0" + "markers": "python_version >= '3.6'", + "version": "==0.9.1" }, "requirementslib": { "hashes": [ - "sha256:c2c00c7bd3bd4984c97d10cd4d143efbe33b5ed9e55961bea30ca7a9a4927289", - "sha256:dc6b692e8dee03d6e90c29db1e337b0bf8152cce84a57f0fb4765e596afde4e0" + "sha256:50731ac1052473e4c7df59a44a1f3aa20f32e687110bc05d73c3b4109eebc23d", + "sha256:8b594ab8b6280ee97cffd68fc766333345de150124d5b76061dd575c3a21fe5a" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.3.3" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.5.3" }, "resolvelib": { "hashes": [ @@ -980,56 +1047,98 @@ }, "scandir": { "hashes": [ - "sha256:04b8adb105f2ed313a7c2ef0f1cf7aff4871aa7a1883fa4d8c44b5551ab052d6", - "sha256:1444134990356c81d12f30e4b311379acfbbcd03e0bab591de2696a3b126d58e", - "sha256:1b5c314e39f596875e5a95dd81af03730b338c277c54a454226978d5ba95dbb6", - "sha256:346619f72eb0ddc4cf355ceffd225fa52506c92a2ff05318cfabd02a144e7c4e", - "sha256:44975e209c4827fc18a3486f257154d34ec6eaec0f90fef0cca1caa482db7064", - "sha256:61859fd7e40b8c71e609c202db5b0c1dbec0d5c7f1449dec2245575bdc866792", - "sha256:a5e232a0bf188362fa00123cc0bb842d363a292de7126126df5527b6a369586a", - "sha256:c14701409f311e7a9b7ec8e337f0815baf7ac95776cc78b419a1e6d49889a383", - "sha256:c7708f29d843fc2764310732e41f0ce27feadde453261859ec0fca7865dfc41b", - "sha256:c9009c527929f6e25604aec39b0a43c3f831d2947d89d6caaab22f057b7055c8", - "sha256:f5c71e29b4e2af7ccdc03a020c626ede51da471173b4a6ad1e904f2b2e04b4bd" - ], - "markers": "python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.9.0" + "sha256:2586c94e907d99617887daed6c1d102b5ca28f1085f90446554abf1faf73123e", + "sha256:2ae41f43797ca0c11591c0c35f2f5875fa99f8797cb1a1fd440497ec0ae4b022", + "sha256:2b8e3888b11abb2217a32af0766bc06b65cc4a928d8727828ee68af5a967fa6f", + "sha256:2c712840c2e2ee8dfaf36034080108d30060d759c7b73a01a52251cc8989f11f", + "sha256:4d4631f6062e658e9007ab3149a9b914f3548cb38bfb021c64f39a025ce578ae", + "sha256:67f15b6f83e6507fdc6fca22fedf6ef8b334b399ca27c6b568cbfaa82a364173", + "sha256:7d2d7a06a252764061a020407b997dd036f7bd6a175a5ba2b345f0a357f0b3f4", + "sha256:8c5922863e44ffc00c5c693190648daa6d15e7c1207ed02d6f46a8dcc2869d32", + "sha256:92c85ac42f41ffdc35b6da57ed991575bdbe69db895507af88b9f499b701c188", + "sha256:b24086f2375c4a094a6b51e78b4cf7ca16c721dcee2eddd7aa6494b42d6d519d", + "sha256:cb925555f43060a1745d0a321cca94bcea927c50114b623d73179189a4e100ac" + ], + "markers": "python_version < '3.5' and python_version < '3.6' and python_version < '3.8' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version < '3.6' and python_version < '3.8' and python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version < '3.6' and python_version < '3.8' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version < '3.6' and python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version < '3.6' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.10.0" }, "six": { "hashes": [ - "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", - "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" ], - "version": "==1.11.0" + "markers": "python_version < '3.0' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.6' and python_version < '3.8' and python_version >= '2.6' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.6' and python_version < '3.8' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.6' and python_version >= '2.6' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.6' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.6' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.6' and python_version >= '3.6' and python_version not in '3.0, 3.1' or python_version >= '2.6' and python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.12.0" }, "snowballstemmer": { "hashes": [ - "sha256:919f26a68b2c17a7634da993d91339e288964f93c274f1343e3bbbe2096e1128", - "sha256:9f3bcd3c401c3e862ec0ebe6d2c069ebc012ce142cce209c098ccb5b09136e89" + "sha256:713e53b79cbcf97bc5245a06080a33d54a77e7cce2f789c835a143bcdb5c033e" ], - "version": "==1.2.1" + "markers": "python_version >= '3.5'", + "version": "==1.9.1" }, "sphinx": { "hashes": [ - "sha256:120732cbddb1b2364471c3d9f8bfd4b0c5b550862f99a65736c77f970b142aea", - "sha256:b348790776490894e0424101af9c8413f2a86831524bd55c5f379d3e3e12ca64" + "sha256:0d586b0f8c2fc3cc6559c5e8fd6124628110514fda0e5d7c82e682d749d2e845", + "sha256:839a3ed6f6b092bb60f492024489cc9e6991360fb9f52ed6361acd510d261069" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.8.2" + "markers": "python_version >= '3.5'", + "version": "==2.2.0" }, "sphinx-rtd-theme": { "hashes": [ - "sha256:02f02a676d6baabb758a20c7a479d58648e0f64f13e07d1b388e9bb2afe86a09", - "sha256:d0f6bc70f98961145c5b0e26a992829363a197321ba571b31b24ea91879e0c96" + "sha256:00cf895504a7895ee433807c62094cf1e95f065843bf3acd17037c3e9a2becd4", + "sha256:728607e34d60456d736cc7991fd236afb828b21b82f956c5ea75f94c8414040a" ], - "version": "==0.4.2" + "version": "==0.4.3" }, - "sphinxcontrib-websupport": { + "sphinxcontrib-applehelp": { "hashes": [ - "sha256:68ca7ff70785cbe1e7bccc71a48b5b6d965d79ca50629606c7861a21b206d9dd", - "sha256:9de47f375baf1ea07cdb3436ff39d7a9c76042c10a769c52353ec46e4e8fc3b9" + "sha256:edaa0ab2b2bc74403149cb0209d6775c96de797dfd5b5e2a71981309efab3897", + "sha256:fb8dee85af95e5c30c91f10e7eb3c8967308518e0f7488a2828ef7bc191d0d5d" ], - "version": "==1.1.0" + "markers": "python_version >= '3.5'", + "version": "==1.0.1" + }, + "sphinxcontrib-devhelp": { + "hashes": [ + "sha256:6c64b077937330a9128a4da74586e8c2130262f014689b4b89e2d08ee7294a34", + "sha256:9512ecb00a2b0821a146736b39f7aeb90759834b07e81e8cc23a9c70bacb9981" + ], + "markers": "python_version >= '3.5'", + "version": "==1.0.1" + }, + "sphinxcontrib-htmlhelp": { + "hashes": [ + "sha256:4670f99f8951bd78cd4ad2ab962f798f5618b17675c35c5ac3b2132a14ea8422", + "sha256:d4fd39a65a625c9df86d7fa8a2d9f3cd8299a3a4b15db63b50aac9e161d8eff7" + ], + "markers": "python_version >= '3.5'", + "version": "==1.0.2" + }, + "sphinxcontrib-jsmath": { + "hashes": [ + "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178", + "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8" + ], + "markers": "python_version >= '3.5'", + "version": "==1.0.1" + }, + "sphinxcontrib-qthelp": { + "hashes": [ + "sha256:513049b93031beb1f57d4daea74068a4feb77aa5630f856fcff2e50de14e9a20", + "sha256:79465ce11ae5694ff165becda529a600c754f4bc459778778c7017374d4d406f" + ], + "markers": "python_version >= '3.5'", + "version": "==1.0.2" + }, + "sphinxcontrib-serializinghtml": { + "hashes": [ + "sha256:c0efb33f8052c04fd7a26c0a07f1678e8512e0faec19f4aa8f2473a8b81d5227", + "sha256:db6615af393650bf1151a6cd39120c29abaf93cc60db8c48eb2dddbfdc3a9768" + ], + "markers": "python_version >= '3.5'", + "version": "==1.1.3" }, "toml": { "hashes": [ @@ -1048,42 +1157,43 @@ }, "towncrier": { "hashes": [ - "sha256:3c0da1de042861df9030c0608d346d55cabe14cfa7cf04045f22b0af63eb8aa7", - "sha256:cfde66ada782db9269407eaefb38562d3d2a4bc4d7647c0b2197ed184b869aae" + "sha256:d0c49060522e4b45d02c8ffe0b63e5ffa660830f681446945212e5c66314228b", + "sha256:f0375efce604977b4098c18dc805d5ce0de59b232db8374cb3ca98b078b5afae" ], - "version": "==18.6.0" + "version": "==19.9.0rc1" }, "tqdm": { "hashes": [ - "sha256:3c4d4a5a41ef162dd61f1edb86b0e1c7859054ab656b2e7c7b77e7fbf6d9f392", - "sha256:5b4d5549984503050883bc126280b386f5f4ca87e6c023c5d015655ad75bdebb" + "sha256:abc25d0ce2397d070ef07d8c7e706aede7920da163c64997585d42d3537ece3d", + "sha256:dd3fcca8488bb1d416aa7469d2f277902f26260c45aa86b667b074cd44b3b115" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1'", - "version": "==4.28.1" + "markers": "python_version >= '2.6' and python_version >= '3.6' and python_version not in '3.0, 3.1'", + "version": "==4.36.1" }, "twine": { "hashes": [ - "sha256:7d89bc6acafb31d124e6e5b295ef26ac77030bf098960c2a4c4e058335827c5c", - "sha256:fad6f1251195f7ddd1460cb76d6ea106c93adb4e56c41e0da79658e56e547d2c" + "sha256:5319dd3e02ac73fcddcd94f035b9631589ab5d23e1f4699d57365199d85261e1", + "sha256:9fe7091715c7576df166df8ef6654e61bada39571783f2fd415bdcba867c6993" ], - "version": "==1.12.1" + "markers": "python_version >= '3.6'", + "version": "==2.0.0" }, "typing": { "hashes": [ - "sha256:4027c5f6127a6267a435201981ba156de91ad0d1d98e9ddc2aa173453453492d", - "sha256:57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4", - "sha256:a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a" + "sha256:91dfe6f3f706ee8cc32d38edbbf304e9b7583fb37108fef38229617f8b3eba23", + "sha256:c8cabb5ab8945cd2f54917be357d134db9cc1eb039e59d1606dc1e60cb1d9d36", + "sha256:f38d83c5a7a7086543a0f649564d661859c5146a85775ab90c0d2f93ffaa9714" ], - "markers": "python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==3.6.6" + "markers": "python_version < '3.5' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.5' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==3.7.4.1" }, "urllib3": { "hashes": [ - "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", - "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" + "sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398", + "sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86" ], - "markers": "python_version < '4' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '4' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.24.1" + "markers": "python_version < '4' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' or python_version < '4' and python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' or python_version < '4' and python_version >= '2.7' and python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' or python_version < '4' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==1.25.6" }, "virtualenv": { "hashes": [ @@ -1098,34 +1208,46 @@ "spinner" ], "hashes": [ - "sha256:3a1020fb7be000b268af96641ced9ead844b1f75840c41e20e473647688fc630", - "sha256:6d2005ad670f77bd9c9b5415c4e2a4a20dce5b0cf0e0d11598eb463b2e0ebe44" + "sha256:2166e3148a67c438c9e3edbba0cde153d42dec6e3bf5d8f4624feb27686c0990", + "sha256:3a0529b4b6c2e842fd19b5ceaa95b6c9201321314825c110406d4af3331a0709" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.2.5" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.4.3" + }, + "wcwidth": { + "hashes": [ + "sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e", + "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c" + ], + "markers": "python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.1.7" }, "webencodings": { "hashes": [ "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78", "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923" ], + "markers": "python_version >= '2.7' and python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.5.1" }, "wheel": { "hashes": [ - "sha256:029703bf514e16c8271c3821806a1c171220cc5bdd325cbf4e7da1e056a01db6", - "sha256:1e53cdb3f808d5ccd0df57f964263752aa74ea7359526d3da6c02114ec1e1d44" + "sha256:10c9da68765315ed98850f8e048347c3eb06dd81822dc2ab1d4fde9dc9702646", + "sha256:f4da1763d3becf2e2cd92a14a7c920f0f00eca30fdde9ea992c836685b9faf28" ], "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.32.3" + "version": "==0.33.6" }, "yaspin": { + "file": "https://github.com/sarugaku/yaspin/releases/download/v0.15.0post1/yaspin-0.15.0.post1-py2.py3-none-any.whl" + }, + "zipp": { "hashes": [ - "sha256:36fdccc5e0637b5baa8892fe2c3d927782df7d504e9020f40eb2c1502518aa5a", - "sha256:8e52bf8079a48e2a53f3dfeec9e04addb900c101d1591c85df69cf677d3237e7" + "sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e", + "sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.14.0" + "markers": "python_version < '3.8' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.8' and python_version >= '2.7' and python_version >= '3.5' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version < '3.8' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.6.0" } } } diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index a9ecca6..0000000 --- a/appveyor.yml +++ /dev/null @@ -1,22 +0,0 @@ -branches: - only: - - master - -install: - - "SET PATH=C:\\Python36-x64;%PATH%" - - "python --version" - - "python -m pip install --upgrade pip setuptools pytest-timeout pytest-xdist" - - "python -m pip install --upgrade -e .[pack,tests,virtualenv]" - -build_script: - - "python -m invoke pack" - -test_script: - # Shorten paths, workaround https://bugs.python.org/issue18199 - - "subst T: %TEMP%" - - "set TEMP=T:\\" - - "set TMP=T:\\" - - "python -m pytest -v -n 8 tests" - -artifacts: - - path: ".\\pack\\passa.zip" diff --git a/news/66.bugfix.rst b/news/66.bugfix.rst new file mode 100644 index 0000000..de50275 --- /dev/null +++ b/news/66.bugfix.rst @@ -0,0 +1 @@ +Fix a bug that editable packages are not installed and locked properly. diff --git a/news/66.feature.rst b/news/66.feature.rst new file mode 100644 index 0000000..b33d425 --- /dev/null +++ b/news/66.feature.rst @@ -0,0 +1,6 @@ +Improve the integration testing +* Introduce ``pytest-pypi`` to mock the PyPI server for testing. +* Refactor ``synchronizers.py`` to make it easy to mock installation and uninstallation operations. +* Mock the VCS downloads for testing. +* Improve the performance of metadata resolving by deduplicating the metasests items. +* Switch to Github Pages as CI service. diff --git a/setup.cfg b/setup.cfg index 51bc08d..04bbb40 100644 --- a/setup.cfg +++ b/setup.cfg @@ -43,14 +43,13 @@ install_requires = installer packagebuilder packaging - pip-shims>=0.1.2 + pip-shims>=0.3.1 plette[validation]>=0.2.2 recursive-monkey-patch requests - requirementslib>=1.1.1 + requirementslib>=1.1.7 resolvelib>=0.2.1,!=1.0.0.dev0 six - virtualenv vistir[spinner]>=0.1.4 [options.extras_require] @@ -63,6 +62,8 @@ tests = pytest-xdist pytest-timeout pytest-cov + pytest-mock + virtualenv pytest [options.entry_points] @@ -97,7 +98,7 @@ ignore = # E231: missing whitespace after ',' # E402: module level import not at top of file # E501: line too long - E127,E128,E129,E222,E231,E402,E501 + E231,E402,E501,W503 [tool:pytest] strict = true @@ -110,3 +111,8 @@ filterwarnings = [build-system] requires = ["setuptools", "wheel"] + +[mypy] +ignore_missing_imports=true +follow_imports=skip +python_version=2.7 diff --git a/src/passa/actions/add.py b/src/passa/actions/add.py index 49e4bce..19471de 100644 --- a/src/passa/actions/add.py +++ b/src/passa/actions/add.py @@ -47,8 +47,7 @@ def add_packages(packages=[], editables=[], project=None, dev=False, sync=False, develop = any(lockfile_diff.develop) syncer = Synchronizer( - project, default=default, develop=develop, - clean_unneeded=clean, venv=getattr(project, "venv", None) + project, default=default, develop=develop, clean_unneeded=clean ) success = sync(syncer) if not success: diff --git a/src/passa/actions/clean.py b/src/passa/actions/clean.py index 9006f22..d7def19 100644 --- a/src/passa/actions/clean.py +++ b/src/passa/actions/clean.py @@ -4,12 +4,14 @@ def clean(project, default=True, dev=False, sync=True): - from passa.models.synchronizers import Cleaner + from passa.models.synchronizers import Synchronizer from passa.operations.sync import clean - cleaner = Cleaner(project, default=default, develop=dev, sync=sync) + syncer = Synchronizer( + project, default=default, develop=dev, clean_unneeded=True, dry_run=not sync + ) - success = clean(cleaner) + success = clean(syncer) if not success: return 1 diff --git a/src/passa/actions/remove.py b/src/passa/actions/remove.py index 92f6168..ef6006e 100644 --- a/src/passa/actions/remove.py +++ b/src/passa/actions/remove.py @@ -27,11 +27,11 @@ def remove(project=None, only="default", packages=[], clean=True, sync=False): if not clean: return - from passa.models.synchronizers import Cleaner + from passa.models.synchronizers import Synchronizer from passa.operations.sync import clean - cleaner = Cleaner(project, default=True, develop=True) - success = clean(cleaner) + syncer = Synchronizer(project, default=True, develop=True, clean_unneeded=True) + success = clean(syncer) if not success: return 1 diff --git a/src/passa/actions/sync.py b/src/passa/actions/sync.py index 23e36ee..2a4b69d 100644 --- a/src/passa/actions/sync.py +++ b/src/passa/actions/sync.py @@ -10,7 +10,7 @@ def sync(project=None, dev=False, clean=True): project = project syncer = Synchronizer( project, default=True, develop=dev, - clean_unneeded=clean, + clean_unneeded=clean ) success = sync(syncer) diff --git a/src/passa/cli/add.py b/src/passa/cli/add.py index 2635b98..cb8d651 100644 --- a/src/passa/cli/add.py +++ b/src/passa/cli/add.py @@ -21,7 +21,8 @@ def run(self, options): editables=options.editables, project=options.project, dev=options.dev, - clean=options.clean + clean=options.clean, + sync=options.sync ) diff --git a/src/passa/cli/install.py b/src/passa/cli/install.py index 1c0b459..b178655 100644 --- a/src/passa/cli/install.py +++ b/src/passa/cli/install.py @@ -15,7 +15,7 @@ class Command(BaseCommand): def run(self, options): return install(project=options.project, check=options.check, dev=options.dev, - clean=options.clean) + clean=options.clean) if __name__ == "__main__": diff --git a/src/passa/cli/options.py b/src/passa/cli/options.py index 0306d5b..b27f428 100644 --- a/src/passa/cli/options.py +++ b/src/passa/cli/options.py @@ -10,9 +10,10 @@ import tomlkit.exceptions import passa.models.projects -import mork import vistir +from ..models.environments import Environment + PYTHON_VERSION = ".".join(str(v) for v in sys.version_info[:2]) @@ -21,23 +22,23 @@ class Project(passa.models.projects.Project): def __init__(self, root, *args, **kwargs): root = vistir.compat.Path(root).absolute() pipfile = root.joinpath("Pipfile") + environment = kwargs.pop("environment", self.get_env()) if not pipfile.is_file(): raise argparse.ArgumentError( - "project", "{0!r} is not a Pipfile project".format(root), + None, "{0!r} is not a Pipfile project".format(root), ) - self.venv = kwargs.pop("venv", self.get_venv(root)) try: - super(Project, self).__init__(root.as_posix(), env_prefix=self.venv.prefix, - *args, **kwargs) + super(Project, self).__init__(root.as_posix(), environment=environment, + *args, **kwargs) except tomlkit.exceptions.ParseError as e: raise argparse.ArgumentError( - "project", "failed to parse Pipfile: {0!r}".format(str(e)), + None, "failed to parse Pipfile: {0!r}".format(str(e)), ) - def get_venv(self, root): + def get_env(self): if 'VIRTUAL_ENV' in os.environ: - return mork.VirtualEnv(os.environ['VIRTUAL_ENV']) - return mork.VirtualEnv.from_project_path(root) + return Environment(prefix=os.environ['VIRTUAL_ENV'], is_venv=True) + return Environment() def __name__(self): return "Project Root" diff --git a/src/passa/cli/remove.py b/src/passa/cli/remove.py index 041c195..055ec59 100644 --- a/src/passa/cli/remove.py +++ b/src/passa/cli/remove.py @@ -15,7 +15,7 @@ class Command(BaseCommand): def run(self, options): return remove(project=options.project, only=options.only, - packages=options.packages, clean=options.clean, sync=options.sync) + packages=options.packages, clean=options.clean, sync=options.sync) if __name__ == "__main__": diff --git a/src/passa/cli/upgrade.py b/src/passa/cli/upgrade.py index c7696c2..04b12a7 100644 --- a/src/passa/cli/upgrade.py +++ b/src/passa/cli/upgrade.py @@ -14,7 +14,7 @@ class Command(BaseCommand): def run(self, options): return upgrade(project=options.project, strategy=options.strategy, - sync=options.sync, packages=options.packages) + sync=options.sync, packages=options.packages) if __name__ == "__main__": diff --git a/src/passa/internals/_pip.py b/src/passa/internals/_pip.py index 8146df3..3f46ef6 100644 --- a/src/passa/internals/_pip.py +++ b/src/passa/internals/_pip.py @@ -3,10 +3,11 @@ from __future__ import absolute_import, unicode_literals import contextlib +import distutils.log import io import itertools -import distutils.log import os +import re import distlib.database import distlib.metadata @@ -15,14 +16,13 @@ import packaging.utils import pip_shims import six -import sys -import sysconfig import vistir from ..models.caches import CACHE_DIR -from ._pip_shims import ( - SETUPTOOLS_SHIM, VCS_SUPPORT, build_wheel as _build_wheel, unpack_url -) +from ..models.environments import Environment +from ._pip_shims import SETUPTOOLS_SHIM +from ._pip_shims import build_wheel as _build_wheel +from ._pip_shims import unpack_url from .utils import filter_sources @@ -91,21 +91,21 @@ def _get_pip_session(trusted_hosts): options, _ = cmd.parser.parse_args([]) options.cache_dir = CACHE_DIR options.trusted_hosts = trusted_hosts - session = cmd._build_session(options) - return session + return cmd._build_session(options) +@contextlib.contextmanager def _get_finder(sources): index_urls, trusted_hosts = _get_pip_index_urls(sources) - session = _get_pip_session(trusted_hosts) - finder = pip_shims.PackageFinder( - find_links=[], - index_urls=index_urls, - trusted_hosts=trusted_hosts, - allow_all_prereleases=True, - session=session, - ) - return finder + with contextlib.closing(_get_pip_session(trusted_hosts)) as session: + finder = pip_shims.PackageFinder( + find_links=[], + index_urls=index_urls, + trusted_hosts=trusted_hosts, + allow_all_prereleases=True, + session=session, + ) + yield finder def _get_wheel_cache(): @@ -152,77 +152,68 @@ def build_wheel(ireq, sources, hashes=None): `RuntimeError` subclass) if the wheel cannot be built. """ kwargs = _prepare_wheel_building_kwargs(ireq) - finder = _get_finder(sources) - - # Not for upgrade, hash not required. Hashes are not required here even - # when we provide them, because pip skips local wheel cache if we set it - # to True. Hashes are checked later if we need to download the file. - ireq.populate_link(finder, False, False) - - # Ensure ireq.source_dir is set. - # This is intentionally set to build_dir, not src_dir. Comments from pip: - # [...] if filesystem packages are not marked editable in a req, a non - # deterministic error occurs when the script attempts to unpack the - # build directory. - # Also see comments in `_prepare_wheel_building_kwargs()` -- If the ireq - # is editable, build_dir is actually src_dir, making the build in-place. - ireq.ensure_has_source_dir(kwargs["build_dir"]) - - # Ensure the source is fetched. For wheels, it is enough to just download - # because we'll use them directly. For an sdist, we need to unpack so we - # can build it. - if not ireq.editable or not pip_shims.is_file_url(ireq.link): - if ireq.is_wheel: - only_download = True - download_dir = kwargs["wheel_download_dir"] - else: - only_download = False - download_dir = kwargs["download_dir"] - ireq.options["hashes"] = _convert_hashes(hashes) - unpack_url( - ireq.link, ireq.source_dir, download_dir, - only_download=only_download, session=finder.session, - hashes=ireq.hashes(False), progress_bar="off", - ) - - if ireq.is_wheel: - # If this is a wheel, use the downloaded thing. - output_dir = kwargs["wheel_download_dir"] - wheel_path = os.path.join(output_dir, ireq.link.filename) - else: - # Othereise we need to build an ephemeral wheel. - wheel_path = _build_wheel( - ireq, vistir.path.create_tracked_tempdir(prefix="ephem"), - finder, _get_wheel_cache(), kwargs, - ) - if wheel_path is None or not os.path.exists(wheel_path): - raise WheelBuildError - return distlib.wheel.Wheel(wheel_path) + with _get_finder(sources) as finder: + # Not for upgrade, hash not required. Hashes are not required here even + # when we provide them, because pip skips local wheel cache if we set it + # to True. Hashes are checked later if we need to download the file. + ireq.populate_link(finder, False, False) + + # Ensure ireq.source_dir is set. + # This is intentionally set to build_dir, not src_dir. Comments from pip: + # [...] if filesystem packages are not marked editable in a req, a non + # deterministic error occurs when the script attempts to unpack the + # build directory. + # Also see comments in `_prepare_wheel_building_kwargs()` -- If the ireq + # is editable, build_dir is actually src_dir, making the build in-place. + ireq.ensure_has_source_dir(kwargs["build_dir"]) + + # Ensure the source is fetched. For wheels, it is enough to just download + # because we'll use them directly. For an sdist, we need to unpack so we + # can build it. + if not ireq.editable or not pip_shims.is_file_url(ireq.link): + if ireq.is_wheel: + only_download = True + download_dir = kwargs["wheel_download_dir"] + else: + only_download = False + download_dir = kwargs["download_dir"] + ireq.options["hashes"] = _convert_hashes(hashes) + unpack_url( + ireq.link, ireq.source_dir, download_dir, + only_download=only_download, session=finder.session, + hashes=ireq.hashes(False), progress_bar="off", + ) -def _obtrain_ref(vcs_obj, src_dir, name, rev=None): - target_dir = os.path.join(src_dir, name) - target_rev = vcs_obj.make_rev_options(rev) - if not os.path.exists(target_dir): - vcs_obj.obtain(target_dir) - if (not vcs_obj.is_commit_id_equal(target_dir, rev) and - not vcs_obj.is_commit_id_equal(target_dir, target_rev)): - vcs_obj.update(target_dir, target_rev) - return vcs_obj.get_revision(target_dir) + if ireq.is_wheel: + # If this is a wheel, use the downloaded thing. + output_dir = kwargs["wheel_download_dir"] + wheel_path = os.path.join(output_dir, ireq.link.filename) + elif ireq.editable: + # For editable sdist, only produce egg_info and raise. + # TODO: support pep517 builds. + ireq.run_egg_info() + raise WheelBuildError + else: + # Othereise we need to build an ephemeral wheel. + wheel_path = _build_wheel( + ireq, vistir.path.create_tracked_tempdir(prefix="ephem"), + finder, _get_wheel_cache(), kwargs, + ) + if wheel_path is None or not os.path.exists(wheel_path): + raise WheelBuildError + return distlib.wheel.Wheel(wheel_path) def get_vcs_ref(requirement): - backend = VCS_SUPPORT.get_backend(requirement.vcs) - vcs = backend(url=requirement.req.vcs_uri) - src = _get_src_dir() - name = requirement.normalized_name - ref = _obtrain_ref(vcs, src, name, rev=requirement.req.ref) - return ref + return requirement.commit_hash def find_installation_candidates(ireq, sources): - finder = _get_finder(sources) - return finder.find_all_candidates(ireq.name) + candidates = [] + with _get_finder(sources) as finder: + candidates = finder.find_all_candidates(ireq.name) + return candidates class RequirementUninstaller(object): @@ -231,17 +222,24 @@ class RequirementUninstaller(object): This uses `UninstallPathSet` to control the workflow. If the inner block exits correctly, the uninstallation is committed, otherwise rolled back. """ - def __init__(self, ireq, auto_confirm, verbose): + def __init__(self, ireq, auto_confirm, verbose, env=None): self.ireq = ireq self.pathset = None self.auto_confirm = auto_confirm self.verbose = verbose + self.env = env if env else Environment() + + def check_permitted(self, pathset, path): + if self.env.is_venv and self.env.is_installed(self.ireq.name): + return True + return pathset._permitted(path) def __enter__(self): self.pathset = self.ireq.uninstall( auto_confirm=self.auto_confirm, verbose=self.verbose, ) + self.pathset._permitted = self.check_permitted return self.pathset def __exit__(self, exc_type, exc_value, traceback): @@ -294,18 +292,67 @@ def install(self): pass -class VenvInstaller(NoopInstaller): +dist_info_re = re.compile(r"""^(?P(?P.+?)(-(?P.+?))?) + \.dist-info$""", re.VERBOSE) + + +def root_is_purelib(name, wheeldir): + """ + Return True if the extracted wheel in wheeldir should go into purelib. + """ + name_folded = name.replace("-", "_") + for item in os.listdir(wheeldir): + match = dist_info_re.match(item) + if match and match.group('name') == name_folded: + with open(os.path.join(wheeldir, item, 'WHEEL')) as wheel: + for line in wheel: + line = line.lower().rstrip() + if line == "root-is-purelib: true": + return True + return False + + +def get_entrypoints(filename): + import pkg_resources + if not os.path.exists(filename): + return {}, {} + + # This is done because you can pass a string to entry_points wrappers which + # means that they may or may not be valid INI files. The attempt here is to + # strip leading and trailing whitespace in order to make them valid INI + # files. + with open(filename) as fp: + data = io.StringIO() + for line in fp: + data.write(line.strip()) + data.write("\n") + data.seek(0) + + # get the entry points and then the script names + entry_points = pkg_resources.EntryPoint.parse_map(data) + console = entry_points.get('console_scripts', {}) + gui = entry_points.get('gui_scripts', {}) + + def _split_ep(s): + """get the string representation of EntryPoint, remove space and split + on '='""" + return str(s).replace(" ", "").split("=") + + # convert the EntryPoint objects into strings with module:function + console = dict(_split_ep(v) for v in console.values()) + gui = dict(_split_ep(v) for v in gui.values()) + return console, gui + + +class BaseInstaller(NoopInstaller): """Virtualenv-capable installer""" - def __init__(self, requirement, sources=None, paths=None, venv=None): + def __init__(self, requirement, sources=None, environment=None): self.ireq = requirement.as_ireq() self.sources = filter_sources(requirement, sources) self.hashes = requirement.hashes or None - self.paths = paths - self.venv = venv + self.environment = environment if environment else Environment() self.built = None - self.python = sys.executable if not self.venv else self.venv.python - self.py_version = self.venv.python_version if self.venv else sysconfig.get_python_version() self.metadata = None self.is_wheel = False @@ -328,12 +375,14 @@ def installation_args(self): setup_path = self.setup_dir.joinpath("setup.py") install_keys = ["headers", "purelib", "platlib", "scripts", "data"] install_args = [ - self.python, "-u", "-c", SETUPTOOLS_SHIM % setup_path.as_posix(), install_arg, - "--single-version-externally-managed", "--no-deps", - "--prefix={0}".format(self.paths["prefix"]) + self.environment.python, "-u", "-c", SETUPTOOLS_SHIM % setup_path.as_posix(), + install_arg, "--single-version-externally-managed", "--no-deps", + "--prefix={0}".format(self.environment.paths["prefix"]) ] for key in install_keys: - install_args.append("--install-{0}={1}".format(key, self.paths[key])) + install_args.append( + "--install-{0}={1}".format(key, self.environment.paths[key]) + ) return install_args def build_wheel(self): @@ -342,20 +391,22 @@ def build_wheel(self): self.is_wheel = True def build_sdist(self): - finder = _get_finder(self.sources) - self.ireq.populate_link(finder, False, False) - self.ireq.ensure_has_source_dir(self.src_dir) - self.built = get_sdist(self.ireq) - self.metadata = read_sdist_metadata(self.ireq) + with _get_finder(self.sources) as finder: + self.ireq.populate_link(finder, False, False) + self.ireq.ensure_has_source_dir(self.src_dir) + self.built = get_sdist(self.ireq) + self.metadata = read_sdist_metadata(self.ireq) def install_wheel(self): scripts = distlib.scripts.ScriptMaker(None, None) - self.built.install(self.paths, scripts) + self.built.install(self.environment.paths, scripts) def install_sdist(self): with vistir.cd(self.setup_dir.as_posix()), _suppress_distutils_logs(): - c = vistir.misc.run(self.installation_args, return_object=True, block=True, - nospin=True) + c = self.environment.run( + self.installation_args, return_object=True, block=True, nospin=True, + combine_stderr=False, write_to_stdout=False + ) if c.returncode != 0: err_text = "{0!r}: {1!r}".format(c.err, c.out) raise RuntimeError("Failed to install package: {0!r}".format(err_text)) @@ -365,14 +416,11 @@ def prepare(self): pass def install(self): - if self.venv: - with self.venv.activated(): - self._install() - else: + with self.environment.activated(): self._install() -class SdistInstaller(VenvInstaller): +class SdistInstaller(BaseInstaller): """Installer for SDists""" def __init__(self, *args, **kwargs): super(SdistInstaller, self).__init__(*args, **kwargs) diff --git a/src/passa/internals/_pip_shims.py b/src/passa/internals/_pip_shims.py index 112bbc6..8d72c2e 100644 --- a/src/passa/internals/_pip_shims.py +++ b/src/passa/internals/_pip_shims.py @@ -16,7 +16,7 @@ import recursive_monkey_patch -class LegacyMetadata(): +class LegacyMetadata(object): def set_metadata_version(self): metadata_version = self._fields.get("Metadata-Version") if metadata_version == "2.1": @@ -58,8 +58,8 @@ def _unpack_url_pre10(*args, **kwargs): return pip_shims.unpack_url(*args, **kwargs) -PIP_VERSION = pip_shims.utils._parse(pip_shims.pip_version) -VERSION_10 = pip_shims.utils._parse("10") +PIP_VERSION = pip_shims._parse(pip_shims.pip_version) +VERSION_10 = pip_shims._parse("10") VCS_SUPPORT = pip_shims.VcsSupport() diff --git a/src/passa/internals/dependencies.py b/src/passa/internals/dependencies.py index 6ad5629..a7f5b50 100644 --- a/src/passa/internals/dependencies.py +++ b/src/passa/internals/dependencies.py @@ -10,13 +10,12 @@ import packaging.utils import packaging.version import requests -import requirementslib import six +import requirementslib + from ..models.caches import DependencyCache, RequiresPythonCache -from ._pip import ( - WheelBuildError, build_wheel, get_sdist, read_sdist_metadata -) +from ._pip import WheelBuildError, build_wheel, read_sdist_metadata from .markers import contains_extra, get_contained_extras, get_without_extra from .utils import get_pinned_version, is_pinned @@ -142,20 +141,20 @@ def _get_dependencies_from_json(ireq, sources): if proc_url.endswith("/simple") ] - session = requests.session() - - for prefix in url_prefixes: - url = "{prefix}/pypi/{name}/{version}/json".format( - prefix=prefix, - name=packaging.utils.canonicalize_name(ireq.name), - version=version, - ) - try: - dependencies = _get_dependencies_from_json_url(url, session) - if dependencies is not None: - return dependencies - except Exception as e: - print("unable to read dependencies via {0} ({1})".format(url, e)) + with requests.session() as session: + + for prefix in url_prefixes: + url = "{prefix}/pypi/{name}/{version}/json".format( + prefix=prefix, + name=packaging.utils.canonicalize_name(ireq.name), + version=version, + ) + try: + dependencies = _get_dependencies_from_json_url(url, session) + if dependencies is not None: + return dependencies + except Exception as e: + print("unable to read dependencies via {0} ({1})".format(url, e)) return @@ -233,7 +232,7 @@ def _get_dependencies_from_pip(ireq, sources): # reached when it fails to build an sdist, where the sdist would have # been downloaded, extracted into `ireq.source_dir`, and partially # built (hopefully containing .egg-info). - built = get_sdist(ireq) + built = None metadata = read_sdist_metadata(ireq) if not metadata: raise @@ -261,7 +260,7 @@ def get_dependencies(requirement, sources): for getter in getters: try: result = getter(ireq) - except Exception as e: + except Exception: last_exc = sys.exc_info() continue if result is not None: diff --git a/src/passa/internals/markers.py b/src/passa/internals/markers.py index 3b2ce0e..6d8839c 100644 --- a/src/passa/internals/markers.py +++ b/src/passa/internals/markers.py @@ -102,9 +102,11 @@ def get_without_pyversion(marker): def _markers_collect_extras(markers, collection): # Optimization: the marker element is usually appended at the end. for el in reversed(markers): - if (isinstance(el, tuple) and - el[0].value == "extra" and - el[1].value == "=="): + if ( + isinstance(el, tuple) + and el[0].value == "extra" + and el[1].value == "==" + ): collection.add(el[2].value) elif isinstance(el, list): _markers_collect_extras(el, collection) diff --git a/src/passa/internals/specifiers.py b/src/passa/internals/specifiers.py index 40433b8..420df90 100644 --- a/src/passa/internals/specifiers.py +++ b/src/passa/internals/specifiers.py @@ -14,7 +14,7 @@ from vistir.misc import dedup six.add_move(six.MovedAttribute("Set", "collections", "collections.abc")) -from six.moves import reduce, Set +from six.moves import Set try: @@ -165,7 +165,7 @@ def fix_version_tuple(version_tuple): @lru_cache(maxsize=128) def get_versions(specset, group_by_operator=True): specs = [_get_specs(x) for x in list(tuple(specset))] - initial_sort_key = lambda k: (k[0], k[1]) + initial_sort_key = lambda k: (k[0], k[1]) # noqa initial_grouping_key = operator.itemgetter(0) if not group_by_operator: initial_grouping_key = operator.itemgetter(1) diff --git a/src/passa/internals/utils.py b/src/passa/internals/utils.py index 8f8e6fd..edd7cbe 100644 --- a/src/passa/internals/utils.py +++ b/src/passa/internals/utils.py @@ -1,9 +1,33 @@ # -*- coding=utf-8 -*- -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, unicode_literals, print_function + +import atexit +import os + +import requests + + +def is_type_checking(): + # type: () -> bool + try: + from typing import TYPE_CHECKING + except ImportError: + return False + return TYPE_CHECKING + + +MYPY_RUNNING = os.environ.get("MYPY_RUNNING", is_type_checking()) + + +if MYPY_RUNNING: + from typing import Any, List, Dict, Union # noqa + from requirementslib.models.requirements import Requirement # noqa + from pip_shims.shims import InstallRequirement # noqa def identify_requirment(r): + # type: (Requirement) -> str """Produce an identifier for a requirement to use in the resolver. Note that we are treating the same package with different extras as @@ -18,6 +42,7 @@ def identify_requirment(r): def get_pinned_version(ireq): + # type: (InstallRequirement) -> str """Get the pinned version of an InstallRequirement. An InstallRequirement is considered pinned if: @@ -60,6 +85,7 @@ def get_pinned_version(ireq): def is_pinned(ireq): + # type: (InstallRequirement) -> bool """Returns whether an InstallRequirement is a "pinned" requirement. An InstallRequirement is considered pinned if: @@ -83,6 +109,7 @@ def is_pinned(ireq): def filter_sources(requirement, sources): + # type: (Requirement, List[Dict[str, Union[bool, str]]]) -> List[Dict[str, Union[bool, str]]] """Returns a filtered list of sources for this requirement. This considers the index specified by the requirement, and returns only @@ -98,21 +125,36 @@ def filter_sources(requirement, sources): def get_allow_prereleases(requirement, global_setting): + # type: (Requirement, bool) -> bool # TODO: Implement per-package prereleases flag. (pypa/pipenv#1696) return global_setting def are_requirements_equal(this, that): + # type: (Requirement, Requirement) -> bool return ( - this.as_line(include_hashes=False) == - that.as_line(include_hashes=False) + this.as_line(include_hashes=False) + == that.as_line(include_hashes=False) ) def strip_extras(requirement): + # type: (Requirement) -> Requirement """Returns a new requirement object with extras removed. """ line = requirement.as_line() new = type(requirement).from_line(line) new.extras = None return new + + +REQUESTS_SESSIONS = [] # type: List[requests.Session] + + +def get_tracked_session(): + # type: () -> requests.Session + """Build a requests session and register it to be closed with interpreter exit""" + + session = requests.Session() + atexit.register(session.close) + return session diff --git a/src/passa/models/caches.py b/src/passa/models/caches.py index c6d29b5..f3589aa 100644 --- a/src/passa/models/caches.py +++ b/src/passa/models/caches.py @@ -7,28 +7,37 @@ import json import os import sys +import errno import appdirs import pip_shims -import requests import vistir from ..internals._pip_shims import VCS_SUPPORT -from ..internals.utils import get_pinned_version +from ..internals.utils import get_pinned_version, get_tracked_session, MYPY_RUNNING -CACHE_DIR = os.environ.get("PASSA_CACHE_DIR", appdirs.user_cache_dir("passa")) +if MYPY_RUNNING: + import requests # noqa + from typing import Optional # noqa + + +CACHE_DIR = os.environ.get("PASSA_CACHE_DIR", appdirs.user_cache_dir("passa")) # type: ignore class HashCache(pip_shims.SafeFileCache): + """Caches hashes of PyPI artifacts so we do not need to re-download them. Hashes are only cached when the URL appears to contain a hash in it and the cache key includes the hash value returned from the server). This ought to avoid ssues where the location on the server changes. """ + def __init__(self, *args, **kwargs): - session = kwargs.pop('session', requests.session()) + session = kwargs.pop('session', None) # type: requests.Session + if session is None: + session = get_tracked_session() self.session = session kwargs.setdefault('directory', os.path.join(CACHE_DIR, 'hash-cache')) super(HashCache, self).__init__(*args, **kwargs) @@ -76,6 +85,7 @@ def __str__(self): def _key_from_req(req): """Get an all-lowercase version of the requirement's name.""" + if hasattr(req, 'key'): # from pkg_resources, such as installed dists for pip-sync key = req.key @@ -100,6 +110,7 @@ def _read_cache_file(cache_file_path): class _JSONCache(object): + """A persistent cache backed by a JSON file. The cache file is written to the appropriate user cache dir for the @@ -109,10 +120,16 @@ class _JSONCache(object): Where X.Y indicates the Python version. """ - filename_format = None + + filename_format = None # type: Optional[str] def __init__(self, cache_dir=CACHE_DIR): - vistir.mkdir_p(cache_dir) + try: + vistir.mkdir_p(cache_dir) + except OSError as e: + # makedir may fail due to parallelism. + if e.errno != errno.EEXIST: + raise python_version = ".".join(str(digit) for digit in sys.version_info[:2]) cache_filename = self.filename_format.format( python_version=python_version, @@ -122,10 +139,12 @@ def __init__(self, cache_dir=CACHE_DIR): @property def cache(self): - """The dictionary that is the actual in-memory cache. + """ + The dictionary that is the actual in-memory cache. This property lazily loads the cache from disk. """ + if self._cache is None: self.read_cache() return self._cache @@ -144,6 +163,7 @@ def as_cache_key(self, ireq): ("ipython", "2.1.0[nbconvert,notebook]") """ + extras = tuple(sorted(ireq.extras)) if not extras: extras_string = "" @@ -154,16 +174,16 @@ def as_cache_key(self, ireq): return name, "{}{}".format(version, extras_string) def read_cache(self): - """Reads the cached contents into memory. - """ + """Reads the cached contents into memory.""" + if os.path.exists(self._cache_file): self._cache = _read_cache_file(self._cache_file) else: self._cache = {} def write_cache(self): - """Writes the cache to disk as JSON. - """ + """Writes the cache to disk as JSON.""" + doc = { '__format__': 1, 'dependencies': self._cache, @@ -203,12 +223,18 @@ def get(self, ireq, default=None): class DependencyCache(_JSONCache): - """Cache the dependency of cancidates. + """ + Cache the dependency of cancidates. + """ + filename_format = "depcache-py{python_version}.json" class RequiresPythonCache(_JSONCache): - """Cache a candidate's Requires-Python information. + """ + Cache a candidate's Requires-Python information. + """ + filename_format = "pyreqcache-py{python_version}.json" diff --git a/src/passa/models/environments.py b/src/passa/models/environments.py new file mode 100644 index 0000000..c016cb9 --- /dev/null +++ b/src/passa/models/environments.py @@ -0,0 +1,452 @@ +# -*- coding=utf-8 -*- + +import contextlib +import importlib +import json +import os +import site +import sys + +from distutils.sysconfig import get_python_lib +from functools import partial +from sysconfig import get_paths + +import pkg_resources +import six + +from cached_property import cached_property + +import vistir + + +BASE_WORKING_SET = pkg_resources.WorkingSet(sys.path) +run = partial(vistir.misc.run, write_to_stdout=False, nospin=True, block=True) + + +class Environment(object): + def __init__(self, prefix=None, is_venv=False, base_working_set=None): + self.base_working_set = base_working_set if base_working_set else BASE_WORKING_SET + self._modules = {'pkg_resources': pkg_resources} + self.extra_dists = [] + prefix = prefix if prefix else sys.prefix + prefix = vistir.path.normalize_path(prefix) + sys_prefix = vistir.path.normalize_path(sys.prefix) + prefix = prefix if prefix else sys_prefix + self.is_venv = is_venv or prefix != sys_prefix + self.prefix = vistir.compat.Path(prefix) + self.sys_paths = get_paths() + super(Environment, self).__init__() + + def safe_import(self, name): + """Helper utility for reimporting previously imported modules while inside the env""" + module = None + if name not in self._modules: + self._modules[name] = importlib.import_module(name) + module = self._modules[name] + if not module: + dist = next(iter( + dist for dist in self.base_working_set if dist.project_name == name + ), None) + if dist: + dist.activate() + module = importlib.import_module(name) + if name in sys.modules: + try: + six.moves.reload_module(module) + six.moves.reload_module(sys.modules[name]) + except TypeError: + del sys.modules[name] + sys.modules[name] = self._modules[name] + return module + + @classmethod + def resolve_dist(cls, dist, working_set): + """Given a local distribution and a working set, returns all dependencies from the set. + + :param dist: A single distribution to find the dependencies of + :type dist: :class:`pkg_resources.Distribution` + :param working_set: A working set to search for all packages + :type working_set: :class:`pkg_resources.WorkingSet` + :return: A set of distributions which the package depends on, including the package + :rtype: set(:class:`pkg_resources.Distribution`) + """ + + deps = set() + deps.add(dist) + try: + reqs = dist.requires() + except (AttributeError, OSError, IOError): # The METADATA file can't be found + return deps + for req in reqs: + dist = working_set.find(req) + deps |= cls.resolve_dist(dist, working_set) + return deps + + def add_dist(self, dist_name): + dist = pkg_resources.get_distribution(pkg_resources.Requirement(dist_name)) + extras = self.resolve_dist(dist, self.base_working_set) + if extras: + self.extra_dists.extend(extras) + + @cached_property + def python_version(self): + with self.activated(): + sysconfig = self.safe_import("sysconfig") + py_version = sysconfig.get_python_version() + return py_version + + @property + def python_info(self): + include_dir = self.prefix / "include" + python_path = next(iter(list(include_dir.iterdir())), None) + if python_path and python_path.name.startswith("python"): + python_version = python_path.name.replace("python", "") + py_version_short, abiflags = python_version[:3], python_version[3:] + return {"py_version_short": py_version_short, "abiflags": abiflags} + return {} + + @cached_property + def base_paths(self): + """ + Returns the context appropriate paths for the environment. + + :return: A dictionary of environment specific paths to be used for installation operations + :rtype: dict + + .. note:: The implementation of this is borrowed from a combination of pip and + virtualenv and is likely to change at some point in the future. + + >>> from pipenv.core import project + >>> from pipenv.environment import Environment + >>> env = Environment(prefix=project.virtualenv_location, is_venv=True, sources=project.sources) + >>> import pprint + >>> pprint.pprint(env.base_paths) + {'PATH': '/home/hawk/.virtualenvs/pipenv-MfOPs1lW/bin::/bin:/usr/bin', + 'PYTHONPATH': '/home/hawk/.virtualenvs/pipenv-MfOPs1lW/lib/python3.7/site-packages', + 'data': '/home/hawk/.virtualenvs/pipenv-MfOPs1lW', + 'include': '/home/hawk/.pyenv/versions/3.7.1/include/python3.7m', + 'libdir': '/home/hawk/.virtualenvs/pipenv-MfOPs1lW/lib/python3.7/site-packages', + 'platinclude': '/home/hawk/.pyenv/versions/3.7.1/include/python3.7m', + 'platlib': '/home/hawk/.virtualenvs/pipenv-MfOPs1lW/lib/python3.7/site-packages', + 'platstdlib': '/home/hawk/.virtualenvs/pipenv-MfOPs1lW/lib/python3.7', + 'prefix': '/home/hawk/.virtualenvs/pipenv-MfOPs1lW', + 'purelib': '/home/hawk/.virtualenvs/pipenv-MfOPs1lW/lib/python3.7/site-packages', + 'scripts': '/home/hawk/.virtualenvs/pipenv-MfOPs1lW/bin', + 'stdlib': '/home/hawk/.pyenv/versions/3.7.1/lib/python3.7'} + """ + + prefix = self.prefix.as_posix() + install_scheme = 'nt' if (os.name == 'nt') else 'posix_prefix' + paths = get_paths(install_scheme, vars={ + 'base': prefix, + 'platbase': prefix, + }) + paths["PATH"] = paths["scripts"] + os.pathsep + os.defpath + if "prefix" not in paths: + paths["prefix"] = prefix + purelib = get_python_lib(plat_specific=0, prefix=prefix) + platlib = get_python_lib(plat_specific=1, prefix=prefix) + if purelib == platlib: + lib_dirs = purelib + else: + lib_dirs = purelib + os.pathsep + platlib + paths["libdir"] = purelib + paths["purelib"] = purelib + paths["platlib"] = platlib + paths['PYTHONPATH'] = lib_dirs + paths["libdirs"] = lib_dirs + return paths + + @cached_property + def script_basedir(self): + """Path to the environment scripts dir""" + script_dir = self.base_paths["scripts"] + return script_dir + + @property + def python(self): + """Path to the environment python""" + py = vistir.compat.Path(self.base_paths["scripts"]).joinpath("python").as_posix() + if not py: + return vistir.compat.Path(sys.executable).as_posix() + return py + + @cached_property + def sys_path(self): + """ + The system path inside the environment + + :return: The :data:`sys.path` from the environment + :rtype: list + """ + + current_executable = vistir.compat.Path(sys.executable).as_posix() + if not self.python or self.python == current_executable: + return sys.path + elif any([sys.prefix == self.prefix, not self.is_venv]): + return sys.path + cmd_args = [self.python, "-c", "import json, sys; print(json.dumps(sys.path))"] + path, _ = run(cmd_args, return_object=False, combine_stderr=False) + path = json.loads(path.strip()) + return path + + @cached_property + def sys_prefix(self): + """ + The prefix run inside the context of the environment + + :return: The python prefix inside the environment + :rtype: :data:`sys.prefix` + """ + + command = [self.python, "-c", "import sys; print(sys.prefix)"] + c = run(command, return_object=True) + sys_prefix = vistir.compat.Path(vistir.misc.to_text(c.out).strip()).as_posix() + return sys_prefix + + @property + def scripts_dir(self): + return self.paths["scripts"] + + @property + def libdir(self): + purelib = self.paths.get("purelib", None) + if purelib and os.path.exists(purelib): + return "purelib", purelib + return "platlib", self.paths["platlib"] + + @cached_property + def paths(self): + paths = {} + with vistir.contextmanagers.temp_environ(), vistir.contextmanagers.temp_path(): + os.environ["PYTHONIOENCODING"] = vistir.compat.fs_str("utf-8") + os.environ["PYTHONDONTWRITEBYTECODE"] = vistir.compat.fs_str("1") + paths = self.base_paths + os.environ["PATH"] = paths["PATH"] + os.environ["PYTHONPATH"] = paths["PYTHONPATH"] + if "headers" not in paths: + paths["headers"] = paths["include"] + return paths + + def get_distributions(self): + """Retrives the distributions installed on the library path of the environment + + :return: A set of distributions found on the library path + :rtype: iterator + """ + + pkg_resources = self.safe_import("pkg_resources") + return pkg_resources.find_distributions(self.paths["PYTHONPATH"], only=True) + + def find_egg(self, egg_dist): + """Find an egg by name in the given environment""" + + site_packages = self.libdir[1] + search_filename = "{0}.egg-link".format(egg_dist.project_name) + try: + user_site = site.getusersitepackages() + except AttributeError: + user_site = site.USER_SITE + search_locations = [site_packages, user_site] + for site_directory in search_locations: + egg = os.path.join(site_directory, search_filename) + if os.path.isfile(egg): + return egg + + def locate_dist(self, dist): + """ + Given a distribution, try to find a corresponding egg link first. + + If the egg - link doesn 't exist, return the supplied distribution.""" + + location = self.find_egg(dist) + return location or dist.location + + def dist_is_in_project(self, dist): + """Determine whether the supplied distribution is in the environment.""" + + prefix = vistir.path.normalize_path(self.base_paths["prefix"]) + location = self.locate_dist(dist) + if not location: + return False + return vistir.path.normalize_path(location).startswith(prefix) + + def get_installed_packages(self): + """ + Returns all of the installed packages in a given environment""" + + workingset = self.get_working_set() + packages = [pkg for pkg in workingset if self.dist_is_in_project(pkg)] + return packages + + def get_working_set(self): + """Retrieve the working set of installed packages for the environment. + + :return: The working set for the environment + :rtype: :class:`pkg_resources.WorkingSet` + """ + + working_set = None + import pkg_resources + working_set = pkg_resources.WorkingSet(self.sys_path) + return working_set + + def is_installed(self, pkgname): + """Given a package name, returns whether it is installed in the environment + + :param str pkgname: The name of a package + :return: Whether the supplied package is installed in the environment + :rtype: bool + """ + + return any(d for d in self.get_distributions() if d.project_name == pkgname) + + def run(self, cmd, cwd=os.curdir): + """Run a command with :class:`~subprocess.Popen` in the context of the environment + + :param cmd: A command to run in the environment + :type cmd: str or list + :param str cwd: The working directory in which to execute the command, defaults to :data:`os.curdir` + :return: A finished command object + :rtype: :class:`~subprocess.Popen` + """ + + c = None + with self.activated(): + script = vistir.cmdparse.Script.parse(cmd) + c = run(script._parts, return_object=True, cwd=cwd) + return c + + def run_py(self, cmd, cwd=os.curdir): + """Run a python command in the enviornment context. + + :param cmd: A command to run in the environment - runs with `python -c` + :type cmd: str or list + :param str cwd: The working directory in which to execute the command, defaults to :data:`os.curdir` + :return: A finished command object + :rtype: :class:`~subprocess.Popen` + """ + + c = None + if isinstance(cmd, six.string_types): + script = vistir.cmdparse.Script.parse("{0} -c {1}".format(self.python, cmd)) + else: + script = vistir.cmdparse.Script.parse([self.python, "-c"] + list(cmd)) + with self.activated(): + c = run(script._parts, return_object=True, cwd=cwd) + return c + + def run_activate_this(self): + """Runs the environment's inline activation script""" + if self.is_venv: + activate_this = os.path.join(self.scripts_dir, "activate_this.py") + if not os.path.isfile(activate_this): + raise OSError("No such file: {0!s}".format(activate_this)) + with open(activate_this, "r") as f: + code = compile(f.read(), activate_this, "exec") + exec(code, dict(__file__=activate_this)) + + @contextlib.contextmanager + def activated(self, include_extras=True, extra_dists=None): + """Helper context manager to activate the environment. + + This context manager will set the following variables for the duration + of its activation: + + * sys.prefix + * sys.path + * os.environ["VIRTUAL_ENV"] + * os.environ["PATH"] + + In addition, it will make any distributions passed into `extra_dists` available + on `sys.path` while inside the context manager, as well as making `passa` itself + available. + + The environment's `prefix` as well as `scripts_dir` properties are both prepended + to `os.environ["PATH"]` to ensure that calls to `~Environment.run()` use the + environment's path preferentially. + """ + + if not extra_dists: + extra_dists = [] + original_prefix = sys.prefix + parent_path = vistir.compat.Path(__file__).absolute().parent.parent.as_posix() + prefix = self.prefix.as_posix() + with vistir.contextmanagers.temp_environ(), vistir.contextmanagers.temp_path(): + os.environ["PATH"] = os.pathsep.join([ + vistir.compat.fs_str(self.scripts_dir), + vistir.compat.fs_str(self.prefix.as_posix()), + os.environ.get("PATH", os.defpath) + ]) + os.environ["PYTHONIOENCODING"] = vistir.compat.fs_str("utf-8") + os.environ["PYTHONDONTWRITEBYTECODE"] = vistir.compat.fs_str("1") + os.environ["PYTHONPATH"] = self.base_paths["PYTHONPATH"] + if self.is_venv: + os.environ["VIRTUAL_ENV"] = vistir.compat.fs_str(prefix) + pkg_resources = self.safe_import("pkg_resources") + site = self.safe_import("site") + sys.path = self.sys_path + sys.prefix = self.sys_prefix + site.addsitedir(self.libdir[1]) + if include_extras: + site.addsitedir(parent_path) + extra_dists = list(self.extra_dists) + extra_dists + for extra_dist in extra_dists: + if extra_dist not in self.get_working_set(): + extra_dist.activate(self.sys_path) + try: + yield + finally: + sys.prefix = original_prefix + six.moves.reload_module(pkg_resources) + + @contextlib.contextmanager + def uninstall(self, pkgname, *args, **kwargs): + """A context manager which allows uninstallation of packages from the environment + + :param str pkgname: The name of a package to uninstall + + >>> env = Environment("/path/to/env/root") + >>> with env.uninstall("pytz", auto_confirm=True, verbose=False) as uninstaller: + cleaned = uninstaller.paths + >>> if cleaned: + print("uninstalled packages: %s" % cleaned) + """ + + auto_confirm = kwargs.pop("auto_confirm", True) + verbose = kwargs.pop("verbose", False) + with self.activated(): + monkey_patch = next(iter( + dist for dist in self.base_working_set + if dist.project_name == "recursive-monkey-patch" + ), None) + if monkey_patch: + monkey_patch.activate() + pip_shims = self.safe_import("pip_shims") + pathset_base = pip_shims.UninstallPathSet + import recursive_monkey_patch + recursive_monkey_patch.monkey_patch( + PatchedUninstaller, pathset_base + ) + dist = next( + iter(filter(lambda d: d.project_name == pkgname, self.get_working_set())), + None + ) + pathset = pathset_base.from_dist(dist) + if pathset is not None: + pathset.remove(auto_confirm=auto_confirm, verbose=verbose) + try: + yield pathset + except Exception: + if pathset is not None: + pathset.rollback() + else: + if pathset is not None: + pathset.commit() + if pathset is None: + return + + +class PatchedUninstaller(object): + def _permitted(self, path): + return True diff --git a/src/passa/models/lockers.py b/src/passa/models/lockers.py index 53f1cab..27275aa 100644 --- a/src/passa/models/lockers.py +++ b/src/passa/models/lockers.py @@ -94,12 +94,14 @@ class AbstractLocker(object): """ def __init__(self, project): self.project = project - self.default_requirements = _get_requirements( - project.pipfile, "packages", - ) - self.develop_requirements = _get_requirements( - project.pipfile, "dev-packages", - ) + with vistir.cd(self.project.root): + # Change dir to the project to resolve the relative paths properly. + self.default_requirements = _get_requirements( + project.pipfile, "packages", + ) + self.develop_requirements = _get_requirements( + project.pipfile, "dev-packages", + ) # This comprehension dance ensures we merge packages from both # sections, and definitions in the default section win. diff --git a/src/passa/models/metadata.py b/src/passa/models/metadata.py index e3cde70..4d7641a 100644 --- a/src/passa/models/metadata.py +++ b/src/passa/models/metadata.py @@ -123,17 +123,19 @@ def __or__(self, other): def _build_metasets(dependencies, pythons, key, trace, all_metasets): - all_parent_metasets = [] + all_parent_metasets = {} for route in trace: parent = route[-1] + if parent in all_parent_metasets: + continue try: parent_metasets = all_metasets[parent] except KeyError: # Parent not calculated yet. Wait for it. return - all_parent_metasets.append((parent, parent_metasets)) + all_parent_metasets[parent] = parent_metasets metasets = set() - for parent, parent_metasets in all_parent_metasets: + for parent, parent_metasets in all_parent_metasets.items(): r = dependencies[parent][key] python = pythons[key] markers = None if r.editable else get_without_extra(r.markers) diff --git a/src/passa/models/projects.py b/src/passa/models/projects.py index 058a5cc..86e71e3 100644 --- a/src/passa/models/projects.py +++ b/src/passa/models/projects.py @@ -10,10 +10,10 @@ import packaging.markers import packaging.utils import six +import plette import tomlkit -import plette -import plette.models +from .environments import Environment SectionDifference = collections.namedtuple("SectionDifference", [ @@ -85,7 +85,7 @@ def dumps(self): class Project(object): root = attr.ib() - env_prefix = attr.ib(default=None) + environment = attr.ib(default=attr.Factory(Environment)) _p = attr.ib(init=False) _l = attr.ib(init=False) @@ -140,8 +140,10 @@ def contains_key_in_pipfile(self, key): self._get_pipfile_section(develop=True, insert=False), ] return any( - (packaging.utils.canonicalize_name(name) == - packaging.utils.canonicalize_name(key)) + ( + packaging.utils.canonicalize_name(name) + == packaging.utils.canonicalize_name(key) + ) for section in sections for name in section ) diff --git a/src/passa/models/providers.py b/src/passa/models/providers.py index 36b2f2e..86cd9a6 100644 --- a/src/passa/models/providers.py +++ b/src/passa/models/providers.py @@ -170,8 +170,8 @@ def __init__(self, tracked_names, *args, **kwargs): def is_satisfied_by(self, requirement, candidate): # If this is a tracking package, tell the resolver out of using the # preferred pin, and into a "normal" candidate selection process. - if (self.identify(requirement) in self.tracked_names and - getattr(candidate, "_preferred_by_provider", False)): + if (self.identify(requirement) in self.tracked_names + and getattr(candidate, "_preferred_by_provider", False)): return False return super(EagerUpgradeProvider, self).is_satisfied_by( requirement, candidate, diff --git a/src/passa/models/synchronizers.py b/src/passa/models/synchronizers.py index 2ade9c5..7b002d9 100644 --- a/src/passa/models/synchronizers.py +++ b/src/passa/models/synchronizers.py @@ -3,12 +3,9 @@ from __future__ import absolute_import, unicode_literals, print_function import collections -import contextlib import os import sys -import sysconfig -import distlib.wheel import pkg_resources import packaging.markers @@ -18,19 +15,6 @@ from ..internals._pip import uninstall, Installer -def _is_installation_local(name, venv=None): - """Check whether the distribution is in the current Python installation. - - This is used to distinguish packages seen by a virtual environment. A venv - may be able to see global packages, but we don't want to mess with them. - """ - if venv: - return venv.is_installed(name) - loc = os.path.normcase(pkg_resources.working_set.by_key[name].location) - pre = os.path.normcase(sys.prefix) - return os.path.commonprefix([loc, pre]) == pre - - def _is_up_to_date(distro, version): # This is done in strings to avoid type mismatches caused by vendering. return str(version) == str(packaging.version.parse(distro.version)) @@ -41,61 +25,6 @@ def _is_up_to_date(distro, version): ]) -def _group_installed_names(packages, venv=None): - """Group locally installed packages based on given specifications. - - `packages` is a name-package mapping that are used as baseline to - determine how the installed package should be grouped. - - `venv` is the virtual environment object of the virtualenv being installed into. - - Returns a 3-tuple of disjoint sets, all containing names of installed - packages: - - * `uptodate`: These match the specifications. - * `outdated`: These installations are specified, but don't match the - specifications in `packages`. - * `unneeded`: These are installed, but not specified in `packages`. - """ - groupcoll = GroupCollection(set(), set(), set(), set()) - - if venv: - working_set = venv.get_working_set() - else: - working_set = pkg_resources.working_set - - for dist in working_set: - name = dist.key - try: - package = packages[name] - except KeyError: - groupcoll.unneeded.add(name) - continue - - r = requirementslib.Requirement.from_pipfile(name, package) - if not r.is_named: - # Always mark non-named. I think pip does something similar? - groupcoll.outdated.add(name) - elif not _is_up_to_date(dist, r.get_version()): - groupcoll.outdated.add(name) - else: - groupcoll.uptodate.add(name) - - return groupcoll - - -@contextlib.contextmanager -def _remove_package(name, venv=None): - if name is None or not _is_installation_local(name, venv=venv): - yield None - return - _uninstall = uninstall - if venv: - _uninstall = venv.uninstall - with _uninstall(name, auto_confirm=True, verbose=False) as uninstaller: - yield uninstaller - - def _get_packages(lockfile, default, develop): # Don't need to worry about duplicates because only extras can differ. # Extras don't matter because they only affect dependencies, and we @@ -108,68 +37,126 @@ def _get_packages(lockfile, default, develop): return packages -def _build_paths(venv=None): - """Prepare paths for distlib.wheel.Wheel to install into. - """ - if venv: - paths = venv.paths - else: - paths = sysconfig.get_paths() - return { - "prefix": sys.prefix if not venv else venv.venv_dir.as_posix(), - "data": paths["data"], - "scripts": paths["scripts"], - "headers": paths["include"], - "purelib": paths["purelib"], - "platlib": paths["platlib"], - } - - PROTECTED_FROM_CLEAN = {"setuptools", "pip", "wheel"} -def _clean(names, venv=None): - cleaned = set() - for name in names: - if name in PROTECTED_FROM_CLEAN: - continue - with _remove_package(name, venv=venv) as uninst: - if uninst: +class InstallManager(object): + """A centrialized manager object to handle install and uninstall operations.""" + + def __init__(self, sources=None, environment=None): + self.sources = sources + self.environment = environment + + def get_working_set(self): + if self.environment: + return self.environment.get_working_set() + return pkg_resources.working_set + + def is_installation_local(self, name): + """Check whether the distribution is in the current Python installation. + + This is used to distinguish packages seen by a virtual environment. A environment + may be able to see global packages, but we don't want to mess with them. + """ + if self.environment: + return self.environment.is_installed(name) + loc = os.path.normcase(self.get_working_set().by_key[name].location) + pre = os.path.normcase(sys.prefix) + return os.path.commonprefix([loc, pre]) == pre + + def remove(self, name): + if name is None or not self.is_installation_local(name): + return False + _uninstall = uninstall + if self.environment: + _uninstall = self.environment.uninstall + with _uninstall(name, auto_confirm=True, verbose=False): + return True + + def install(self, req): + installer = Installer(req, sources=self.sources, environment=self.environment) + try: + installer.prepare() + except Exception as e: + if os.environ.get("PASSA_NO_SUPPRESS_EXCEPTIONS"): + raise + print("failed to prepare {0!r}: {1}".format( + req.as_line(include_hashes=False), e, + )) + return False + + try: + installer.install() + except Exception as e: + if os.environ.get("PASSA_NO_SUPPRESS_EXCEPTIONS"): + raise + print("failed to install {0!r}: {1}".format( + req.as_line(include_hashes=False), e, + )) + return False + return True + + def clean(self, names): + cleaned = set() + for name in names: + if name in PROTECTED_FROM_CLEAN: + continue + if self.remove(name): cleaned.add(name) - return cleaned + return cleaned class Synchronizer(object): """Helper class to install packages from a project's lock file. """ - def __init__(self, project, default, develop, clean_unneeded, venv=None): + def __init__(self, project, default, develop, clean_unneeded, dry_run=False): self._root = project.root # Only for repr. + self.project = project self.packages = _get_packages(project.lockfile, default, develop) - self.sources = project.lockfile.meta.sources._data self.clean_unneeded = clean_unneeded - if not venv: - self._venv = getattr(project, "venv", None) - else: - self._venv = venv - self.paths = _build_paths(venv=self.venv) - - @property - def venv(self): - if self._venv: - return self._venv - return self.project.venv + self.dry_run = dry_run + super(Synchronizer, self).__init__() + sources = project.lockfile.meta.sources._data + environment = self.project.environment + self.install_manager = InstallManager(sources, environment) def __repr__(self): return "<{0} @ {1!r}>".format(type(self).__name__, self._root) - def sync(self): - if not self.venv: - return self._sync() - with self.venv.activated(): - return self._sync() + def group_installed_names(self): + """Group locally installed packages based on given specifications. + + Returns a 3-tuple of disjoint sets, all containing names of installed + packages: + + * `uptodate`: These match the specifications. + * `outdated`: These installations are specified, but don't match the + specifications in `packages`. + * `unneeded`: These are installed, but not specified in `packages`. + """ + groupcoll = GroupCollection(set(), set(), set(), set()) + + for dist in self.install_manager.get_working_set(): + name = dist.key + try: + package = self.packages[name] + except KeyError: + groupcoll.unneeded.add(name) + continue - def _sync(self): - groupcoll = _group_installed_names(self.packages, venv=self.venv) + r = requirementslib.Requirement.from_pipfile(name, package) + if not r.is_named: + # Always mark non-named. I think pip does something similar? + groupcoll.outdated.add(name) + elif not _is_up_to_date(dist, r.get_version()): + groupcoll.outdated.add(name) + else: + groupcoll.uptodate.add(name) + + return groupcoll + + def sync(self): + groupcoll = self.group_installed_names() installed = set() updated = set() @@ -178,11 +165,11 @@ def _sync(self): # TODO: Show a prompt to confirm cleaning. We will need to implement a # reporter pattern for this as well. if self.clean_unneeded: - names = _clean(groupcoll.unneeded, venv=self.venv) + if not self.dry_run: + names = self.install_manager.clean(groupcoll.unneeded) cleaned.update(names) # TODO: Specify installation order? (pypa/pipenv#2274) - installers = [] for name, package in self.packages.items(): r = requirementslib.Requirement.from_pipfile(name, package) name = r.normalized_name @@ -192,33 +179,13 @@ def _sync(self): if markers and not packaging.markers.Marker(markers).evaluate(): continue r.markers = None - installer = Installer(r, sources=self.sources, paths=self.paths, venv=self.venv) - try: - installer.prepare() - except Exception as e: - if os.environ.get("PASSA_NO_SUPPRESS_EXCEPTIONS"): - raise - print("failed to prepare {0!r}: {1}".format( - r.as_line(include_hashes=False), e, - )) - else: - installers.append((name, installer)) + if not self.dry_run: + if name in groupcoll.outdated: + self.install_manager.remove(name) + success = self.install_manager.install(r) + if not success: + continue - for name, installer in installers: - if name in groupcoll.outdated: - name_to_remove = name - else: - name_to_remove = None - try: - with _remove_package(name_to_remove, venv=self.venv): - installer.install() - except Exception as e: - if os.environ.get("PASSA_NO_SUPPRESS_EXCEPTIONS"): - raise - print("failed to install {0!r}: {1}".format( - r.as_line(include_hashes=False), e, - )) - continue if name in groupcoll.outdated or name in groupcoll.noremove: updated.add(name) else: @@ -226,31 +193,9 @@ def _sync(self): return installed, updated, cleaned - -class Cleaner(object): - """Helper class to clean packages not in a project's lock file. - """ - def __init__(self, project, default, develop, sync=True, verbose=False): - self._root = project.root # Only for repr. - self.packages = _get_packages(project.lockfile, default, develop) - self.sync = sync - self.project = project - - def __repr__(self): - return "<{0} @ {1!r}>".format(type(self).__name__, self._root) - - def print(self, packages): - if not self.sync: - message = "Would clean: {0}" - else: - message = "Cleaned: {0}" - print(message.format(", ".join(sorted(set(packages))))) - def clean(self): - groupcoll = _group_installed_names(self.packages, venv=self.project.venv) - cleaned = set() - if self.sync: - cleaned = _clean(groupcoll.unneeded, venv=self.project.venv) + groupcoll = self.group_installed_names() + if not self.dry_run: + return self.install_manager.clean(groupcoll.unneeded) else: return groupcoll.unneeded - return cleaned diff --git a/src/passa/operations/sync.py b/src/passa/operations/sync.py index 45502a4..7977a1c 100644 --- a/src/passa/operations/sync.py +++ b/src/passa/operations/sync.py @@ -15,9 +15,11 @@ def sync(syncer): return True -def clean(cleaner): +def clean(syncer): print("Cleaning...") - cleaned = cleaner.clean() - if cleaned: - cleaner.print(cleaned) + cleaned = syncer.clean() + if syncer.dry_run: + print("Would clean: {}".format(", ".join(sorted(cleaned)))) + else: + print("Cleaned: {}".format(", ".join(sorted(cleaned)))) return True diff --git a/tasks/admin.py b/tasks/admin.py index 2fabe99..fa1be74 100644 --- a/tasks/admin.py +++ b/tasks/admin.py @@ -14,6 +14,14 @@ INIT_PY = ROOT.joinpath('src', PACKAGE_NAME, '__init__.py') +@invoke.task() +def typecheck(ctx): + src_dir = ROOT / "src" / PACKAGE_NAME + src_dir = src_dir.as_posix() + env = {"MYPYPATH": src_dir} + ctx.run(f"mypy {src_dir}", env=env) + + @invoke.task() def clean(ctx): """Clean previously built package artifacts. diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..7d71f84 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,5 @@ +import os + +TESTS_ROOT = os.path.dirname(os.path.abspath(__file__)) +FIXTURES_DIR = os.path.join(TESTS_ROOT, "fixtures") +PYPI_VENDOR_DIR = os.path.join(TESTS_ROOT, 'pypi') diff --git a/tests/actions/test_add.py b/tests/actions/test_add.py index e7cd883..0fea04e 100644 --- a/tests/actions/test_add.py +++ b/tests/actions/test_add.py @@ -1,20 +1,33 @@ # -*- coding=utf-8 -*- +import os import passa.actions.init import passa.actions.add import passa.cli.options import passa.models.projects +import pytest -def test_add_one(project_directory): - project = passa.cli.options.Project(project_directory.strpath) - retcode = passa.actions.add.add_packages(["pytz"], project=project) +@pytest.mark.parametrize('req,deps', [ + ('pytz', ['pytz']), ('requests', ['requests', 'idna']) +]) +def test_add_one(project, is_dev, sync, req, deps): + retcode = passa.actions.add.add_packages([req], project=project, dev=is_dev, sync=sync) assert not retcode - assert 'pytz' in project.lockfile.default + section = project.lockfile.develop if is_dev else project.lockfile.default + for dep in deps: + assert dep in section._data + if sync: + assert project.is_installed(dep) -def test_add_one_with_deps(project_directory): - project = passa.cli.options.Project(project_directory.strpath) - retcode = passa.actions.add.add_packages(["requests"], project=project) +def test_add_one_with_markers(project, is_dev): + retcode = passa.actions.add.add_packages( + ["requests; os_name == 'nt'"], + project=project, dev=is_dev, sync=True + ) assert not retcode - assert 'requests' in project.lockfile.default - assert 'idna' in project.lockfile.default + for dep in ('requests', 'idna'): + if os.name == 'nt': + assert project.is_installed(dep) + else: + assert not project.is_installed(dep) diff --git a/tests/actions/test_clean.py b/tests/actions/test_clean.py index 2160301..e9ef3ed 100644 --- a/tests/actions/test_clean.py +++ b/tests/actions/test_clean.py @@ -3,18 +3,12 @@ import passa.actions.clean -def test_clean(project): +def test_clean(project, install_manager): retcode = passa.actions.add.add_packages(["requests"], project=project) assert not retcode packages = ["requests", "chardet", "certifi", "idna"] - c = project.venv.run("pip install pytz") - assert c.returncode == 0 - assert project.venv.is_installed("pytz") - c = project.venv.run("python -c 'import pytz'") - assert c.returncode == 0 + install_manager.install('pytz==2018.4') clean_retcode = passa.actions.clean.clean(project=project) assert not clean_retcode - assert not project.venv.is_installed("pytz") - c = project.venv.run("python -c 'import pytz'") - assert c.returncode != 0 + assert not project.is_installed("pytz") assert all(pkg in project.lockfile.default for pkg in packages) diff --git a/tests/actions/test_freeze.py b/tests/actions/test_freeze.py index 53d34f9..3c8f465 100644 --- a/tests/actions/test_freeze.py +++ b/tests/actions/test_freeze.py @@ -5,17 +5,16 @@ import passa.models.projects -def test_freeze(project_directory): - project = passa.cli.options.Project(project_directory.strpath) +def test_freeze(project): retcode = passa.actions.add.add_packages(["requests"], project=project) assert not retcode packages = ["requests", "chardet", "certifi", "idna"] assert all(pkg in project.lockfile.default for pkg in packages) - freeze_file = project_directory.join("requirements.txt") + freeze_file = project.path.joinpath("requirements.txt") freeze_retcode = passa.actions.freeze.freeze( - project=project, include_hashes=False, target=freeze_file.strpath + project=project, include_hashes=False, target=freeze_file.as_posix() ) assert not freeze_retcode - lines = [line.strip() for line in freeze_file.readlines() if line.strip() != ''] + lines = [line.strip() for line in freeze_file.read_text().splitlines() if line.strip() != ''] for pkg in packages: assert any(line.startswith(pkg) for line in lines) diff --git a/tests/actions/test_init.py b/tests/actions/test_init.py index 2f12530..e8412db 100644 --- a/tests/actions/test_init.py +++ b/tests/actions/test_init.py @@ -1,6 +1,7 @@ # -*- coding=utf-8 -*- import pytest +import os import passa.actions.init import passa.cli.options @@ -14,6 +15,25 @@ def test_init(tmpdir): assert project.pipfile.dev_packages._data == {} -def test_init_exists(project_directory): +def test_init_exists(project): with pytest.raises(RuntimeError, match=r'.* is already a Pipfile project'): - passa.actions.init.init_project(root=project_directory.strpath) + passa.actions.init.init_project(root=project.root) + + +def test_init_inherit_pip_source(tmpdir): + pip_config_dir = os.path.join( + os.path.expanduser('~'), + "pip" if os.name == "nt" else ".pip" + ) + if not os.path.exists(pip_config_dir): + os.makedirs(pip_config_dir) + pip_config_path = os.path.join(pip_config_dir, "pip.ini" if os.name == "nt" else "pip.conf") + with open(pip_config_path, 'w') as f: + f.write('[global]\nindex-url=https://foo.pypi.org/simple') + try: + init_retcode = passa.actions.init.init_project(root=tmpdir.strpath) + assert init_retcode == 0 + project = passa.cli.options.Project(tmpdir.strpath) + assert project.pipfile.source._data[0]['url'] == 'https://foo.pypi.org/simple' + finally: + os.remove(pip_config_path) diff --git a/tests/actions/test_install.py b/tests/actions/test_install.py index bb205e6..36c90e6 100644 --- a/tests/actions/test_install.py +++ b/tests/actions/test_install.py @@ -1,40 +1,20 @@ # -*- coding=utf-8 -*- - import passa.actions.install import passa.actions.add import passa.cli.options import passa.models.projects import pytest - -@pytest.mark.parametrize( - 'is_dev', (True, False) -) -def test_install_one(project, is_dev): - add_kwargs = { - "project": project, - "packages": ["pytz",], - "editables": [], - "dev": is_dev, - "sync": False, - "clean": False - } - retcode = passa.actions.add.add_packages(**add_kwargs) - assert not retcode - lockfile_section = "default" if not is_dev else "develop" - assert 'pytz' in project.lockfile._data[lockfile_section].keys() - install = passa.actions.install.install(project=project, check=True, dev=is_dev, clean=False) - assert install == 0 - assert project.venv.is_installed("pytz") or project.is_installed("pytz") +from tests import FIXTURES_DIR -@pytest.mark.parametrize( - 'is_dev', (True, False) -) -def test_install_one_with_deps(project, is_dev): +@pytest.mark.parametrize('req,deps', [ + ('pytz', ['pytz']), ('requests', ['requests', 'idna']) +]) +def test_install_one(project, is_dev, req, deps): add_kwargs = { "project": project, - "packages": ["requests",], + "packages": [req], "editables": [], "dev": is_dev, "sync": False, @@ -43,22 +23,23 @@ def test_install_one_with_deps(project, is_dev): retcode = passa.actions.add.add_packages(**add_kwargs) assert not retcode lockfile_section = "default" if not is_dev else "develop" - assert 'requests' in project.lockfile._data[lockfile_section].keys() - assert 'idna' in project.lockfile._data[lockfile_section].keys() + for dep in deps: + assert dep in project.lockfile._data[lockfile_section].keys() install = passa.actions.install.install(project=project, check=True, dev=is_dev, clean=False) assert install == 0 - assert project.venv.is_installed("requests") or project.is_installed("requests") - assert project.venv.is_installed("idna") or project.is_installed("idna") + for dep in deps: + assert project.is_installed(dep) -@pytest.mark.parametrize( - 'is_dev', (True, False) -) -def test_install_editable(project, is_dev): +@pytest.mark.parametrize('line', [ + "git+https://github.com/testing/demo.git#egg=demo", + "{}/git/github.com/testing/demo.git".format(FIXTURES_DIR) +]) +def test_install_editable(project, is_dev, line): add_kwargs = { "project": project, "packages": [], - "editables": ["git+https://github.com/sarugaku/shellingham.git@1.2.1#egg=shellingham",], + "editables": [line], "dev": is_dev, "sync": False, "clean": False @@ -66,21 +47,22 @@ def test_install_editable(project, is_dev): retcode = passa.actions.add.add_packages(**add_kwargs) assert not retcode lockfile_section = "default" if not is_dev else "develop" - assert 'shellingham' in project.lockfile._data[lockfile_section].keys() + assert 'demo' in project.lockfile._data[lockfile_section] + assert 'requests' in project.lockfile._data[lockfile_section] install = passa.actions.install.install(project=project, check=True, dev=is_dev, clean=False) assert install == 0 - project.reload() - assert (project.venv.is_installed("shellingham") or - project.is_installed("shellingham")), list([dist.project_name for dist in project.venv.get_distributions()]) + assert project.is_installed("demo") + assert project.is_installed("requests") -@pytest.mark.parametrize( - 'is_dev', (True, False) -) -def test_install_sdist(project, is_dev): +@pytest.mark.parametrize('link', [ + "flask/Flask-0.12.2-py2.py3-none-any.whl", + "flask/Flask-0.12.2.tar.gz" +]) +def test_install_file_links(project, is_dev, link, pypi): add_kwargs = { "project": project, - "packages": ["arrow",], + "packages": ["{}/{}".format(pypi.url, link)], "editables": [], "dev": is_dev, "sync": False, @@ -89,8 +71,8 @@ def test_install_sdist(project, is_dev): retcode = passa.actions.add.add_packages(**add_kwargs) assert not retcode lockfile_section = "default" if not is_dev else "develop" - assert 'arrow' in project.lockfile._data[lockfile_section].keys() + assert 'flask' in project.lockfile._data[lockfile_section].keys() install = passa.actions.install.install(project=project, check=True, dev=is_dev, clean=False) assert install == 0 - project.reload() - assert project.venv.is_installed("arrow") or project.is_installed("arrow") + assert project.is_installed("flask") + assert project.is_installed("jinja2") diff --git a/tests/actions/test_lock.py b/tests/actions/test_lock.py index 962493e..8d9a0f4 100644 --- a/tests/actions/test_lock.py +++ b/tests/actions/test_lock.py @@ -5,49 +5,100 @@ import passa.cli.options import passa.models.projects import pytest +import shutil +from tests import FIXTURES_DIR -@pytest.mark.parametrize( - 'is_dev', (True, False) -) -def test_lock_one(project, is_dev): - line = "pytz" + +@pytest.mark.parametrize('req,deps', [ + ('pytz', ['pytz']), ('requests', ['requests', 'idna']) +]) +def test_lock_one(project, is_dev, req, deps): + project.add_line_to_pipfile(req, develop=is_dev) + retcode = passa.actions.lock.lock(project=project) + assert retcode == 0 + lockfile_section = "default" if not is_dev else "develop" + for dep in deps: + assert dep in project.lockfile._data[lockfile_section].keys() + + +@pytest.mark.parametrize('line', [ + "-e git+https://github.com/testing/no_dep.git#egg=no_dep", + "-e {}/git/github.com/testing/no_dep.git".format(FIXTURES_DIR) +]) +def test_lock_editable(project, is_dev, line): project.add_line_to_pipfile(line, develop=is_dev) retcode = passa.actions.lock.lock(project=project) - project.reload() assert retcode == 0 lockfile_section = "default" if not is_dev else "develop" - assert 'pytz' in project.lockfile._data[lockfile_section].keys() - install = passa.actions.install.install(project=project, check=True, dev=is_dev, clean=False) - assert install == 0 + assert 'no-dep' in project.lockfile._data[lockfile_section].keys(), project.lockfile._data -@pytest.mark.parametrize( - 'is_dev', (True, False) -) -def test_lock_one_with_deps(project, is_dev): - line = "requests" +@pytest.mark.parametrize('line', [ + "-e git+https://github.com/testing/demo.git#egg=demo", + "-e {}/git/github.com/testing/demo.git".format(FIXTURES_DIR) +]) +def test_lock_editable_with_deps(project, is_dev, line): project.add_line_to_pipfile(line, develop=is_dev) retcode = passa.actions.lock.lock(project=project) - project.reload() assert retcode == 0 lockfile_section = "default" if not is_dev else "develop" + assert 'demo' in project.lockfile._data[lockfile_section].keys() assert 'requests' in project.lockfile._data[lockfile_section].keys() - assert 'idna' in project.lockfile._data[lockfile_section].keys() - install = passa.actions.install.install(project=project, check=True, dev=is_dev, clean=False) - assert install == 0 -@pytest.mark.parametrize( - 'is_dev', (True, False) -) -def test_lock_editable(project, is_dev): - line = "-e git+https://github.com/sarugaku/shellingham.git@1.2.1#egg=shellingham" - project.add_line_to_pipfile(line, develop=is_dev) +@pytest.mark.parametrize('link', [ + "flask/Flask-0.12.2-py2.py3-none-any.whl", + "flask/Flask-0.12.2.tar.gz" +]) +def test_lock_file_links(project, link, pypi): + project.add_line_to_pipfile("{}/{}".format(pypi.url, link), develop=False) + retcode = passa.actions.lock.lock(project=project) + assert retcode == 0 + lockfile_section = project.lockfile._data["default"] + assert 'flask' in lockfile_section.keys() + assert 'jinja2' in lockfile_section.keys() + + +def test_lock_vcs_link(project): + project.add_line_to_pipfile("git+https://github.com/testing/demo.git#egg=demo", develop=False) + retcode = passa.actions.lock.lock(project=project) + assert retcode == 0 + assert 'demo' in project.lockfile._data['default'].keys() + assert project.lockfile._data['default']['demo']['ref'] == 'c55ee5cc8230a338a8a942704a9fe7eff8f88a1c' + assert 'requests' in project.lockfile._data['default'].keys() + + +def test_lock_editable_relative_path(project, is_dev): + shutil.copytree( + "{}/git/github.com/testing/demo.git".format(FIXTURES_DIR), + project.path.joinpath("demo").as_posix() + ) + project.add_line_to_pipfile("-e ./demo", develop=is_dev) retcode = passa.actions.lock.lock(project=project) - project.reload() assert retcode == 0 lockfile_section = "default" if not is_dev else "develop" - assert 'shellingham' in project.lockfile._data[lockfile_section].keys(), project.lockfile._data - install = passa.actions.install.install(project=project, check=True, dev=is_dev, clean=False) - assert install == 0 + assert 'demo' in project.lockfile._data[lockfile_section].keys() + assert 'requests' in project.lockfile._data[lockfile_section].keys() + + +@pytest.mark.skip(reason="TODO: fix extras locking") +def test_lock_with_extras(project): + project.add_line_to_pipfile("requests[socks]", develop=False) + retcode = passa.actions.lock.lock(project=project) + assert retcode == 0 + lockfile_section = project.lockfile.default._data + assert 'requests' in lockfile_section + assert lockfile_section['requests']['extras'] == ['socks'] + assert 'idna' in lockfile_section + assert 'pysocks' in lockfile_section + + +def test_lock_inherit_markers(project, is_dev): + project.add_line_to_pipfile("requests; os_name == 'nt'", develop=is_dev) + section = "develop" if is_dev else "default" + retcode = passa.actions.lock.lock(project=project) + assert retcode == 0 + for pkg in ('requests', 'idna'): + assert pkg in project.lockfile._data[section] + assert "os_name == 'nt'" in project.lockfile._data[section][pkg]['markers'] diff --git a/tests/actions/test_remove_and_sync.py b/tests/actions/test_remove_and_sync.py index e14ec8a..7e5e1fa 100644 --- a/tests/actions/test_remove_and_sync.py +++ b/tests/actions/test_remove_and_sync.py @@ -5,50 +5,17 @@ import passa.cli.options import passa.models.projects import pytest -import vistir - -@pytest.mark.parametrize( - 'sync', (True, False) -) -@pytest.mark.parametrize( - 'is_dev', (True, False) -) -def test_remove_one(project, sync, is_dev): - pkg = "xlrd" - add_kwargs = { - "project": project, - "packages": [pkg,], - "editables": [], - "dev": is_dev, - "sync": sync, - "clean": False - } - retcode = passa.actions.add.add_packages(**add_kwargs) - assert not retcode - lockfile_section = "default" if not is_dev else "develop" - assert pkg in project.lockfile._data[lockfile_section].keys() - if sync: - assert project.venv.is_installed(pkg) or project.is_installed(pkg) - remove = "default" if not is_dev else "dev" - retcode = passa.actions.remove.remove(project=project, packages=[pkg,], sync=sync, only=remove) - assert not retcode - project.reload() - assert pkg not in project.lockfile._data[lockfile_section].keys() - if sync: - assert not project.venv.is_installed(pkg) +from tests import FIXTURES_DIR -@pytest.mark.parametrize( - 'sync', (True, False), -) -@pytest.mark.parametrize( - 'is_dev', (True, False) -) -def test_remove_one_with_deps(project, sync, is_dev): +@pytest.mark.parametrize('req,deps', [ + ('pytz', ['pytz']), ('requests', ['requests', 'idna']) +]) +def test_remove_one(project, sync, is_dev, req, deps): add_kwargs = { "project": project, - "packages": ["requests",], + "packages": [req], "editables": [], "dev": is_dev, "sync": sync, @@ -57,36 +24,29 @@ def test_remove_one_with_deps(project, sync, is_dev): retcode = passa.actions.add.add_packages(**add_kwargs) assert not retcode lockfile_section = "default" if not is_dev else "develop" - assert 'requests' in project.lockfile._data[lockfile_section].keys() - assert 'idna' in project.lockfile._data[lockfile_section].keys() - if sync: - c = vistir.misc.run(["{0}".format(project.venv.python), "-c", "import requests"], - nospin=True, block=True, return_object=True) - assert c.returncode == 0, (c.out, c.err) - assert project.venv.is_installed("requests") or project.is_installed("requests") - assert project.venv.is_installed("idna") or project.is_installed("idna") + for pkg in deps: + assert pkg in project.lockfile._data[lockfile_section].keys() + if sync: + assert project.is_installed(pkg) remove = "default" if not is_dev else "dev" - retcode = passa.actions.remove.remove(project=project, packages=["requests",], sync=sync, only=remove) + retcode = passa.actions.remove.remove(project=project, packages=[req], sync=sync, only=remove) assert not retcode project.reload() - assert "requests" not in project.lockfile._data[lockfile_section].keys() - assert "idna" not in project.lockfile._data[lockfile_section].keys() - if sync: - assert not project.venv.is_installed("requests") - assert not project.venv.is_installed("idna") + for pkg in deps: + assert pkg not in project.lockfile._data[lockfile_section].keys() + if sync: + assert not project.is_installed(pkg) -@pytest.mark.parametrize( - 'sync', (True, False), -) -@pytest.mark.parametrize( - 'is_dev', (True, False) -) -def test_remove_editable(project, sync, is_dev): +@pytest.mark.parametrize('line', [ + "git+https://github.com/testing/demo.git#egg=demo", + "{}/git/github.com/testing/demo.git".format(FIXTURES_DIR) +]) +def test_remove_editable(project, sync, is_dev, line): add_kwargs = { "project": project, "packages": [], - "editables": ["git+https://github.com/sarugaku/shellingham.git@1.2.1#egg=shellingham",], + "editables": [line], "dev": is_dev, "sync": sync, "clean": False @@ -94,31 +54,26 @@ def test_remove_editable(project, sync, is_dev): retcode = passa.actions.add.add_packages(**add_kwargs) assert not retcode lockfile_section = "default" if not is_dev else "develop" - assert 'shellingham' in project.lockfile._data[lockfile_section].keys() + assert 'demo' in project.lockfile._data[lockfile_section].keys() if sync: - c = vistir.misc.run(["{0}".format(project.venv.python), "-c", "import shellingham"], - nospin=True, block=True, return_object=True) - assert c.returncode == 0, (c.out, c.err) - assert project.venv.is_installed("shellingham") or project.is_installed("shellingham") + assert project.is_installed("demo") remove = "default" if not is_dev else "dev" - retcode = passa.actions.remove.remove(project=project, packages=["shellingham",], sync=sync, only=remove) + retcode = passa.actions.remove.remove(project=project, packages=["demo",], sync=sync, only=remove) assert not retcode project.reload() - assert "shellingham" not in project.lockfile._data[lockfile_section].keys() + assert "demo" not in project.lockfile._data[lockfile_section].keys() if sync: - assert not project.venv.is_installed("shellingham") + assert not project.is_installed("demo") -@pytest.mark.parametrize( - 'sync', (True, False), -) -@pytest.mark.parametrize( - 'is_dev', (True, False) -) -def test_remove_sdist(project, is_dev, sync): +@pytest.mark.parametrize('link', [ + "flask/Flask-0.12.2-py2.py3-none-any.whl", + "flask/Flask-0.12.2.tar.gz" +]) +def test_remove_file_links(project, is_dev, sync, pypi, link): add_kwargs = { "project": project, - "packages": ["arrow"], + "packages": ["{}/{}".format(pypi.url, link)], "editables": [], "dev": is_dev, "sync": sync, @@ -127,16 +82,15 @@ def test_remove_sdist(project, is_dev, sync): retcode = passa.actions.add.add_packages(**add_kwargs) assert not retcode lockfile_section = "default" if not is_dev else "develop" - assert 'arrow' in project.lockfile._data[lockfile_section].keys() + assert 'flask' in project.lockfile._data[lockfile_section].keys() if sync: - c = vistir.misc.run(["{0}".format(project.venv.python), "-c", "import arrow"], - nospin=True, block=True, return_object=True) - assert c.returncode == 0, (c.out, c.err) - assert project.venv.is_installed("arrow") or project.is_installed("arrow") + assert project.is_installed("flask") + assert project.is_installed("jinja2") remove = "default" if not is_dev else "dev" - retcode = passa.actions.remove.remove(project=project, packages=["arrow",], sync=sync, only=remove) + retcode = passa.actions.remove.remove(project=project, packages=["flask"], sync=sync, only=remove) assert not retcode project.reload() - assert "arrow" not in project.lockfile._data[lockfile_section].keys() + assert "flask" not in project.lockfile._data[lockfile_section].keys() if sync: - assert not project.venv.is_installed("arrow") + assert not project.is_installed("flask") + assert not project.is_installed("jinja2") diff --git a/tests/conftest.py b/tests/conftest.py index 7c32c48..b71b182 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,22 +1,29 @@ # -*- coding=utf-8 -*- import os -import pytest -import passa -import passa.models.projects -import passa.cli.options -import mork +import six +from collections import deque, namedtuple +import shutil + import pkg_resources import plette -import sys +import pytest import vistir -from collections import deque - +import passa +import passa.cli.options +# import mork +import passa.models.environments +import passa.models.projects +import passa.models.synchronizers +from passa.models.caches import DependencyCache, RequiresPythonCache +from requirementslib import Requirement +from pytest_pypi.app import prepare_packages +from tests import PYPI_VENDOR_DIR, FIXTURES_DIR DEFAULT_PIPFILE_CONTENTS = """ [[source]] name = "pypi" -url = "https://pypi.org/simple" +url = "{pypi}/simple" verify_ssl = true [packages] @@ -24,6 +31,10 @@ [dev-packages] """.strip() +prepare_packages(PYPI_VENDOR_DIR) + +_Distro = namedtuple('Distro', 'key,version') + @pytest.fixture(scope="session") def working_set_extension(): @@ -34,69 +45,114 @@ def working_set_extension(): while requirements: req = requirements.popleft() dist = pkg_resources.working_set.find(req) - assert dist, req dists.add(dist) requirements.extend(dist.requires()) return dists +class InstallManager(passa.models.synchronizers.InstallManager): + + def __init__(self, *args, **kwargs): + super(InstallManager, self).__init__(*args, **kwargs) + self.working_set = set() + + def get_working_set(self): + return self.working_set + + def is_installation_local(self, name): + return any(name == dist.key for dist in self.working_set) + + def install(self, req): + if isinstance(req, six.string_types): + req = Requirement.from_line(req) + if req.is_vcs: + dist = _Distro(req.name, req.req.ref) + else: + dist = _Distro(req.name, req.get_version()) + self.working_set.add(dist) + return True + + def remove(self, name): + dist = next((dist for dist in self.working_set if dist.key == name), None) + if not dist: + return False + self.working_set.remove(dist) + return True + + @pytest.fixture(scope="function") -def virtualenv(tmpdir_factory): - venv_dir = tmpdir_factory.mktemp("passa-testenv") - print("Creating virtualenv {0!r}".format(venv_dir.strpath)) - c = vistir.misc.run([sys.executable, "-m", "virtualenv", venv_dir.strpath], - return_object=True, block=True, nospin=True) - if c.returncode == 0: - print("Virtualenv created...") - return venv_dir - raise RuntimeError("Failed creating virtualenv for testing...{0!r}".format(c.err.strip())) +def install_manager(): + return InstallManager() class _Project(passa.cli.options.Project): - def __init__(self, root, venv=None, working_set_extension=[]): - self.path = root - self.venv = venv + def __init__(self, root, environment=None, working_set_extension=[]): + self.path = vistir.compat.Path(root).absolute() self.working_set_extension = working_set_extension - super(_Project, self).__init__(self.path, venv=venv) + super(_Project, self).__init__(root, environment=environment) self.pipfile_instance = vistir.compat.Path(self.pipfile_location) self.lockfile_instance = vistir.compat.Path(self.lockfile_location) def reload(self): self._p = passa.models.projects.ProjectFile.read( - os.path.join(self.path, "Pipfile"), + self.path.joinpath("Pipfile").as_posix(), plette.Pipfile, ) self._l = passa.models.projects.ProjectFile.read( - os.path.join(self.path, "Pipfile.lock"), + self.path.joinpath("Pipfile.lock").as_posix(), plette.Lockfile, invalid_ok=True, ) + @pytest.fixture(scope="function") -def project_directory(tmpdir_factory): +def project(tmpdir_factory, pypi, install_manager, mocker): project_dir = tmpdir_factory.mktemp("passa-project") - project_dir.join("Pipfile").write(DEFAULT_PIPFILE_CONTENTS) - with vistir.contextmanagers.cd(project_dir.strpath): - yield project_dir + project_dir.join("Pipfile").write(DEFAULT_PIPFILE_CONTENTS.format(pypi=pypi.url)) + with vistir.contextmanagers.cd(project_dir.strpath), vistir.contextmanagers.temp_environ(): + mocker.patch("passa.models.synchronizers.InstallManager", return_value=install_manager) + cache_path = project_dir.join(".cache").strpath + os.environ["PIP_INDEX_URL"] = "{}/simple".format(pypi.url) + os.environ["PASSA_CACHE_DIR"] = cache_path + mocker.patch("passa.models.caches.CACHE_DIR", cache_path) + mocker.patch("passa.internals._pip.CACHE_DIR", cache_path) + mocker.patch("requirementslib.models.setup_info.CACHE_DIR", cache_path) + mocker.patch( + "passa.internals.dependencies.DEPENDENCY_CACHE", + DependencyCache(cache_path) + ) + mocker.patch( + "passa.internals.dependencies.REQUIRES_PYTHON_CACHE", + RequiresPythonCache(cache_path) + ) + os.environ["PIP_SRC"] = project_dir.join("src").strpath + p = _Project(project_dir.strpath) + p.is_installed = lambda x: install_manager.is_installation_local(x) + yield p -@pytest.fixture -def tmpvenv(virtualenv, tmpdir): - venv_srcdir = virtualenv.join("src").mkdir() - venv = mork.virtualenv.VirtualEnv(virtualenv.strpath) - venv.run(["pip", "install", "--upgrade", "mork", "setuptools"]) - with vistir.contextmanagers.temp_environ(): - os.environ["PACKAGEBUILDER_CACHE_DIR"] = tmpdir.strpath - os.environ["PIP_QUIET"] = "1" - os.environ["PIP_SRC"] = venv_srcdir.strpath - venv.is_installed = lambda x: any(d for d in venv.get_distributions() if d.project_name == x) - yield venv +def mock_git_obtain(self, location): + url, _ = self.get_url_rev_options(self.url) + parsed_url = six.moves.urllib_parse.urlparse(url) + path = '{}{}'.format(parsed_url.netloc, parsed_url.path) + source_dir = os.path.join(FIXTURES_DIR, 'git', path) + shutil.rmtree(location, ignore_errors=True) + shutil.copytree(source_dir, location) -@pytest.fixture(scope="function") -def project(project_directory, working_set_extension, tmpvenv): - # resolved = tmpvenv.resolve_dist(passa_dist, tmpvenv.base_working_set) - with tmpvenv.activated(extra_dists=list(working_set_extension)): - project = _Project(project_directory.strpath, venv=tmpvenv, working_set_extension=working_set_extension) - project.is_installed = lambda x: any(d for d in tmpvenv.get_working_set() if d.project_name == x) - yield project +@pytest.fixture(autouse=True) +def setup(mocker): + mocker.patch("pip._internal.vcs.git.Git.obtain", new=mock_git_obtain) + p = mocker.patch("pip._internal.vcs.git.Git.get_revision") + p.return_value = 'c55ee5cc8230a338a8a942704a9fe7eff8f88a1c' + yield + + +@pytest.fixture(params=[True, False]) +def is_dev(request): + return request.param + + +@pytest.fixture(params=[True, False]) +def sync(request): + return request.param diff --git a/tests/fixtures/git/github.com/testing/demo.git/demo.py b/tests/fixtures/git/github.com/testing/demo.git/demo.py new file mode 100644 index 0000000..b8023d8 --- /dev/null +++ b/tests/fixtures/git/github.com/testing/demo.git/demo.py @@ -0,0 +1 @@ +__version__ = '0.0.1' diff --git a/tests/fixtures/git/github.com/testing/demo.git/setup.py b/tests/fixtures/git/github.com/testing/demo.git/setup.py new file mode 100644 index 0000000..1d86082 --- /dev/null +++ b/tests/fixtures/git/github.com/testing/demo.git/setup.py @@ -0,0 +1,14 @@ +from setuptools import setup + +setup( + name="demo", + version="0.0.1", + license="MIT", + py_modules="demo.py", + description="Demo package", + install_requires=["requests"], + extras_require={ + "time": ["pytz"], + "csv": ["tablib"] + } +) diff --git a/tests/fixtures/git/github.com/testing/no_dep.git/no_dep.py b/tests/fixtures/git/github.com/testing/no_dep.git/no_dep.py new file mode 100644 index 0000000..b8023d8 --- /dev/null +++ b/tests/fixtures/git/github.com/testing/no_dep.git/no_dep.py @@ -0,0 +1 @@ +__version__ = '0.0.1' diff --git a/tests/fixtures/git/github.com/testing/no_dep.git/setup.py b/tests/fixtures/git/github.com/testing/no_dep.git/setup.py new file mode 100644 index 0000000..2699c41 --- /dev/null +++ b/tests/fixtures/git/github.com/testing/no_dep.git/setup.py @@ -0,0 +1,9 @@ +from setuptools import setup + +setup( + name="no_dep", + version="0.0.1", + license="MIT", + py_modules="no_dep.py", + description="Demo package" +) diff --git a/tests/pypi b/tests/pypi new file mode 160000 index 0000000..38f55ba --- /dev/null +++ b/tests/pypi @@ -0,0 +1 @@ +Subproject commit 38f55ba5883f1ce47c6f1f46feecc0d318c444a5 diff --git a/tests/pytest-pypi/DESCRIPTION.rst b/tests/pytest-pypi/DESCRIPTION.rst new file mode 100644 index 0000000..823e473 --- /dev/null +++ b/tests/pytest-pypi/DESCRIPTION.rst @@ -0,0 +1,5 @@ +pytest-pypi +=========== + +Easily test your HTTP library against a local copy of PyPI. +This is an internal pytest plugin of pipenv. diff --git a/tests/pytest-pypi/MANIFEST.in b/tests/pytest-pypi/MANIFEST.in new file mode 100644 index 0000000..4104865 --- /dev/null +++ b/tests/pytest-pypi/MANIFEST.in @@ -0,0 +1,4 @@ +# If using Python 2.6 or less, then have to include package data, even though +# it's already declared in setup.py +include pytest_httpbin/certs/* +include DESCRIPTION.rst diff --git a/tests/pytest-pypi/README.md b/tests/pytest-pypi/README.md new file mode 100644 index 0000000..31b3380 --- /dev/null +++ b/tests/pytest-pypi/README.md @@ -0,0 +1,4 @@ +# pytest-pypi + +Easily test your HTTP library against a local copy of PyPI. +This is an internal pytest plugin of pipenv. diff --git a/tests/pytest-pypi/pytest_pypi/__init__.py b/tests/pytest-pypi/pytest_pypi/__init__.py new file mode 100644 index 0000000..7f566e8 --- /dev/null +++ b/tests/pytest-pypi/pytest_pypi/__init__.py @@ -0,0 +1,14 @@ +import os + +import pytest + + +here = os.path.dirname(__file__) +version_file = os.path.join(here, "version.py") + +with open(version_file) as f: + code = compile(f.read(), version_file, 'exec') + exec(code) + +use_class_based_httpbin = pytest.mark.usefixtures("class_based_pypi") +use_class_based_httpbin_secure = pytest.mark.usefixtures("class_based_pypi_secure") diff --git a/tests/pytest-pypi/pytest_pypi/app.py b/tests/pytest-pypi/pytest_pypi/app.py new file mode 100644 index 0000000..05f5c37 --- /dev/null +++ b/tests/pytest-pypi/pytest_pypi/app.py @@ -0,0 +1,227 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function + +import contextlib +import io +import json +import os +from tarfile import is_tarfile +from zipfile import is_zipfile + +import requests +import six +from flask import Flask, abort, jsonify, redirect, render_template, send_file +from six.moves import xmlrpc_client + +app = Flask(__name__) +session = requests.Session() + +packages = {} +ARTIFACTS = {} +if six.PY2: + FileNotFoundError = OSError + + +@contextlib.contextmanager +def xml_pypi_server(server): + transport = xmlrpc_client.Transport() + client = xmlrpc_client.ServerProxy(server, transport) + try: + yield client + finally: + transport.close() + + +def get_pypi_package_names(): + pypi_packages = set() + with xml_pypi_server("https://pypi.org/pypi") as client: + pypi_packages = set(client.list_packages()) + return pypi_packages + + +class Package(object): + """Package represents a collection of releases from one or more directories""" + + def __init__(self, name): + super(Package, self).__init__() + self.name = name + self.releases = {} + self._package_dirs = set() + + @property + def json(self): + for path in self._package_dirs: + try: + with open(os.path.join(path, 'api.json')) as f: + return json.load(f) + except FileNotFoundError: + r = session.get('https://pypi.org/pypi/{0}/json'.format(self.name)) + response = r.json() + releases = response["releases"] + files = { + pkg for pkg_dir in self._package_dirs + for pkg in os.listdir(pkg_dir) + } + for release in list(releases.keys()): + values = ( + r for r in releases[release] if r["filename"] in files + ) + values = list(values) + if values: + releases[release] = values + else: + del releases[release] + response["releases"] = releases + with io.open(os.path.join(path, "api.json"), "w") as fh: + json.dump(response, fh, indent=4) + return response + + def __repr__(self): + return "/') +def simple_package(package): + if package in packages and packages[package].releases: + return render_template('package.html', package=packages[package]) + else: + try: + r = requests.get("https://pypi.org/simple/{0}".format(package)) + r.raise_for_status() + except Exception: + abort(404) + else: + return render_template( + 'package_pypi.html', package_contents=r.text + ) + + +@app.route('/artifacts//') +def simple_artifact(artifact): + if artifact in ARTIFACTS: + return render_template('artifact.html', artifact=ARTIFACTS[artifact]) + else: + abort(404) + + +@app.route('//') +def serve_package(package, release): + if package in packages: + package = packages[package] + + if release in package.releases: + return send_file(package.releases[release]) + + abort(404) + + +@app.route('/artifacts//') +def serve_artifact(artifact, fn): + if artifact in ARTIFACTS: + artifact = ARTIFACTS[artifact] + if fn in artifact.files: + return send_file(artifact.files[fn]) + abort(404) + + +@app.route('/pypi//json') +def json_for_package(package): + return jsonify(packages[package].json) + # try: + # except Exception: + # r = session.get('https://pypi.org/pypi/{0}/json'.format(package)) + # return jsonify(r.json()) + + +if __name__ == '__main__': + PYPI_VENDOR_DIR = os.environ.get('PYPI_VENDOR_DIR', './pypi') + PYPI_VENDOR_DIR = os.path.abspath(PYPI_VENDOR_DIR) + prepare_packages(PYPI_VENDOR_DIR) + prepare_fixtures(os.path.join(PYPI_VENDOR_DIR, "fixtures")) + + app.run() diff --git a/tests/pytest-pypi/pytest_pypi/certs.py b/tests/pytest-pypi/pytest_pypi/certs.py new file mode 100644 index 0000000..b73fc63 --- /dev/null +++ b/tests/pytest-pypi/pytest_pypi/certs.py @@ -0,0 +1,22 @@ +""" +certs.py +~~~~~~~~ + +This module returns the preferred default CA certificate bundle. + +If you are packaging pytest-httpbin, e.g., for a Linux distribution or a +managed environment, you can change the definition of where() to return a +separately packaged CA bundle. +""" + +import os.path + + +def where(): + """Return the preferred certificate bundle.""" + # vendored bundle inside Requests + return os.path.join(os.path.abspath(os.path.dirname(__file__)), 'certs', 'cacert.pem') + + +if __name__ == '__main__': + print(where()) diff --git a/tests/pytest-pypi/pytest_pypi/certs/cacert.pem b/tests/pytest-pypi/pytest_pypi/certs/cacert.pem new file mode 100644 index 0000000..d9a47aa --- /dev/null +++ b/tests/pytest-pypi/pytest_pypi/certs/cacert.pem @@ -0,0 +1,63 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + dd:39:30:16:60:55:90:7c + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=US, ST=Hawaii, O=kevin1024, CN=pytest-httpbin Certificate Authority + Validity + Not Before: Jun 26 18:16:59 2015 GMT + Not After : Jun 18 18:16:59 2045 GMT + Subject: C=US, ST=Hawaii, O=kevin1024, CN=pytest-httpbin Certificate Authority + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + RSA Public Key: (1024 bit) + Modulus (1024 bit): + 00:bd:80:fd:e4:96:0e:3b:5e:35:9b:83:00:34:88: + 64:5a:50:53:0e:1d:94:76:c9:dc:e7:b5:59:1e:d4: + 82:55:36:a6:b4:41:2c:60:ad:76:f0:cd:42:a0:0f: + 4a:1c:0d:d7:29:da:c3:d9:c0:ea:f1:48:e0:66:4d: + 4b:7c:ff:d6:5e:e0:73:89:53:8b:6e:6c:57:7d:bd: + e9:d0:46:39:5d:85:a5:f1:3a:d4:3d:83:19:03:44: + 93:71:2c:5e:d7:61:8e:db:cc:80:d0:f1:c0:47:bf: + 98:8f:06:40:e1:f7:41:ee:ed:a7:57:0d:a6:4c:26: + 75:8e:f1:78:d3:80:ad:9c:e9 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + AE:05:EF:BD:A9:CE:BB:A6:D8:0E:EB:C4:8C:72:2F:13:E5:CD:AA:CA + X509v3 Authority Key Identifier: + keyid:AE:05:EF:BD:A9:CE:BB:A6:D8:0E:EB:C4:8C:72:2F:13:E5:CD:AA:CA + DirName:/C=US/ST=Hawaii/O=kevin1024/CN=pytest-httpbin Certificate Authority + serial:DD:39:30:16:60:55:90:7C + + X509v3 Basic Constraints: + CA:TRUE + Signature Algorithm: sha1WithRSAEncryption + bc:0c:b4:21:03:bf:35:bf:88:9f:de:06:23:f4:e3:8f:bc:34: + b5:8b:af:bf:31:5d:17:44:2c:72:c9:88:25:d1:c7:d0:1c:70: + 06:82:a5:fa:fa:d7:b9:16:64:c2:08:54:1e:4c:93:9f:22:4e: + e5:4f:a7:71:e5:6e:14:31:e9:41:e2:33:23:8b:c8:01:c3:2a: + 66:a8:d8:df:ef:ee:7b:bb:84:f4:78:a6:ca:8f:29:aa:d5:fa: + 8a:73:94:0c:32:53:c8:93:bd:fc:c4:60:4d:9a:80:4f:c6:d4: + 27:44:a2:37:63:6c:97:04:ce:e3:6a:6f:d3:84:0d:b4:74:1f: + 49:eb +-----BEGIN CERTIFICATE----- +MIIDBzCCAnCgAwIBAgIJAN05MBZgVZB8MA0GCSqGSIb3DQEBBQUAMGExCzAJBgNV +BAYTAlVTMQ8wDQYDVQQIEwZIYXdhaWkxEjAQBgNVBAoTCWtldmluMTAyNDEtMCsG +A1UEAxMkcHl0ZXN0LWh0dHBiaW4gQ2VydGlmaWNhdGUgQXV0aG9yaXR5MB4XDTE1 +MDYyNjE4MTY1OVoXDTQ1MDYxODE4MTY1OVowYTELMAkGA1UEBhMCVVMxDzANBgNV +BAgTBkhhd2FpaTESMBAGA1UEChMJa2V2aW4xMDI0MS0wKwYDVQQDEyRweXRlc3Qt +aHR0cGJpbiBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkwgZ8wDQYJKoZIhvcNAQEBBQAD +gY0AMIGJAoGBAL2A/eSWDjteNZuDADSIZFpQUw4dlHbJ3Oe1WR7UglU2prRBLGCt +dvDNQqAPShwN1ynaw9nA6vFI4GZNS3z/1l7gc4lTi25sV3296dBGOV2FpfE61D2D +GQNEk3EsXtdhjtvMgNDxwEe/mI8GQOH3Qe7tp1cNpkwmdY7xeNOArZzpAgMBAAGj +gcYwgcMwHQYDVR0OBBYEFK4F772pzrum2A7rxIxyLxPlzarKMIGTBgNVHSMEgYsw +gYiAFK4F772pzrum2A7rxIxyLxPlzarKoWWkYzBhMQswCQYDVQQGEwJVUzEPMA0G +A1UECBMGSGF3YWlpMRIwEAYDVQQKEwlrZXZpbjEwMjQxLTArBgNVBAMTJHB5dGVz +dC1odHRwYmluIENlcnRpZmljYXRlIEF1dGhvcml0eYIJAN05MBZgVZB8MAwGA1Ud +EwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAvAy0IQO/Nb+In94GI/Tjj7w0tYuv +vzFdF0QscsmIJdHH0BxwBoKl+vrXuRZkwghUHkyTnyJO5U+nceVuFDHpQeIzI4vI +AcMqZqjY3+/ue7uE9Himyo8pqtX6inOUDDJTyJO9/MRgTZqAT8bUJ0SiN2NslwTO +42pv04QNtHQfSes= +-----END CERTIFICATE----- diff --git a/tests/pytest-pypi/pytest_pypi/certs/cert.pem b/tests/pytest-pypi/pytest_pypi/certs/cert.pem new file mode 100644 index 0000000..5d4452b --- /dev/null +++ b/tests/pytest-pypi/pytest_pypi/certs/cert.pem @@ -0,0 +1,73 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + dd:39:30:16:60:55:90:7e + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=US, ST=Hawaii, O=kevin1024, CN=pytest-httpbin Certificate Authority + Validity + Not Before: Jun 26 18:20:35 2015 GMT + Not After : Jun 23 18:20:35 2025 GMT + Subject: C=US, ST=Hawaii, O=kevin1024, OU=kevin1024, CN=127.0.0.1 + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + RSA Public Key: (2048 bit) + Modulus (2048 bit): + 00:ce:b4:0f:0b:86:17:68:24:6f:7c:25:32:73:81: + bd:55:38:05:ae:09:29:00:c0:f1:99:30:5a:7f:05: + 9f:e7:e9:d3:ce:d0:dd:4f:73:c8:bf:65:04:94:e5: + 11:8e:1d:91:f0:88:85:3e:48:d3:09:5b:3f:8f:97: + 95:34:bf:8d:00:cb:70:d2:c1:2b:34:dd:99:1d:86: + 9b:90:54:a5:de:18:c4:03:3d:53:f0:dd:cc:6d:ec: + fb:b9:93:ab:19:85:05:63:2d:34:a6:47:42:71:3b: + e4:1e:4a:4c:d9:60:d4:6b:d6:51:a8:4a:30:70:2e: + 6c:62:a2:34:da:cf:30:34:97:a4:9d:17:72:0b:b2: + 37:69:e2:ca:b6:d5:9f:46:c5:eb:cf:dc:46:b0:fe: + ef:37:5e:4f:eb:f3:50:4d:2c:4e:c2:0c:e4:0c:63: + c2:d8:ab:a3:d6:a0:12:bf:d6:fc:3f:b6:4c:dc:2b: + 9b:c5:ae:83:4d:3b:3c:19:85:50:88:82:a2:5f:ff: + de:98:60:fc:12:3a:55:c3:4f:0a:e9:1f:aa:12:cb: + f8:ce:14:d6:ed:89:ff:c7:ea:3b:fe:97:87:54:eb: + 62:de:cd:ef:6b:e2:9e:47:82:77:55:59:4f:b8:ad: + 1b:e0:9d:1a:28:16:9f:6a:cb:b2:44:f9:65:c3:c4: + 03:09 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: + CA:FALSE + Netscape Comment: + OpenSSL Generated Certificate + X509v3 Subject Key Identifier: + 1E:28:41:6B:12:03:41:29:64:0D:E5:C3:E3:F7:9E:82:0C:66:1E:B9 + X509v3 Authority Key Identifier: + keyid:AE:05:EF:BD:A9:CE:BB:A6:D8:0E:EB:C4:8C:72:2F:13:E5:CD:AA:CA + + Signature Algorithm: sha1WithRSAEncryption + 67:8c:6d:a1:2f:b3:35:87:a3:c0:04:92:5d:8a:8b:f8:51:6e: + 94:88:59:ed:66:b2:54:b0:a2:3d:7a:05:ee:19:17:a6:0b:3b: + 20:f7:d2:73:2c:f0:b9:ad:2e:5d:45:11:5d:8d:33:5c:69:7f: + 4a:c5:8c:10:3e:35:b4:39:d7:52:66:bc:02:d8:4d:d0:ba:a1: + ae:55:f5:36:01:17:97:40:1a:9d:6a:e0:b8:33:be:2d:98:b7: + 5b:92:6a:77:a7:d9:f5:5b:a4:5f:fa:aa:5b:c1:6b:4d:0c:b7: + 5a:4c:47:b2:f7:90:a3:ff:6f:8c:fd:f2:60:38:53:29:71:48: + d7:69 +-----BEGIN CERTIFICATE----- +MIIDODCCAqGgAwIBAgIJAN05MBZgVZB+MA0GCSqGSIb3DQEBBQUAMGExCzAJBgNV +BAYTAlVTMQ8wDQYDVQQIEwZIYXdhaWkxEjAQBgNVBAoTCWtldmluMTAyNDEtMCsG +A1UEAxMkcHl0ZXN0LWh0dHBiaW4gQ2VydGlmaWNhdGUgQXV0aG9yaXR5MB4XDTE1 +MDYyNjE4MjAzNVoXDTI1MDYyMzE4MjAzNVowWjELMAkGA1UEBhMCVVMxDzANBgNV +BAgTBkhhd2FpaTESMBAGA1UEChMJa2V2aW4xMDI0MRIwEAYDVQQLEwlrZXZpbjEw +MjQxEjAQBgNVBAMTCTEyNy4wLjAuMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC +AQoCggEBAM60DwuGF2gkb3wlMnOBvVU4Ba4JKQDA8ZkwWn8Fn+fp087Q3U9zyL9l +BJTlEY4dkfCIhT5I0wlbP4+XlTS/jQDLcNLBKzTdmR2Gm5BUpd4YxAM9U/DdzG3s ++7mTqxmFBWMtNKZHQnE75B5KTNlg1GvWUahKMHAubGKiNNrPMDSXpJ0XcguyN2ni +yrbVn0bF68/cRrD+7zdeT+vzUE0sTsIM5Axjwtiro9agEr/W/D+2TNwrm8Wug007 +PBmFUIiCol//3phg/BI6VcNPCukfqhLL+M4U1u2J/8fqO/6Xh1TrYt7N72vinkeC +d1VZT7itG+CdGigWn2rLskT5ZcPEAwkCAwEAAaN7MHkwCQYDVR0TBAIwADAsBglg +hkgBhvhCAQ0EHxYdT3BlblNTTCBHZW5lcmF0ZWQgQ2VydGlmaWNhdGUwHQYDVR0O +BBYEFB4oQWsSA0EpZA3lw+P3noIMZh65MB8GA1UdIwQYMBaAFK4F772pzrum2A7r +xIxyLxPlzarKMA0GCSqGSIb3DQEBBQUAA4GBAGeMbaEvszWHo8AEkl2Ki/hRbpSI +We1mslSwoj16Be4ZF6YLOyD30nMs8LmtLl1FEV2NM1xpf0rFjBA+NbQ511JmvALY +TdC6oa5V9TYBF5dAGp1q4Lgzvi2Yt1uSanen2fVbpF/6qlvBa00Mt1pMR7L3kKP/ +b4z98mA4UylxSNdp +-----END CERTIFICATE----- diff --git a/tests/pytest-pypi/pytest_pypi/certs/key.pem b/tests/pytest-pypi/pytest_pypi/certs/key.pem new file mode 100644 index 0000000..041c4b6 --- /dev/null +++ b/tests/pytest-pypi/pytest_pypi/certs/key.pem @@ -0,0 +1,28 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEAzrQPC4YXaCRvfCUyc4G9VTgFrgkpAMDxmTBafwWf5+nTztDd +T3PIv2UElOURjh2R8IiFPkjTCVs/j5eVNL+NAMtw0sErNN2ZHYabkFSl3hjEAz1T +8N3Mbez7uZOrGYUFYy00pkdCcTvkHkpM2WDUa9ZRqEowcC5sYqI02s8wNJeknRdy +C7I3aeLKttWfRsXrz9xGsP7vN15P6/NQTSxOwgzkDGPC2Kuj1qASv9b8P7ZM3Cub +xa6DTTs8GYVQiIKiX//emGD8EjpVw08K6R+qEsv4zhTW7Yn/x+o7/peHVOti3s3v +a+KeR4J3VVlPuK0b4J0aKBafasuyRPllw8QDCQIDAQABAoIBAQCJ//iTbwCtjLXJ +omPebd3jyTUxjfgMAsTJy1h/uVea06ePSi6W3uxFq8G1ToG76c4HUn3yqVgLxRnY +WhFJWCFhSHGYo1KfRtr0tWuinoDmmI40w3sJMmtLcI5WxVnT/dUs839VC/o18xBH +kL9h2Z24KSv3OSDBpJzD9Rtogi7izK8DSQoANBMDEmPPJ5UJBLPjdZn04i6BYZCM +U/+ZADHKXbq6I+7RAcbPJbkvrbBEP234KZvIdw1eIAIZufQBQuDhnwS0Fi9iY/EP +awoYa9HLgFjh+iprhwh+2SDyIp8DA+4HrY1tXAyzCqjgLn/X8wifOUrZECYj1i65 +EOiryxMBAoGBAPjmvIwBRxnr1OsKX3gCFoZr+Zu5RjACD9IOSV17cv7glZQVfXBR +REDBoL7CmZrhsW4zBK0YWz30Dx7TGBniTFJ3e8IZJ7Th8PSOhIRYWqqFQ78YBHFi +VcpPOBswy1i8BM9FE0GyF1zusmz8Ak2hFr/IHVkIqHwWvkTI6gGhbJ2RAoGBANSZ +OqEWJKbRX9nuRqSdROqLOtUgWXZ78yvcQaaifyZHEFSKZZjc5MXT96lVd1PyGGAY +uyjAqdd5LiwsS9Rw1cuC5fix2ihH5KFq7EnEJA/zdy91YdO6xmAyBOtjuTHsNj93 +if4ilib290/mRKXeI1zpzzWHsvL9Az5spqlkljH5AoGAfln7ewMnCfSbCJoibrR4 +pNJpSvEZvUM+rr6L5cXGUbbGl/70x7CpekoRBOWavnI19SA3Dnvfzap4hohYosMr +RW3cSGMmsf9Ep5E1mk2T8R5njrltf/WQYXwnmj4B7FC+DE4fgWkbzRRrRUIFFU1i +VAcNRuZLSXruKdLoX92HWtECgYEAhpTlf3n0A8JBKkVjZOvF56/xs19CIvY+LsLE +sIbndMTBurLNs+IJ1I3llsVqv7Je6d5eBGNKYQPuTbpQ2o//V1Bq4m88CgnQ2rpE +EEJhDdPy3BEzt4Ph9p1Tbet4HflJMg4rRbyBTvNCBctgI5wmyLeeG2Xmy1mNhyPi +sRLi3YkCgYEAiHMsniJc1gZBevjtnqGTPdUo0syAnkZ7RUk/Piur/c0Altkgu5vK +I7p3DbkHBAMDjpAZs1kpfmR4sTYKke+IQDxj2pOZEyYnmQxlGdy8xxoE9dWQeDeg +Le+R83OAKjU4LHpH7hhJMR8X60MJaWC1BDACFO35kqIzvtCYxgEoOiI= +-----END RSA PRIVATE KEY----- + diff --git a/tests/pytest-pypi/pytest_pypi/plugin.py b/tests/pytest-pypi/pytest_pypi/plugin.py new file mode 100644 index 0000000..83cd73f --- /dev/null +++ b/tests/pytest-pypi/pytest_pypi/plugin.py @@ -0,0 +1,43 @@ +from __future__ import absolute_import +import pytest +from .app import app as pypi_app +from . import serve, certs + + +@pytest.fixture(scope='session') +def pypi(request): + server = serve.Server(application=pypi_app) + server.start() + request.addfinalizer(server.stop) + return server + + +@pytest.fixture(scope='session') +def pypi_secure(request): + server = serve.SecureServer(application=pypi_app) + server.start() + request.addfinalizer(server.stop) + return server + + +@pytest.fixture(scope='session', params=['http', 'https']) +def pypi_both(request, pypi, pypi_secure): + if request.param == 'http': + return pypi + elif request.param == 'https': + return pypi_secure + + +@pytest.fixture(scope='class') +def class_based_pypi(request, pypi): + request.cls.pypi = pypi + + +@pytest.fixture(scope='class') +def class_based_pypi_secure(request, pypi_secure): + request.cls.pypi_secure = pypi_secure + + +@pytest.fixture(scope='function') +def pypi_ca_bundle(monkeypatch): + monkeypatch.setenv('REQUESTS_CA_BUNDLE', certs.where()) diff --git a/tests/pytest-pypi/pytest_pypi/serve.py b/tests/pytest-pypi/pytest_pypi/serve.py new file mode 100644 index 0000000..07a92dc --- /dev/null +++ b/tests/pytest-pypi/pytest_pypi/serve.py @@ -0,0 +1,134 @@ +import os +import threading +import ssl +from wsgiref.simple_server import WSGIServer, make_server, WSGIRequestHandler +from wsgiref.handlers import SimpleHandler +from six.moves.urllib.parse import urljoin + + +CERT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'certs') + + +class ServerHandler(SimpleHandler): + + server_software = 'Pytest-HTTPBIN/0.1.0' + http_version = '1.1' + + def cleanup_headers(self): + SimpleHandler.cleanup_headers(self) + self.headers['Connection'] = 'Close' + + def close(self): + try: + self.request_handler.log_request( + self.status.split(' ', 1)[0], self.bytes_sent + ) + finally: + SimpleHandler.close(self) + + +class Handler(WSGIRequestHandler): + + def handle(self): + """Handle a single HTTP request""" + + self.raw_requestline = self.rfile.readline() + if not self.parse_request(): # An error code has been sent, just exit + return + + handler = ServerHandler( + self.rfile, self.wfile, self.get_stderr(), self.get_environ() + ) + handler.request_handler = self # backpointer for logging + handler.run(self.server.get_app()) + + def get_environ(self): + """ + wsgiref simple server adds content-type text/plain to everything, this + removes it if it's not actually in the headers. + """ + # Note: Can't use super since this is an oldstyle class in python 2.x + environ = WSGIRequestHandler.get_environ(self).copy() + if self.headers.get('content-type') is None: + del environ['CONTENT_TYPE'] + return environ + + +class SecureWSGIServer(WSGIServer): + + def finish_request(self, request, client_address): + """ + Negotiates SSL and then mimics BaseServer behavior. + """ + request.settimeout(1.0) + try: + ssock = ssl.wrap_socket( + request, + keyfile=os.path.join(CERT_DIR, 'key.pem'), + certfile=os.path.join(CERT_DIR, 'cert.pem'), + server_side=True, + suppress_ragged_eofs=False, + ) + self.RequestHandlerClass(ssock, client_address, self) + except Exception as e: + print("pytest-httpbin server hit an exception serving request: %s" % e) + print("attempting to ignore so the rest of the tests can run") + # WSGIRequestHandler seems to close the socket for us. + # Thanks, WSGIRequestHandler!! + + +class Server(object): + """ + HTTP server running a WSGI application in its own thread. + """ + + port_envvar = 'HTTPBIN_HTTP_PORT' + + def __init__(self, host='127.0.0.1', port=0, application=None, **kwargs): + self.app = application + if self.port_envvar in os.environ: + port = int(os.environ[self.port_envvar]) + self._server = make_server( + host, + port, + self.app, + handler_class=Handler, + **kwargs + ) + self.host = self._server.server_address[0] + self.port = self._server.server_address[1] + self.protocol = 'http' + + self._thread = threading.Thread( + name=self.__class__, + target=self._server.serve_forever, + ) + + def __del__(self): + if hasattr(self, '_server'): + self.stop() + + def start(self): + self._thread.start() + + def __add__(self, other): + return self.url + other + + def stop(self): + self._server.shutdown() + + @property + def url(self): + return '{0}://{1}:{2}'.format(self.protocol, self.host, self.port) + + def join(self, url, allow_fragments=True): + return urljoin(self.url, url, allow_fragments=allow_fragments) + + +class SecureServer(Server): + port_envvar = 'HTTPBIN_HTTPS_PORT' + + def __init__(self, host='127.0.0.1', port=0, application=None, **kwargs): + kwargs['server_class'] = SecureWSGIServer + super(SecureServer, self).__init__(host, port, application, **kwargs) + self.protocol = 'https' diff --git a/tests/pytest-pypi/pytest_pypi/templates/artifact.html b/tests/pytest-pypi/pytest_pypi/templates/artifact.html new file mode 100644 index 0000000..5f4199c --- /dev/null +++ b/tests/pytest-pypi/pytest_pypi/templates/artifact.html @@ -0,0 +1,14 @@ + + + + + Links for {{ artifact.name }} + + +

Links for {{ artifact.name }}

+ {% for fn in artifact.files %} + {{ fn }} +
+ {% endfor %} + + diff --git a/tests/pytest-pypi/pytest_pypi/templates/artifacts.html b/tests/pytest-pypi/pytest_pypi/templates/artifacts.html new file mode 100644 index 0000000..6bee78d --- /dev/null +++ b/tests/pytest-pypi/pytest_pypi/templates/artifacts.html @@ -0,0 +1,13 @@ + + + + + Artifact Index + + + {% for artifact in artifacts %} + {{ artifact.name }} +
+ {% endfor %} + + diff --git a/tests/pytest-pypi/pytest_pypi/templates/package.html b/tests/pytest-pypi/pytest_pypi/templates/package.html new file mode 100644 index 0000000..26ba9ec --- /dev/null +++ b/tests/pytest-pypi/pytest_pypi/templates/package.html @@ -0,0 +1,14 @@ + + + + + Links for {{ package.name }} + + +

Links for {{ package.name }}

+ {% for release in package.releases %} + {{ release }} +
+ {% endfor %} + + diff --git a/tests/pytest-pypi/pytest_pypi/templates/package_pypi.html b/tests/pytest-pypi/pytest_pypi/templates/package_pypi.html new file mode 100644 index 0000000..217d8aa --- /dev/null +++ b/tests/pytest-pypi/pytest_pypi/templates/package_pypi.html @@ -0,0 +1,4 @@ + +{% autoescape false %} + {{ package_contents }} +{% endautoescape %} diff --git a/tests/pytest-pypi/pytest_pypi/templates/simple.html b/tests/pytest-pypi/pytest_pypi/templates/simple.html new file mode 100644 index 0000000..97f6755 --- /dev/null +++ b/tests/pytest-pypi/pytest_pypi/templates/simple.html @@ -0,0 +1,13 @@ + + + + + Simple Index + + + {% for package in packages %} + {{ package.name }} +
+ {% endfor %} + + \ No newline at end of file diff --git a/tests/pytest-pypi/pytest_pypi/version.py b/tests/pytest-pypi/pytest_pypi/version.py new file mode 100644 index 0000000..df9144c --- /dev/null +++ b/tests/pytest-pypi/pytest_pypi/version.py @@ -0,0 +1 @@ +__version__ = '0.1.1' diff --git a/tests/pytest-pypi/runtests.sh b/tests/pytest-pypi/runtests.sh new file mode 100644 index 0000000..6115f03 --- /dev/null +++ b/tests/pytest-pypi/runtests.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +py.test $1 -v -s diff --git a/tests/pytest-pypi/setup.cfg b/tests/pytest-pypi/setup.cfg new file mode 100644 index 0000000..79bc678 --- /dev/null +++ b/tests/pytest-pypi/setup.cfg @@ -0,0 +1,5 @@ +[bdist_wheel] +# This flag says that the code is written to work on both Python 2 and Python +# 3. If at all possible, it is good practice to do this. If you cannot, you +# will need to generate wheels for each Python version that you support. +universal=1 diff --git a/tests/pytest-pypi/setup.py b/tests/pytest-pypi/setup.py new file mode 100644 index 0000000..0bcad74 --- /dev/null +++ b/tests/pytest-pypi/setup.py @@ -0,0 +1,106 @@ +from setuptools import setup, find_packages, Command +import codecs +import os +import sys +from shutil import rmtree + +with open("pytest_pypi/version.py") as f: + code = compile(f.read(), "pytest_pypi/version.py", 'exec') + exec(code) + +__version__ = '0.1.1' +here = os.path.abspath(os.path.dirname(__file__)) + +# Get the long description from the relevant file +with codecs.open(os.path.join(here, 'DESCRIPTION.rst'), encoding='utf-8') as f: + long_description = f.read() + + +class UploadCommand(Command): + """Support setup.py upload.""" + + description = 'Build and publish the package.' + user_options = [] + + @staticmethod + def status(s): + """Prints things in bold.""" + print('\033[1m{0}\033[0m'.format(s)) + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + try: + self.status('Removing previous builds…') + rmtree(os.path.join(here, 'dist')) + except OSError: + pass + + self.status('Building Source and Wheel (universal) distribution…') + os.system('{0} setup.py sdist bdist_wheel --universal'.format(sys.executable)) + + self.status('Uploading the package to PyPI via Twine…') + os.system('twine upload dist/*') + + self.status('Pushing git tags…') + os.system('git tag v{0}'.format(__version__)) + os.system('git push --tags') + + sys.exit() + + +setup( + name="pytest-pypi", + + # There are various approaches to referencing the version. For a discussion, + # see http://packaging.python.org/en/latest/tutorial.html#version + version=__version__, + + description="Easily test your HTTP library against a local copy of pypi", + long_description=long_description, + + # The project URL. + url='https://github.com/pypa/pipenv/tree/master/tests/pytest-pypi', + + # Author details + author='Kenneth Reitz', + author_email='me@kennethreitz.org', + + # Choose your license + license='MIT', + + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Developers', + 'Topic :: Software Development :: Testing', + 'Topic :: Software Development :: Libraries', + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + ], + + # What does your project relate to? + keywords='pytest-pypi testing pytest pypi', + packages=find_packages(exclude=["contrib", "docs", "tests*"]), + include_package_data=True, # include files listed in MANIFEST.in + install_requires=['Flask', 'six'], + + # the following makes a plugin available to pytest + entry_points={ + 'pytest11': [ + 'pypi = pytest_pypi.plugin', + ] + }, + cmdclass={ + 'upload': UploadCommand, + }, +) diff --git a/tests/pytest-pypi/tox.ini b/tests/pytest-pypi/tox.ini new file mode 100644 index 0000000..9437a9e --- /dev/null +++ b/tests/pytest-pypi/tox.ini @@ -0,0 +1,10 @@ +# content of: tox.ini , put in same dir as setup.py + +[tox] +envlist = py26, py27, py33, py34, py35, py36, pypy, pypy3 + +[testenv] +deps = pytest + requests + py26: httpbin==0.5.0 +commands = ./runtests.sh {posargs:tests/} diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_markers.py b/tests/unit/test_markers.py similarity index 100% rename from tests/test_markers.py rename to tests/unit/test_markers.py diff --git a/tests/test_specifiers.py b/tests/unit/test_specifiers.py similarity index 100% rename from tests/test_specifiers.py rename to tests/unit/test_specifiers.py diff --git a/tox.ini b/tox.ini index 64e1959..4260b31 100644 --- a/tox.ini +++ b/tox.ini @@ -33,8 +33,9 @@ commands = [testenv:packaging] deps = - check-manifest - readme_renderer + setuptools + twine + readme_renderer[md] commands = - check-manifest - python setup.py check -m -r -s + python setup.py sdist bdist_wheel + twine check dist/*