diff --git a/Pipfile.lock b/Pipfile.lock index d421743..5f93a56 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -123,7 +123,7 @@ "sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79", "sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1" ], - "markers": "python_version >= '2.6' and python_version >= '2.7' and python_version < '2.8' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version >= '2.7' and python_version < '2.8' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '2.8' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.6" }, "first": { @@ -150,7 +150,7 @@ "hashes": [ "sha256:b6ee7066fea66e35f8d0acee24d98006de1a0a8a94a8ce6efe73a9a23c8d9826" ], - "markers": "python_version < '2.7' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '2.7' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.0.4" }, "importlib-metadata": { @@ -203,7 +203,9 @@ }, "passa": { "editable": true, - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "extras": [ + "virtualenv" + ], "path": "." }, "pathlib2": { @@ -238,7 +240,7 @@ "sha256:c0e3553c1e581d8423daccbd825789c6e7f29b7d9e00e5331b12e1642a1a26d3", "sha256:dde5d525cf5f0cbad4d938c83b93db17887918daf63c13eafed257c4f61b07b4" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.2.2" }, "pyparsing": { @@ -400,7 +402,6 @@ "sha256:a5258b84f76661d558492fa87e42db634df143685a0e51802d59cae7daad8732", "sha256:dc5c0541e7cc2c6033dc0338133436abfac53655624784736e9bc8bd35e56583" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.9.0" }, "atomicwrites": { @@ -503,7 +504,7 @@ "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7" ], - "markers": "python_version >= '2.7' and python_version >= '3.6' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '3.6'", "version": "==7.0" }, "click-default-group": { @@ -597,7 +598,7 @@ "sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79", "sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1" ], - "markers": "python_version >= '2.6' and python_version >= '2.7' and python_version < '2.8' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version >= '2.7' and python_version < '2.8' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '2.8' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.6" }, "execnet": { @@ -624,6 +625,12 @@ "markers": "python_version < '3.0' and python_version < '3.3' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.0.2" }, + "future": { + "hashes": [ + "sha256:e39ced1ab767b5936646cedba8bcce582398233d6a627067d4c6a454c90cfedb" + ], + "version": "==0.16.0" + }, "functools32": { "file": "https://github.com/sarugaku/functools32/releases/download/3.2.3-2/functools32-3.2.3.post2-py2.py3-none-any.whl", "markers": "python_version >= '2.6' and python_version >= '2.7' and python_version < '2.8' and python_version not in '3.0, 3.1, 3.2, 3.3' or python_version >= '2.7' and python_version < '2.8' or python_version >= '2.7' and python_version >= '2.7' and python_version < '2.8' and python_version not in '3.0, 3.1, 3.2, 3.3'" @@ -648,7 +655,7 @@ "hashes": [ "sha256:b6ee7066fea66e35f8d0acee24d98006de1a0a8a94a8ce6efe73a9a23c8d9826" ], - "markers": "python_version < '2.7' and python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version < '2.7' and python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.0.4" }, "importlib-metadata": { @@ -783,7 +790,6 @@ "extras": [ "tests" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "path": "." }, "pathlib2": { @@ -826,7 +832,7 @@ "sha256:c0e3553c1e581d8423daccbd825789c6e7f29b7d9e00e5331b12e1642a1a26d3", "sha256:dde5d525cf5f0cbad4d938c83b93db17887918daf63c13eafed257c4f61b07b4" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.2.2" }, "pluggy": { diff --git a/setup.cfg b/setup.cfg index 9abce3c..04bbb40 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,8 +37,9 @@ python_requires = >=2.7,!=3.0,!=3.1,!=3.2,!=3.3 setup_requires = setuptools>=36.2.2 install_requires = appdirs + backports.functools_lru_cache; python_version <= "3.4" cached-property - distlib>=0.2.8 + distlib installer packagebuilder packaging @@ -55,11 +56,14 @@ install_requires = pack = invoke parver +virtualenv = + mork tests = pytest-xdist pytest-timeout pytest-cov pytest-mock + virtualenv pytest [options.entry_points] @@ -104,8 +108,6 @@ norecursedirs = .* build dist news tasks docs filterwarnings = ignore::DeprecationWarning ignore::PendingDeprecationWarning -markers = - needs_internet: test cases that need access to the internet [build-system] requires = ["setuptools", "wheel"] diff --git a/src/passa/internals/markers.py b/src/passa/internals/markers.py index 505b6f0..6d8839c 100644 --- a/src/passa/internals/markers.py +++ b/src/passa/internals/markers.py @@ -3,27 +3,56 @@ from __future__ import absolute_import, unicode_literals from packaging.markers import Marker +from .specifiers import PySpecs, gen_marker +import six +import distlib.markers + +six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) +from six.moves import Mapping, reduce + +try: + from functools import lru_cache +except ImportError: + from backports.functools_lru_cache import lru_cache + + +def _ensure_marker(marker): + if not isinstance(marker, Marker): + return Marker(str(marker)) + return marker def _strip_extra(elements): - """Remove the "extra == ..." operands from the list. + """Remove the "extra == ..." operands from the list.""" + + return _strip_marker_elem("extra", elements) + + +def _strip_pyversion(elements): + return _strip_marker_elem("python_version", elements) + + +def _strip_marker_elem(elem_name, elements): + """Remove the supplied element from the marker. This is not a comprehensive implementation, but relies on an important - characteristic of metadata generation: The "extra == ..." operand is always + characteristic of metadata generation: The element's operand is always associated with an "and" operator. This means that we can simply remove the operand and the "and" operator associated with it. """ + extra_indexes = [] + preceding_operators = ["and"] if elem_name == "extra" else ["and", "or"] for i, element in enumerate(elements): if isinstance(element, list): - cancelled = _strip_extra(element) + cancelled = _strip_marker_elem(elem_name, element) if cancelled: extra_indexes.append(i) - elif isinstance(element, tuple) and element[0].value == "extra": + elif isinstance(element, tuple) and element[0].value == elem_name: extra_indexes.append(i) for i in reversed(extra_indexes): del elements[i] - if i > 0 and elements[i - 1] == "and": + if i > 0 and elements[i - 1] in preceding_operators: # Remove the "and" before it. del elements[i - 1] elif elements: @@ -34,6 +63,19 @@ def _strip_extra(elements): return (not elements) +def _get_stripped_marker(marker, strip_func): + """Build a new marker which is cleaned according to `strip_func`""" + + if not marker: + return None + marker = _ensure_marker(marker) + elements = marker._markers + strip_func(elements) + if elements: + return marker + return None + + def get_without_extra(marker): """Build a new marker without the `extra == ...` part. @@ -43,18 +85,18 @@ def get_without_extra(marker): This could return `None` if the `extra == ...` part is the only one in the input marker. """ - # TODO: Why is this very deep in the internals? Why is a better solution - # implementing it yourself when someone is already maintaining a codebase - # for this? It's literally a grammar implementation that is required to - # meet the demands of a pep... -d - if not marker: - return None - marker = Marker(str(marker)) - elements = marker._markers - _strip_extra(elements) - if elements: - return marker - return None + + return _get_stripped_marker(marker, _strip_extra) + + +def get_without_pyversion(marker): + """Built a new marker without the `python_version` part. + + This could return `None` if the `python_version` section is the only section in the + marker. + """ + + return _get_stripped_marker(marker, _strip_pyversion) def _markers_collect_extras(markers, collection): @@ -70,6 +112,21 @@ def _markers_collect_extras(markers, collection): _markers_collect_extras(el, collection) +def _markers_collect_pyversions(markers, collection): + local_collection = [] + marker_format_str = "{0}" + for i, el in enumerate(reversed(markers)): + if isinstance(el, tuple) and el[0].value == "python_version": + new_marker = str(gen_marker(el)) + local_collection.append(marker_format_str.format(new_marker)) + elif isinstance(el, list): + _markers_collect_pyversions(el, local_collection) + if local_collection: + # local_collection = "{0}".format(" ".join(local_collection)) + collection.extend(local_collection) + + +@lru_cache(maxsize=128) def get_contained_extras(marker): """Collect "extra == ..." operands from a marker. @@ -77,27 +134,118 @@ def get_contained_extras(marker): """ if not marker: return set() - marker = Marker(str(marker)) extras = set() + marker = _ensure_marker(marker) _markers_collect_extras(marker._markers, extras) return extras +def get_contained_pyversions(marker): + """Collect all `python_version` operands from a marker. + + Returns a set of :class:`~passa.internals.specifiers.PySpecs` instances. + """ + + collection = [] + if not marker: + return set() + marker = _ensure_marker(marker) + # Collect the (Variable, Op, Value) tuples and string joiners from the marker + _markers_collect_pyversions(marker._markers, collection) + marker_str = " ".join(collection) + if not marker_str: + return set() + # Use the distlib dictionary parser to create a dictionary 'trie' which is a bit + # easier to reason about + marker_dict = distlib.markers.parse_marker(marker_str)[0] + version_set = set() + pyversions = parse_marker_dict(marker_dict) + if isinstance(pyversions, set): + version_set.update(pyversions) + else: + version_set.add(pyversions) + # Each distinct element in the set was separated by an "and" operator in the marker + # So we will need to reduce them with an intersection here rather than a union + # in order to find the boundaries + versions = reduce(lambda x, y: x & y, version_set) + if not versions: + return PySpecs() + return versions + + def _markers_contains_extra(markers): # Optimization: the marker element is usually appended at the end. + return _markers_contains_key(markers, "extra") + + +def _markers_contains_pyversion(markers): + return _markers_contains_key(markers, "python_version") + + +def _markers_contains_key(markers, key): for element in reversed(markers): - if isinstance(element, tuple) and element[0].value == "extra": + if isinstance(element, tuple) and element[0].value == key: return True elif isinstance(element, list): - if _markers_contains_extra(element): + if _markers_contains_key(element, key): return True return False +@lru_cache(maxsize=128) def contains_extra(marker): """Check whehter a marker contains an "extra == ..." operand. """ if not marker: return False - marker = Marker(str(marker)) + marker = _ensure_marker(marker) return _markers_contains_extra(marker._markers) + + +@lru_cache(maxsize=128) +def contains_pyversion(marker): + """Check whether a marker contains a python_version operand. + """ + + if not marker: + return False + marker = _ensure_marker(marker) + return _markers_contains_pyversion(marker._markers) + + +def parse_marker_dict(marker_dict): + op = marker_dict["op"] + lhs = marker_dict["lhs"] + rhs = marker_dict["rhs"] + # This is where the spec sets for each side land if we have an "or" operator + sides = set() + # And if we hit the end of the parse tree we use this format string to make a marker + format_string = "{lhs} {op} {rhs}" + # Essentially we will iterate over each side of the parsed marker if either one is + # A mapping instance (i.e. a dictionary) and recursively parse and reduce the specset + # Union the "and" specs, intersect the "or"s to find the most appropriate range + if any(issubclass(type(side), Mapping) for side in (lhs, rhs)): + for side in (lhs, rhs): + specs = PySpecs() + if issubclass(type(side), Mapping): + specs.add(parse_marker_dict(side)) + else: + # This is the easiest way to go from a string to a PySpec instance + specs.add(PySpecs.from_marker(Marker(side))) + sides.add(specs) + if op == "and": + # When we are "and"-ing things together, it probably makes the most sense + # to reduce them here into a single PySpec instance + if not sides: + sides = [lhs, rhs] + sides = reduce(lambda x, y: x | y, sides) + if not sides: + return sides + return PySpecs.from_marker(Marker(str(sides))) + # Actually when we "or" things as well we can also just turn them into a reduced + # set using this logic now + return reduce(lambda x, y: x & y, sides) + else: + # At the tip of the tree we are dealing with strings all around and they just need + # to be smashed together + return PySpecs.from_marker(Marker(format_string.format(**marker_dict))) diff --git a/src/passa/internals/specifiers.py b/src/passa/internals/specifiers.py index 75afb6a..420df90 100644 --- a/src/passa/internals/specifiers.py +++ b/src/passa/internals/specifiers.py @@ -5,14 +5,30 @@ import itertools import operator -from packaging.specifiers import SpecifierSet, Specifier +import six + +from packaging.markers import Marker +from packaging.specifiers import Specifier, SpecifierSet +import packaging.version + from vistir.misc import dedup +six.add_move(six.MovedAttribute("Set", "collections", "collections.abc")) +from six.moves import Set + + +try: + from functools import lru_cache +except ImportError: + from backports.functools_lru_cache import lru_cache + +@lru_cache(maxsize=128) def _tuplize_version(version): - return tuple(int(x) for x in version.split(".")) + return tuple(int(x) for x in filter(lambda i: i != "*", version.split("."))) +@lru_cache(maxsize=128) def _format_version(version): return ".".join(str(i) for i in version) @@ -21,52 +37,73 @@ def _format_version(version): REPLACE_RANGES = {">": ">=", "<=": "<"} +@lru_cache(maxsize=128) def _format_pyspec(specifier): if isinstance(specifier, str): if not any(op in specifier for op in Specifier._operators.keys()): specifier = "=={0}".format(specifier) specifier = Specifier(specifier) - if specifier.operator == "==" and specifier.version.endswith(".*"): - specifier = Specifier("=={0}".format(specifier.version[:-2])) + version = specifier.version.replace(".*", "") + if ".*" in specifier.version: + specifier = Specifier("{0}{1}".format(specifier.operator, version)) try: op = REPLACE_RANGES[specifier.operator] except KeyError: return specifier - version = specifier.version.replace(".*", "") curr_tuple = _tuplize_version(version) try: next_tuple = (curr_tuple[0], curr_tuple[1] + 1) except IndexError: next_tuple = (curr_tuple[0], 1) + if not next_tuple[1] <= PySpecs.MAX_VERSIONS[next_tuple[0]]: + if (specifier.operator == "<" + and next_tuple[1] - 1 <= PySpecs.MAX_VERSIONS[next_tuple[0]]): + op = "<=" + next_tuple = (next_tuple[0], next_tuple[1] - 1) + else: + return specifier specifier = Specifier("{0}{1}".format(op, _format_version(next_tuple))) return specifier +@lru_cache(maxsize=128) def _get_specs(specset): + if specset is None: + return if isinstance(specset, Specifier): specset = str(specset) if isinstance(specset, str): specset = SpecifierSet(specset.replace(".*", "")) - return [ - (spec._spec[0], _tuplize_version(spec._spec[1])) - for spec in getattr(specset, "_specs", []) - ] + result = [] + for spec in set(specset): + version = spec.version + op = spec.operator + if op in ("in", "not in"): + versions = version.split(",") + op = "==" if op == "in" else "!=" + for ver in versions: + result.append((op, _tuplize_version(ver.strip()))) + else: + result.append((spec.operator, _tuplize_version(spec.version))) + return result +@lru_cache(maxsize=128) def _group_by_op(specs): specs = [_get_specs(x) for x in list(specs)] flattened = [(op, version) for spec in specs for op, version in spec] - specs = sorted(flattened, key=operator.itemgetter(1)) + specs = sorted(flattened) grouping = itertools.groupby(specs, key=operator.itemgetter(0)) return grouping +@lru_cache(maxsize=128) def cleanup_pyspecs(specs, joiner="or"): specs = {_format_pyspec(spec) for spec in specs} # for != operator we want to group by version # if all are consecutive, join as a list results = set() - for op, versions in _group_by_op(specs): + for op, versions in _group_by_op(tuple(specs)): versions = [version[1] for version in versions] versions = sorted(dedup(versions)) # if we are doing an or operation, we need to use the min for >= @@ -113,24 +150,360 @@ def cleanup_pyspecs(specs, joiner="or"): return results -def pyspec_from_markers(marker): - if marker._markers[0][0] != 'python_version': +def fix_version_tuple(version_tuple): + op, version = version_tuple + max_major = max(PySpecs.MAX_VERSIONS.keys()) + if version[0] > max_major: + return (op, (max_major, PySpecs.MAX_VERSIONS[max_major])) + max_allowed = PySpecs.MAX_VERSIONS[version[0]] + if op == "<" and version[1] > max_allowed and version[1] - 1 <= max_allowed: + op = "<=" + version = (version[0], version[1] - 1) + return (op, version) + + +@lru_cache(maxsize=128) +def get_versions(specset, group_by_operator=True): + specs = [_get_specs(x) for x in list(tuple(specset))] + initial_sort_key = lambda k: (k[0], k[1]) # noqa + initial_grouping_key = operator.itemgetter(0) + if not group_by_operator: + initial_grouping_key = operator.itemgetter(1) + initial_sort_key = operator.itemgetter(1) + version_tuples = sorted( + set((op, version) for spec in specs for op, version in spec), + key=initial_sort_key + ) + version_tuples = [fix_version_tuple(t) for t in version_tuples] + op_groups = [ + (grp, list(map(operator.itemgetter(1), keys))) + for grp, keys in itertools.groupby(version_tuples, key=initial_grouping_key) + ] + versions = [ + (op, packaging.version.parse(".".join(str(v) for v in val))) + for op, vals in op_groups for val in vals + ] + return versions + + +def gen_marker(mkr): + m = Marker("python_version == '1'") + m._markers.pop() + m._markers.append(mkr) + return m + + +class PySpecs(Set): + + MAX_VERSIONS = { + 2: 7, + 3: 9 + } + + def __init__(self, specs=None): + if not specs: + specs = SpecifierSet() + self.specifierset = specs + self.previous_specifierset = None + self.cleaned_tuples = set() + self.markers = None + self.clean() + + def __key(self): + return tuple(sorted(self.specifierset, key=operator.attrgetter("_spec"))) + + def __contains__(self, other): + # The current specifierset fully has every value in the supplied specifierset + if not other.as_set - self.as_set: + return True + return False + + def __eq__(self, other): + return self.__key() == other.__key() + + def __hash__(self): + return hash(self.__key()) + + def __repr__(self): + return u"PySpecs({0!r})".format(str(self.specifierset)) + + def __len__(self): + return len(self.cleaned_tuples) + + def __iter__(self): + for version in self.as_string_set: + yield version return - op = marker._markers[0][1].value - version = marker._markers[0][2].value - specset = set() - if op == "in": - specset.update( - Specifier("=={0}".format(v.strip())) - for v in version.split(",") - ) - elif op == "not in": - specset.update( - Specifier("!={0}".format(v.strip())) - for v in version.split(",") + + def clean(self): + if len(set(self.specifierset)) == 1: + spec = next(iter(spec for spec in self.specifierset), None) + if spec: + self.cleaned_tuples.add((spec.operator, spec.version)) + else: + self.cleaned_tuples = cleanup_pyspecs(self.specifierset, joiner="and") + self.specifierset = self.as_specset + + def add(self, other): + if not isinstance(other, self.__class__): + if isinstance(other, SpecifierSet): + other = PySpecs(other) + else: + raise TypeError("Cannot add type {0!r} to PySpecs".format(type(other))) + new_specifierset = SpecifierSet() + new_specifierset &= self.as_specset + try: + new_specifierset &= other.as_specset + except AttributeError: + pass + new_pyspec = PySpecs(new_specifierset) + self.specifierset = new_pyspec.specifierset + self.cleaned_tuples = new_pyspec.cleaned_tuples + + @property + @lru_cache(maxsize=128) + def as_specset(self): + specs = set() + for spec in self.cleaned_tuples: + op, value = spec + if op in ('in', 'not in'): + new_op = '!=' if op == 'not in' else '==' + for val in value.split(","): + specs.add(Specifier("{0}{1}".format(new_op, val))) + else: + specs.add(Specifier("{0}{1}".format(op, value))) + specifierset = SpecifierSet() + specifierset._specs = frozenset(specs) + return specifierset + + @property + @lru_cache(maxsize=128) + def as_set(self): + return set(self.specifierset) + + @property + @lru_cache(maxsize=128) + def as_string_set(self): + returnval = set() + if len(self.cleaned_tuples) == 1: + val = next(iter(spec for spec in self.cleaned_tuples), None) + if val: + returnval.add("python_version {0[0]} '{0[1]}'".format(val)) + return returnval + return set( + "python_version {0[0]} '{0[1]}'".format(s) + for s in sorted(self.cleaned_tuples) ) - else: - specset.add(Specifier("".join([op, version]))) - if specset: - return specset - return None + + @property + @lru_cache(maxsize=128) + def marker_set(self): + markerset = {Marker(spec) for spec in self.as_string_set} + return markerset + + @property + @lru_cache(maxsize=128) + def marker_string(self): + marker_string = " and ".join(sorted(str(m) for m in self.marker_set)) + if not marker_string: + return "" + return str(Marker(marker_string)) + + @property + @lru_cache(maxsize=128) + def as_markers(self): + if not self.marker_string: + return "" + marker = Marker(self.marker_string) + return marker + + @lru_cache(maxsize=128) + def __str__(self): + string_repr = "{0}".format(str(self.marker_string)) + return string_repr + + def __bool__(self): + return bool(self.specifierset) + + def __nonzero__(self): # Python 2. + return self.__bool__() + + def get_versions(self, group_by_operator=True): + return get_versions(self.specifierset, group_by_operator=group_by_operator) + + def get_versions_in_specset(self): + return set([v[1] for v in self.get_versions() if v[1] in self.specifierset]) + + def get_version_excludes(self): + return set([ + v[1] for v in self.get_versions() + if v[0] == "!=" and v[1] not in self.specifierset + ]) + + def get_version_includes(self): + return set([v for v in ALL_PYTHON_VERSIONS if v in self.specifierset]) + + def get_specset_from_versions(self, versions, include=True): + _specset = SpecifierSet() + op = "==" if include else "!=" + specs = set([Specifier("{0}{1}".format(op, v)) for v in versions]) + _specset._specs = frozenset(specs) + return _specset + + def group_specs(self, specs=None, handle_exclusions=True): + if not specs: + specs = self.get_versions(group_by_operator=handle_exclusions) + else: + specs = get_versions(specs, group_by_operator=handle_exclusions) + pyversions = enumerate(specs) + + def get_version(v): + return ALL_PYTHON_VERSIONS.index(v[1][1]) + + excludes = set() + ranges = set() + + # group the versions on their index from ALL_PYTHON_VERSIONS - their index here + # consecutive elements will share a group, e.g. + # ALL_PYTHON_VERSIONS.index(parse_version("2.7")) == 7, 3.0 == 8, 3.1 == 9 + # if 2.7 is element 1, (7 - 1) = 6, if 3.0 is element 2, (8 - 2) = 6 + # and they will share a group (i.e. they are consecutive) + for k, grp in itertools.groupby(pyversions, lambda t: get_version(t) - t[0]): + version_group = list(grp) + op = next(iter(v[1][0] for v in version_group), None) + _versions = [v[1][1] for v in version_group] + if op == "!=": + excludes.update(set(_versions)) + else: + min_ = min(_versions) + max_ = max(_versions) + if len(_versions) == 1 or str(min_) == str(max_): + ranges.add((min_,)) + else: + ranges.add((min_, max_)) + return ranges, excludes + + def create_specset_from_ranges(self, specset=None, ranges=None, excludes=None): + """This method takes a specifier set and simplifies it down to some range sets. + + The goal is to consume a list of matching individual version specifiers in + "==" notation (accompanied by a set of excluded versions, that is a set() of + Version objects) and produce a SpecifierSet with a min and max range and the + appropriate excludes (i.e. the simplified set). + + :param specset: A specifierset with the enumerated versions + :param ranges: The ranges to use as inputs (or the specset will be generated from it) + :param excludes: A set of Version objects to exclude in the specifierset + :return: A specifierset with the desired ranges + """ + + group_args = {"handle_exclusions": False} + if ranges and not specset and isinstance(ranges, SpecifierSet): + group_args["specs"] = ranges + ranges, _ = self.group_specs(**group_args) + if specset: + group_args["specs"] = specset + if not ranges: + ranges, _ = self.group_specs(**group_args) + if not excludes: + group_args["handle_exclusions"] = True + _, excludes = self.group_specs(**group_args) + spec_ranges = set() + if len(ranges) == 1 and not isinstance(next(iter(ranges)), tuple): + spec_ranges.add(Specifier("=={0}".format(str(next(iter(ranges[0])))))) + else: + min_version = min([r[0] for r in ranges]) + rhs_versions = [ + r[1] for r in ranges if isinstance(r, tuple) and len(r) > 1 + ] + max_version = max(rhs_versions) if rhs_versions else None + spec_ranges.add(Specifier(">={0}".format(str(min_version)))) + if max_version and max_version != ALL_PYTHON_VERSIONS[-1]: + spec_ranges.add(Specifier("<={0}".format(str(max_version)))) + for exclude in excludes: + spec_ranges.add(Specifier("!={0}".format(str(exclude)))) + new_specset = SpecifierSet() + new_specset._specs = frozenset(spec_ranges) + return new_specset + + @lru_cache(maxsize=128) + def __and__(self, other): + # Unintuitive perhaps, but this is for "x or y" and needs to handle the + # widest possible range encapsulated by the two using the intersection + if not isinstance(other, PySpecs): + other = PySpecs(other) + if self == other: + return self + new_specset = SpecifierSet() + # In order to do an "or" propertly we need to intersect the "good" versions + intersection = self.get_version_includes() | other.get_version_includes() + intersection_specset = self.get_specset_from_versions(intersection) + # And then we need to union the "bad" versions + excludes = self.get_version_excludes() & other.get_version_excludes() + new_specset = self.create_specset_from_ranges(ranges=intersection_specset, excludes=excludes) + return PySpecs(new_specset) + + @lru_cache(maxsize=128) + def __or__(self, specset): + if not isinstance(specset, PySpecs): + specset = PySpecs(specset) + if str(self) == str(specset): + return self + combined_set = self.specifierset & specset.specifierset + # new_specset._specs = frozenset(combined_set) + new_pyspec = PySpecs(combined_set) + return new_pyspec + + @classmethod + def from_marker(cls, marker): + if not marker: + return PySpecs() + if len(marker._markers) > 1: + specs = PySpecs() + markers = sorted([ + el for el in marker._markers + if isinstance(el, tuple) + ], key=lambda x: x[2].value) + for mkr in markers: + specs.add(cls.from_marker(gen_marker(mkr))) + return specs + if marker._markers[0][0].value != 'python_version': + return + op = marker._markers[0][1].value + version = marker._markers[0][2].value + specset = set() + if op == "in": + specset.update( + Specifier("=={0}".format(v.strip())) + for v in version.split(",") + ) + elif op == "not in": + versions = [v.strip() for v in version.split(",")] + bad_versions = ["3.0", "3.1", "3.2", "3.3"] + if len(versions) >= 2 and any(v in versions for v in bad_versions): + versions = bad_versions + specset.update( + Specifier("!={0}".format(v.strip())) + for v in sorted(bad_versions) + ) + else: + specset.add(Specifier("".join([op, version]))) + if specset: + specifierset = SpecifierSet() + specifierset._specs = frozenset(specset) + newset = cls(specifierset) + return newset + return None + + +def get_all_python_versions(): + major_versions = list(PySpecs.MAX_VERSIONS.keys()) + versions = ( + "{0}.{1}".format(major, minor) for major in major_versions + for minor in range(PySpecs.MAX_VERSIONS[major] + 1) + ) + versions = (packaging.version.parse(v) for v in versions) + return versions + + +ALL_PYTHON_VERSIONS = sorted(get_all_python_versions()) diff --git a/src/passa/models/lockers.py b/src/passa/models/lockers.py index 6fec7bb..27275aa 100644 --- a/src/passa/models/lockers.py +++ b/src/passa/models/lockers.py @@ -71,6 +71,9 @@ def _collect_derived_entries(state, traces, identifiers): extras[name].extend(requirement.extras) except KeyError: extras[name] = list(requirement.extras) + if requirement.editable and requirement.markers: + requirement.markers = set() + requirement.req.req.markers = set() entries[name] = next(iter(requirement.as_pipfile().values())) for name, ext in extras.items(): entries[name]["extras"] = ext diff --git a/src/passa/models/metadata.py b/src/passa/models/metadata.py index 06131ff..4d7641a 100644 --- a/src/passa/models/metadata.py +++ b/src/passa/models/metadata.py @@ -4,14 +4,19 @@ import copy import itertools +import operator import packaging.markers import packaging.specifiers import vistir import vistir.misc -from ..internals.markers import get_without_extra -from ..internals.specifiers import cleanup_pyspecs, pyspec_from_markers +from six.moves import reduce + +from ..internals.markers import ( + get_without_extra, get_without_pyversion, get_contained_pyversions +) +from ..internals.specifiers import PySpecs def dedup_markers(s): @@ -28,36 +33,53 @@ class MetaSet(object): """ def __init__(self): self.markerset = frozenset() - self.pyspecset = packaging.specifiers.SpecifierSet() + self.pyspecset = PySpecs() def __repr__(self): return "MetaSet(markerset={0!r}, pyspecset={1!r})".format( ",".join(sorted(self.markerset)), str(self.pyspecset), ) + def __key(self): + return (tuple(self.markerset), hash(tuple(self.pyspecset))) + + def __hash__(self): + return hash(self.__key()) + + def __eq__(self, other): + return self.__key() == other.__key() + + def __len__(self): + return len(self.markerset) + len(self.pyspecset) + + def __iter__(self): + return itertools.chain(self.markerset, self.pyspecset) + + def __lt__(self, other): + return operator.lt(self.__key(), other.__key()) + + def __le__(self, other): + return operator.le(self.__key(), other.__key()) + + def __ge__(self, other): + return operator.ge(self.__key(), other.__key()) + + def __gt__(self, other): + return operator.gt(self.__key(), other.__key()) + def __str__(self): - pyspecs = set() - markerset = set() - for m in self.markerset: - marker_specs = pyspec_from_markers(packaging.markers.Marker(m)) - if marker_specs: - pyspecs.add(marker_specs) - else: - markerset.add(m) - if pyspecs: - self.pyspecset._specs &= pyspecs - self.markerset = frozenset(markerset) - return " and ".join(dedup_markers(itertools.chain( + self.markerset = frozenset(filter(None, self.markerset)) + return " and ".join([mkr_part for mkr_part in dedup_markers(itertools.chain( # Make sure to always use the same quotes so we can dedup properly. ( "{0}".format(ms) if " or " in ms else ms for ms in (str(m).replace('"', "'") for m in self.markerset) + if ms ), ( - "python_version {0[0]} '{0[1]}'".format(spec) - for spec in cleanup_pyspecs(self.pyspecset) - ), - ))) + "{0}".format(str(self.pyspecset)) if self.pyspecset else "", + ))) if mkr_part + ]) def __bool__(self): return bool(self.markerset or self.pyspecset) @@ -65,28 +87,38 @@ def __bool__(self): def __nonzero__(self): # Python 2. return self.__bool__() - # Identity check for deduplication. - def __eq__(self, other): - if not isinstance(other, MetaSet): - return False - return (self.markerset, self.pyspecset) == (other.markerset, other.pyspecset) - - def __hash__(self): - return hash((self.markerset, self.pyspecset)) - - def __or__(self, pair): + @classmethod + def from_tuple(cls, pair): marker, specset = pair - markerset = set(self.markerset) + pyspecs = PySpecs(specset) + markerset = set() if marker: - marker_specs = pyspec_from_markers(marker) - if not marker_specs: - markerset.add(str(marker)) - else: - specset._specs &= marker_specs + # Returns a PySpec instance or None + marker_pyversions = get_contained_pyversions(marker) + if marker_pyversions: + pyspecs.add(marker_pyversions) + # The remainder of the marker, if there is any + cleaned_marker = get_without_pyversion(marker) + if cleaned_marker: + markerset.add(str(cleaned_marker)) + metaset = cls() + metaset.markerset = frozenset(markerset) + metaset.pyspecset = pyspecs + return metaset + + def __or__(self, other): + if not isinstance(other, type(self)): + other = self.from_tuple(other) metaset = MetaSet() + markerset = set() + specset = PySpecs() + for meta in (self, other): + if meta.markerset: + markerset |= set(meta.markerset) + if meta.pyspecset: + specset = specset | meta.pyspecset metaset.markerset = frozenset(markerset) - # TODO: Implement some logic to clean up dups like '3.0.*' and '3.0'. - metaset.pyspecset &= self.pyspecset & specset + metaset.pyspecset = specset return metaset @@ -102,19 +134,19 @@ def _build_metasets(dependencies, pythons, key, trace, all_metasets): return all_parent_metasets[parent] = parent_metasets - metaset_iters = [] + metasets = set() for parent, parent_metasets in all_parent_metasets.items(): r = dependencies[parent][key] - python = pythons.get(key, "") + python = pythons[key] + markers = None if r.editable else get_without_extra(r.markers) metaset = ( - get_without_extra(r.markers), + markers, packaging.specifiers.SpecifierSet(python), ) - metaset_iters.append( - parent_metaset | metaset - for parent_metaset in parent_metasets - ) - return set(itertools.chain.from_iterable(metaset_iters)) + for parent_metaset in parent_metasets: + child_metaset = parent_metaset | metaset + metasets.add(child_metaset) + return list(metasets) def _calculate_metasets_mapping(dependencies, pythons, traces): @@ -128,7 +160,7 @@ def _calculate_metasets_mapping(dependencies, pythons, traces): metasets = _build_metasets( dependencies, pythons, key, trace, all_metasets, ) - if metasets is None: + if metasets is None or len(metasets) == 0: continue new_metasets[key] = metasets if not new_metasets: @@ -141,15 +173,17 @@ def _calculate_metasets_mapping(dependencies, pythons, traces): def _format_metasets(metasets): - # If there is an unconditional route, this needs to be unconditional. - if not metasets or not all(metasets): - return None + metasets = dedup_markers(metaset for metaset in metasets if metaset) + # If there is an unconditional route, this needs to be unconditional. + if not metasets: + return "" + combined_metaset = str(MetaSet() | reduce(lambda x, y: x | y, metasets)) + if not combined_metaset: + return "" # This extra str(Marker()) call helps simplify the expression. - return str(packaging.markers.Marker(" or ".join( - "{0}".format(s) if " and " in s else s - for s in dedup_markers(str(metaset) for metaset in metasets if metaset) - ))) + metaset_string = str(packaging.markers.Marker(combined_metaset)) + return metaset_string def set_metadata(candidates, traces, dependencies, pythons): diff --git a/src/passa/models/projects.py b/src/passa/models/projects.py index 8b94ee2..86e71e3 100644 --- a/src/passa/models/projects.py +++ b/src/passa/models/projects.py @@ -9,9 +9,8 @@ import attr import packaging.markers import packaging.utils -import plette -import plette.models import six +import plette import tomlkit from .environments import Environment diff --git a/tasks/package.py b/tasks/package.py index bba878b..ff8cd47 100644 --- a/tasks/package.py +++ b/tasks/package.py @@ -5,7 +5,8 @@ import distlib.scripts import distlib.wheel import invoke -import passa.internals._pip +import packagebuilder +import passa.models.caches import plette import requirementslib @@ -74,8 +75,8 @@ def pack(ctx, remove_lib=True): package.pop('editable', None) # Don't install things as editable. package.pop('markers', None) # Always install everything. r = requirementslib.Requirement.from_pipfile(name, package) - wheel = passa.internals._pip.build_wheel( - r.as_ireq(), sources, r.hashes or None, + wheel = packagebuilder._pip.build_wheel( + r.as_ireq(), sources, r.hashes or None, cache_dir=passa.models.caches.CACHE_DIR ) wheel.install(paths, maker, lib_only=True) diff --git a/tox.ini b/tox.ini index 5c0a6dd..4260b31 100644 --- a/tox.ini +++ b/tox.ini @@ -14,7 +14,7 @@ deps = pytest-sugar -e .[tests,virtualenv] commands = coverage run --parallel -m pytest --timeout 300 [] -install_command = python -m pip install --upgrade {opts} {packages} +install_command = python -m pip install --upgrade --upgrade-strategy=eager {opts} {packages} usedevelop = True [testenv:coverage-report]