From 356b4f74891e58fc64c550b6192eb4d2fa86aa16 Mon Sep 17 00:00:00 2001 From: Xinyang Chen Date: Thu, 2 May 2024 11:44:52 -0500 Subject: [PATCH 01/17] try to implement precedence --- regexfactory/chars.py | 18 ++--- regexfactory/pattern.py | 150 ++++++++++++++++++++++----------------- regexfactory/patterns.py | 58 ++++++++------- 3 files changed, 124 insertions(+), 102 deletions(-) diff --git a/regexfactory/chars.py b/regexfactory/chars.py index 8bde3bb..1308116 100644 --- a/regexfactory/chars.py +++ b/regexfactory/chars.py @@ -10,28 +10,28 @@ from .pattern import RegexPattern #: (Dot.) In the default mode, this matches any character except a newline. If the :data:`re.DOTALL` flag has been specified, this matches any character including a newline. -ANY = RegexPattern(r".") +ANY = RegexPattern(r".", _precedence=10) #: (Caret.) Matches the start of the string, and in :data:`re.MULTILINE` mode also matches immediately after each newline. -ANCHOR_START = RegexPattern(r"^", _precedence=2) +ANCHOR_START = RegexPattern(r"^", _precedence=-1) #: Matches the end of the string or just before the newline at the end of the string, and in :data:`re.MULTILINE` mode also matches before a newline. foo matches both :code:`foo` and :code:`foobar`, while the regular expression :code:`foo$` matches only :code:`foo`. More interestingly, searching for :code:`foo.$` in :code:`foo1\nfoo2\n` matches :code:`foo2` normally, but :code:`foo1` in :data:`re.MULTILINE` mode; searching for a single $ in :code:`foo\n` will find two (empty) matches: one just before the newline, and one at the end of the string. -ANCHOR_END = RegexPattern(r"$", _precedence=2) +ANCHOR_END = RegexPattern(r"$", _precedence=-1) #: Matches Unicode whitespace characters (which includes :code:`[ \t\n\r\f\v]`, and also many other characters, for example the non-breaking spaces mandated by typography rules in many languages). If the :data:`re.ASCII` flag is used, only :code:`[ \t\n\r\f\v]` is matched. -WHITESPACE = RegexPattern(r"\s") +WHITESPACE = RegexPattern(r"\s", _precedence=10) #: Matches any character which is not a whitespace character. This is the opposite of \s. If the :data:`re.ASCII` flag is used this becomes the equivalent of :code:`[^ \t\n\r\f\v]`. -NOTWHITESPACE = RegexPattern(r"\S") +NOTWHITESPACE = RegexPattern(r"\S", _precedence=10) #: Matches Unicode word characters; this includes most characters that can be part of a word in any language, as well as numbers and the underscore. If the :data:`re.ASCII` flag is used, only :code:`[a-zA-Z0-9_]` is matched. -WORD = RegexPattern(r"\w") +WORD = RegexPattern(r"\w", _precedence=10) #: Matches any character which is not a word character. This is the opposite of \w. If the :data:`re.ASCII` flag is used this becomes the equivalent of :code:`[^a-zA-Z0-9_]`. If the :data:`re.LOCALE` flag is used, matches characters which are neither alphanumeric in the current locale nor the underscore. -NOTWORD = RegexPattern(r"\W") +NOTWORD = RegexPattern(r"\W", _precedence=10) #: Matches any Unicode decimal digit (that is, any character in Unicode character category [Nd]). This includes :code:`[0-9]`, and also many other digit characters. If the :data:`re.ASCII` flag is used only :code:`[0-9]` is matched. -DIGIT = RegexPattern(r"\d") +DIGIT = RegexPattern(r"\d", _precedence=10) #: Matches any character which is not a decimal digit. This is the opposite of \d. If the :data:`re.ASCII` flag is used this becomes the equivalent of :code:`[^0-9]`. -NOTDIGIT = RegexPattern(r"\D") +NOTDIGIT = RegexPattern(r"\D", _precedence=10) diff --git a/regexfactory/pattern.py b/regexfactory/pattern.py index 7d428b4..c73c0e6 100644 --- a/regexfactory/pattern.py +++ b/regexfactory/pattern.py @@ -20,15 +20,19 @@ def join(*patterns: ValidPatternType) -> "RegexPattern": """Umbrella function for combining :class:`ValidPatternType`'s into a :class:`RegexPattern`.""" - joined = RegexPattern("") - for pattern in patterns: - joined += RegexPattern(pattern) - return joined + from .patterns import Concat # pylint: disable=import-outside-toplevel + + return Concat(*patterns) def escape(string: str) -> "RegexPattern": """Escapes special characters in a string to use them without their special meanings.""" - return RegexPattern(re.escape(string)) + + if len(string) == 0: + return RegexPattern("", _precedence=0) + if len(string) == 1: + return RegexPattern(re.escape(string), _precedence=10) + return RegexPattern(re.escape(string), _precedence=0) class RegexPattern: @@ -40,13 +44,64 @@ class RegexPattern: #: The precedence of the pattern. Higher precedence patterns are evaluated first. # Precedence order here (https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04_08) - precedence: int - def __init__(self, pattern: ValidPatternType, /, _precedence: int = 1) -> None: - self.regex = self.get_regex(pattern) - self.precedence = ( - _precedence if not isinstance(pattern, RegexPattern) else pattern.precedence - ) + # 5 Collation-related bracket symbols [==] [::] [..] + # 4 Escaped characters \ + # 3 Bracket expression [] + # 2 Grouping () + # 1 Single-character-ERE duplication * + ? {m,n} + # 0 Concatenation + # -1 Anchoring ^ $ + # -2 Alternation | + + def __init__(self, regex: str, *, _precedence: int = -10) -> None: + self.regex = regex + if _precedence is not None: + self._precedence = _precedence + + @staticmethod + def from_regex_str(regex: str) -> "RegexPattern": + # if regex is self escaping, then it just matches a literal string + if regex == re.escape(regex): + if len(regex) == 0: + return RegexPattern("", _precedence=0) + if len(regex) == 1: + return RegexPattern(regex, _precedence=10) + return RegexPattern(regex, _precedence=0) + + # TODO: maybe validate this regex? + return RegexPattern(regex, _precedence=-10) + + @staticmethod + def create(obj: ValidPatternType) -> "RegexPattern": + if isinstance(obj, RegexPattern): + return obj + if isinstance(obj, str): + return RegexPattern.from_regex_str(obj) + if isinstance(obj, re.Pattern): + return RegexPattern.from_regex_str(obj.pattern) + raise TypeError(f"Can't get regex from {obj.__class__.__qualname__} object.") + + @staticmethod + def _ensure_precedence( + pattern: ValidPatternType, precedence: int + ) -> "RegexPattern": + from .patterns import Group # pylint: disable=import-outside-toplevel + + p = RegexPattern.create(pattern) + + if p._precedence >= precedence: + return p + + assert precedence <= 2 + return Group(pattern, capturing=False) + + @staticmethod + def _ensure_precedence_fn(precedence: int): + def inner(pattern: ValidPatternType): + return RegexPattern._ensure_precedence(pattern, precedence).regex + + return inner def __repr__(self) -> str: raw_regex = f"{self.regex!r}".replace("\\\\", "\\") @@ -57,49 +112,15 @@ def __str__(self) -> str: def __add__(self, other: ValidPatternType) -> "RegexPattern": """Adds two :class:`ValidPatternType`'s together, into a :class:`RegexPattern`""" - from .patterns import Group # pylint: disable=import-outside-toplevel - - try: - other_pattern = ( - RegexPattern(other) if not isinstance(other, RegexPattern) else other - ) - except TypeError: - return NotImplemented - - if self.precedence > other_pattern.precedence: - return RegexPattern( - self.regex + self.get_regex(Group(other_pattern, capturing=False)) - ) - if self.precedence < other_pattern.precedence: - return RegexPattern( - self.get_regex(Group(self, capturing=False)) + other_pattern.regex - ) - return RegexPattern(self.regex + other_pattern.regex) + return join(self, other) def __radd__(self, other: ValidPatternType) -> "RegexPattern": """Adds two :class:`ValidPatternType`'s together, into a :class:`RegexPattern`""" - from .patterns import Group # pylint: disable=import-outside-toplevel - - try: - other_pattern = ( - RegexPattern(other) if not isinstance(other, RegexPattern) else other - ) - except TypeError: - return NotImplemented - - if self.precedence > other_pattern.precedence: - return RegexPattern( - self.get_regex(Group(other_pattern, capturing=False)) + self.regex - ) - if self.precedence < other_pattern.precedence: - return RegexPattern( - other_pattern.regex + self.get_regex(Group(self, capturing=False)) - ) - return RegexPattern(other_pattern.regex + self.regex) + return join(other, self) def __mul__(self, coefficient: int) -> "RegexPattern": - """Treats :class:`RegexPattern` as a string and multiplies it by an integer.""" - return RegexPattern(self.regex * coefficient) + """matches exactly coefficient counts of self""" + return join(*([self.regex] * coefficient)) def __eq__(self, other: Any) -> bool: """ @@ -107,29 +128,26 @@ def __eq__(self, other: Any) -> bool: Otherwise return false. """ if isinstance(other, (str, re.Pattern, RegexPattern)): - return ( - self.regex == RegexPattern(other).regex - and self.precedence == RegexPattern(other).precedence - ) + return self.regex == self.create(other).regex return super().__eq__(other) def __hash__(self) -> int: """Hashes the regex string.""" return hash(self.regex) - @staticmethod - def get_regex(obj: ValidPatternType, /) -> str: - """ - Extracts the regex content from :class:`RegexPattern` or :class:`re.Pattern` objects - else return the input :class:`str`. - """ - if isinstance(obj, RegexPattern): - return obj.regex - if isinstance(obj, str): - return obj - if isinstance(obj, re.Pattern): - return obj.pattern - raise TypeError(f"Can't get regex from {obj.__class__.__qualname__} object.") + # @staticmethod + # def get_regex(obj: ValidPatternType, /) -> str: + # """ + # Extracts the regex content from :class:`RegexPattern` or :class:`re.Pattern` objects + # else return the input :class:`str`. + # """ + # if isinstance(obj, RegexPattern): + # return obj.regex + # if isinstance(obj, str): + # return obj + # if isinstance(obj, re.Pattern): + # return obj.pattern + # raise TypeError(f"Can't get regex from {obj.__class__.__qualname__} object.") def compile( self, diff --git a/regexfactory/patterns.py b/regexfactory/patterns.py index 69bd993..161c8d1 100644 --- a/regexfactory/patterns.py +++ b/regexfactory/patterns.py @@ -11,6 +11,14 @@ from regexfactory.pattern import RegexPattern, ValidPatternType +class Concat(RegexPattern): + def __init__(self, *patterns: ValidPatternType) -> None: + super().__init__( + "".join(map(self._ensure_precedence_fn(0), patterns)), + _precedence=0, + ) + + class Or(RegexPattern): """ For matching multiple patterns. @@ -28,23 +36,10 @@ class Or(RegexPattern): """ - def __init__( - self, - *patterns: ValidPatternType, - ) -> None: - regex = "|".join( - map( - self.get_regex, - ( - Group( - pattern, - capturing=False, - ) - for pattern in patterns - ), - ) + def __init__(self, *patterns: ValidPatternType) -> None: + super().__init__( + "|".join(map(self._ensure_precedence_fn(-2), patterns)), _precedence=-2 ) - super().__init__((regex)) class Range(RegexPattern): @@ -110,13 +105,13 @@ class Set(RegexPattern): """ - def __init__(self, *patterns: ValidPatternType) -> None: + def __init__(self, *patterns: Range | str) -> None: regex = "" for pattern in patterns: if isinstance(pattern, Range): regex += f"{pattern.start}-{pattern.stop}" else: - regex += self.get_regex(pattern) + regex += pattern super().__init__(f"[{regex}]") @@ -138,13 +133,13 @@ class NotSet(RegexPattern): """ - def __init__(self, *patterns: ValidPatternType) -> None: + def __init__(self, *patterns: Range | str) -> None: regex = "" for pattern in patterns: if isinstance(pattern, Range): regex += f"{pattern.start}-{pattern.stop}" else: - regex += self.get_regex(pattern) + regex += pattern super().__init__(f"[^{regex}]") @@ -198,8 +193,15 @@ def __init__( amount = f"{i}," else: amount = f"{i}" - regex = self.get_regex(pattern) + "{" + amount + "}" + ("" if greedy else "?") - super().__init__(regex) + regex = ( + # TODO: should this be 1? + self._ensure_precedence(pattern, 2).regex + + "{" + + amount + + "}" + + ("" if greedy else "?") + ) + super().__init__(regex, _precedence=1) class Multi(RegexPattern): @@ -217,7 +219,8 @@ def __init__( suffix = "*" if match_zero else "+" if greedy is False: suffix += "?" - regex = self.get_regex(Group(pattern, capturing=False)) + # TODO: should this be 1? + regex = self._ensure_precedence(pattern, 2).regex super().__init__(regex + suffix) @@ -227,17 +230,17 @@ class Optional(RegexPattern): Functions the same as :code:`Amount(pattern, 0, 1)`. """ - def __init__(self, pattern: ValidPatternType, greedy: bool = True) -> None: + def __init__(self, pattern: ValidPatternType, greedy: bool = True): regex = Group(pattern, capturing=False) + "?" + ("" if greedy else "?") - super().__init__(regex) + super().__init__(regex.regex) class Extension(RegexPattern): """Base class for extension pattern classes.""" def __init__(self, prefix: str, pattern: ValidPatternType): - regex = self.get_regex(pattern) - super().__init__(f"(?{prefix}{regex})") + regex = self.create(pattern).regex + super().__init__(f"(?{prefix}{regex})", _precedence=2) class NamedGroup(Extension): @@ -444,6 +447,7 @@ def __init__(self, pattern: ValidPatternType, capturing: bool = True) -> None: RegexPattern.__init__( # pylint: disable=non-parent-init-called self, f"({pattern})", + _precedence=2, ) From e9ecaac50586cfb3a473c809c060154f34b2a353 Mon Sep 17 00:00:00 2001 From: Xinyang Chen Date: Thu, 2 May 2024 12:21:18 -0500 Subject: [PATCH 02/17] set dont work, temporarily use or instead --- regexfactory/patterns.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/regexfactory/patterns.py b/regexfactory/patterns.py index 161c8d1..f640de6 100644 --- a/regexfactory/patterns.py +++ b/regexfactory/patterns.py @@ -105,14 +105,16 @@ class Set(RegexPattern): """ - def __init__(self, *patterns: Range | str) -> None: - regex = "" - for pattern in patterns: - if isinstance(pattern, Range): - regex += f"{pattern.start}-{pattern.stop}" - else: - regex += pattern - super().__init__(f"[{regex}]") + def __init__(self, *patterns: ValidPatternType) -> None: + # TODO + # regex = "" + # for pattern in patterns: + # if isinstance(pattern, Range): + # regex += f"{pattern.start}-{pattern.stop}" + # else: + # regex += pattern + # super().__init__(f"[{regex}]") + super().__init__(Or(*patterns).regex) class NotSet(RegexPattern): @@ -231,8 +233,10 @@ class Optional(RegexPattern): """ def __init__(self, pattern: ValidPatternType, greedy: bool = True): - regex = Group(pattern, capturing=False) + "?" + ("" if greedy else "?") - super().__init__(regex.regex) + regex = ( + self._ensure_precedence(pattern, 2).regex + "?" + ("" if greedy else "?") + ) + super().__init__(regex) class Extension(RegexPattern): From d4f1d6943dfd4beadd97980ab079a8e49a8c88f8 Mon Sep 17 00:00:00 2001 From: Xinyang Chen Date: Thu, 2 May 2024 21:54:56 -0500 Subject: [PATCH 03/17] rewrite sets to have its own types and structure --- .envrc | 1 + regexfactory/__init__.py | 4 +- regexfactory/chars.py | 17 ++--- regexfactory/pattern.py | 11 ++- regexfactory/patterns.py | 107 +-------------------------- regexfactory/sets.py | 151 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 173 insertions(+), 118 deletions(-) create mode 100644 .envrc create mode 100644 regexfactory/sets.py diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..501f70e --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +path_add PYTHONPATH $(pwd)/ diff --git a/regexfactory/__init__.py b/regexfactory/__init__.py index e8bd145..7c1a39d 100644 --- a/regexfactory/__init__.py +++ b/regexfactory/__init__.py @@ -37,12 +37,10 @@ Multi, NamedGroup, NamedReference, - NotSet, NumberedReference, Optional, Or, - Range, - Set, ) +from .sets import NotSet, Range, Set __version__ = "1.0.1" diff --git a/regexfactory/chars.py b/regexfactory/chars.py index 1308116..61e30b3 100644 --- a/regexfactory/chars.py +++ b/regexfactory/chars.py @@ -8,30 +8,31 @@ """ from .pattern import RegexPattern +from .sets import CharClass #: (Dot.) In the default mode, this matches any character except a newline. If the :data:`re.DOTALL` flag has been specified, this matches any character including a newline. ANY = RegexPattern(r".", _precedence=10) #: (Caret.) Matches the start of the string, and in :data:`re.MULTILINE` mode also matches immediately after each newline. -ANCHOR_START = RegexPattern(r"^", _precedence=-1) +ANCHOR_START = RegexPattern(r"^", _precedence=0) #: Matches the end of the string or just before the newline at the end of the string, and in :data:`re.MULTILINE` mode also matches before a newline. foo matches both :code:`foo` and :code:`foobar`, while the regular expression :code:`foo$` matches only :code:`foo`. More interestingly, searching for :code:`foo.$` in :code:`foo1\nfoo2\n` matches :code:`foo2` normally, but :code:`foo1` in :data:`re.MULTILINE` mode; searching for a single $ in :code:`foo\n` will find two (empty) matches: one just before the newline, and one at the end of the string. -ANCHOR_END = RegexPattern(r"$", _precedence=-1) +ANCHOR_END = RegexPattern(r"$", _precedence=0) #: Matches Unicode whitespace characters (which includes :code:`[ \t\n\r\f\v]`, and also many other characters, for example the non-breaking spaces mandated by typography rules in many languages). If the :data:`re.ASCII` flag is used, only :code:`[ \t\n\r\f\v]` is matched. -WHITESPACE = RegexPattern(r"\s", _precedence=10) +WHITESPACE = CharClass(r"\s", _precedence=10) #: Matches any character which is not a whitespace character. This is the opposite of \s. If the :data:`re.ASCII` flag is used this becomes the equivalent of :code:`[^ \t\n\r\f\v]`. -NOTWHITESPACE = RegexPattern(r"\S", _precedence=10) +NOTWHITESPACE = CharClass(r"\S", _precedence=10) #: Matches Unicode word characters; this includes most characters that can be part of a word in any language, as well as numbers and the underscore. If the :data:`re.ASCII` flag is used, only :code:`[a-zA-Z0-9_]` is matched. -WORD = RegexPattern(r"\w", _precedence=10) +WORD = CharClass(r"\w", _precedence=10) #: Matches any character which is not a word character. This is the opposite of \w. If the :data:`re.ASCII` flag is used this becomes the equivalent of :code:`[^a-zA-Z0-9_]`. If the :data:`re.LOCALE` flag is used, matches characters which are neither alphanumeric in the current locale nor the underscore. -NOTWORD = RegexPattern(r"\W", _precedence=10) +NOTWORD = CharClass(r"\W", _precedence=10) #: Matches any Unicode decimal digit (that is, any character in Unicode character category [Nd]). This includes :code:`[0-9]`, and also many other digit characters. If the :data:`re.ASCII` flag is used only :code:`[0-9]` is matched. -DIGIT = RegexPattern(r"\d", _precedence=10) +DIGIT = CharClass(r"\d", _precedence=10) #: Matches any character which is not a decimal digit. This is the opposite of \d. If the :data:`re.ASCII` flag is used this becomes the equivalent of :code:`[^0-9]`. -NOTDIGIT = RegexPattern(r"\D", _precedence=10) +NOTDIGIT = CharClass(r"\D", _precedence=10) diff --git a/regexfactory/pattern.py b/regexfactory/pattern.py index c73c0e6..75b2bf1 100644 --- a/regexfactory/pattern.py +++ b/regexfactory/pattern.py @@ -7,8 +7,8 @@ # pylint: disable=cyclic-import - import re +import sys from typing import Any, Iterator, List, Optional, Tuple, Union #: @@ -55,6 +55,13 @@ class RegexPattern: # -2 Alternation | def __init__(self, regex: str, *, _precedence: int = -10) -> None: + if type(self) is RegexPattern: + # https://stackoverflow.com/questions/19630994/how-to-check-if-a-string-is-a-valid-regex-in-python + try: + re.compile(regex) + except re.error: + raise ValueError(f"invalid regex {regex}") + self.regex = regex if _precedence is not None: self._precedence = _precedence @@ -237,7 +244,7 @@ def search( content: str, /, pos: int = 0, - endpos: int = 0, + endpos: int = sys.maxsize, *, flags: int = 0, ) -> Optional[re.Match]: diff --git a/regexfactory/patterns.py b/regexfactory/patterns.py index f640de6..00e6598 100644 --- a/regexfactory/patterns.py +++ b/regexfactory/patterns.py @@ -42,109 +42,6 @@ def __init__(self, *patterns: ValidPatternType) -> None: ) -class Range(RegexPattern): - """ - For matching characters between two character indices - (using the Unicode numbers of the input characters.) - You can find use :func:`chr` and :func:`ord` - to translate characters their Unicode numbers and back again. - For example, :code:`chr(97)` returns the string :code:`'a'`, - while :code:`chr(8364)` returns the string :code:`'€'` - Thus, matching characters between :code:`'a'` and :code:`'z'` - is really checking whether a characters unicode number - is between :code:`ord('a')` and :code:`ord('z')` - - .. exec_code:: - - from regexfactory import Range, Or - - patt = Or("Bob", Range("a", "z")) - - print(patt.findall("my job is working for Bob")) - - """ - - def __init__(self, start: str, stop: str) -> None: - self.start = start - self.stop = stop - regex = f"[{start}-{stop}]" - super().__init__(regex) - - -class Set(RegexPattern): - """ - For matching a single character from a list of characters. - Keep in mind special characters like :code:`+` and :code:`.` - lose their meanings inside a set/list, - so need to escape them here to use them. - - In practice, :code:`Set("a", ".", "z")` - functions the same as :code:`Or("a", ".", "z")` - The difference being that :class:`Or` accepts :class:`RegexPattern` 's - and :class:`Set` accepts characters only. - Special characters do **NOT** lose their special meaings inside an :class:`Or` though. - The other big difference is performance, - :class:`Or` is a lot slower than :class:`Set`. - - .. exec_code:: - - import time - from regexfactory import Or, Set - - start_set = time.time() - print(patt := Set(*"a.z").compile()) - print("Set took", time.time() - start_set, "seconds to compile") - print("And the resulting match is", patt.match("b")) - - print() - - start_or = time.time() - print(patt := Or(*"a.z").compile()) - print("Or took", time.time() - start_or, "seconds to compile") - print("And the resulting match is", patt.match("b")) - - """ - - def __init__(self, *patterns: ValidPatternType) -> None: - # TODO - # regex = "" - # for pattern in patterns: - # if isinstance(pattern, Range): - # regex += f"{pattern.start}-{pattern.stop}" - # else: - # regex += pattern - # super().__init__(f"[{regex}]") - super().__init__(Or(*patterns).regex) - - -class NotSet(RegexPattern): - """ - For matching a character that is **NOT** in a list of characters. - Keep in mind special characters lose their special meanings inside :class:`NotSet`'s as well. - - .. exec_code:: - - from regexfactory import NotSet, Set - - not_abc = NotSet(*"abc") - - is_abc = Set(*"abc") - - print(not_abc.match("x")) - print(is_abc.match("x")) - - """ - - def __init__(self, *patterns: Range | str) -> None: - regex = "" - for pattern in patterns: - if isinstance(pattern, Range): - regex += f"{pattern.start}-{pattern.stop}" - else: - regex += pattern - super().__init__(f"[^{regex}]") - - class Amount(RegexPattern): """ For matching multiple occurences of a :class:`ValidPatternType`. @@ -223,7 +120,7 @@ def __init__( suffix += "?" # TODO: should this be 1? regex = self._ensure_precedence(pattern, 2).regex - super().__init__(regex + suffix) + super().__init__(regex + suffix, _precedence=1) class Optional(RegexPattern): @@ -236,7 +133,7 @@ def __init__(self, pattern: ValidPatternType, greedy: bool = True): regex = ( self._ensure_precedence(pattern, 2).regex + "?" + ("" if greedy else "?") ) - super().__init__(regex) + super().__init__(regex, _precedence=1) class Extension(RegexPattern): diff --git a/regexfactory/sets.py b/regexfactory/sets.py new file mode 100644 index 0000000..4286b2a --- /dev/null +++ b/regexfactory/sets.py @@ -0,0 +1,151 @@ +import itertools +import re +import typing as t + +from regexfactory.pattern import RegexPattern +from regexfactory.patterns import IfNotAhead + +CharSet = t.Union[str, "Set", "CharClass", "Range"] + +NEVER = IfNotAhead("") + + +def _charset_normalize(c: CharSet) -> list[str]: + if isinstance(c, str): + return [re.escape(x) for x in c] + if isinstance(c, Range): + return [c._regex_inner] + if isinstance(c, Set): + return c.members + if isinstance(c, CharClass): + return [c.regex] + + raise TypeError(f"invalid charset: {c!r}") + + +class CharClass(RegexPattern): + """ + base class for patterns that matches exactly one from a set of characters, + for example DIGIT or WORD + """ + + pass + + +class Range(RegexPattern): + """ + For matching characters between two character indices + (using the Unicode numbers of the input characters.) + You can find use :func:`chr` and :func:`ord` + to translate characters their Unicode numbers and back again. + For example, :code:`chr(97)` returns the string :code:`'a'`, + while :code:`chr(8364)` returns the string :code:`'€'` + Thus, matching characters between :code:`'a'` and :code:`'z'` + is really checking whether a characters unicode number + is between :code:`ord('a')` and :code:`ord('z')` + + .. exec_code:: + + from regexfactory import Range, Or + + patt = Or("Bob", Range("a", "z")) + + print(patt.findall("my job is working for Bob")) + + """ + + def __init__(self, start: str, stop: str) -> None: + if len(start) != 1: + raise ValueError(f"invalid start: {start!r}") + if len(stop) != 1: + raise ValueError(f"invalid stop: {stop!r}") + if ord(stop) < ord(start): + raise ValueError(f"invalid range: {start!r} to {stop!r}") + + self.start = start + self.stop = stop + + self._regex_inner = f"{re.escape(start)}-{re.escape(stop)}" + super().__init__(f"[{self._regex_inner}]") + + +class Set(RegexPattern): + """ + For matching a single character from a list of characters. + Keep in mind special characters like :code:`+` and :code:`.` + lose their meanings inside a set/list, + so need to escape them here to use them. + + In practice, :code:`Set("a", ".", "z")` + functions the same as :code:`Or("a", ".", "z")` + The difference being that :class:`Or` accepts :class:`RegexPattern` 's + and :class:`Set` accepts characters only. + Special characters do **NOT** lose their special meaings inside an :class:`Or` though. + The other big difference is performance, + :class:`Or` is a lot slower than :class:`Set`. + + .. exec_code:: + + import time + from regexfactory import Or, Set + + start_set = time.time() + print(patt := Set(*"a.z").compile()) + print("Set took", time.time() - start_set, "seconds to compile") + print("And the resulting match is", patt.match("b")) + + print() + + start_or = time.time() + print(patt := Or(*"a.z").compile()) + print("Or took", time.time() - start_or, "seconds to compile") + print("And the resulting match is", patt.match("b")) + + """ + + members: list[str] = [] + + def __init__(self, *charsets: CharSet) -> None: + self.members = list( + itertools.chain(*(_charset_normalize(arg) for arg in charsets)) + ) + if len(self.members) == 0: + regex = str(NEVER) + elif len(self.members) == 1: + regex = self.members[0] + else: + regex = "[" + "".join(self.members) + "]" + + super().__init__(regex, _precedence=3) + + +class NotSet(RegexPattern): + """ + For matching a character that is **NOT** in a list of characters. + Keep in mind special characters lose their special meanings inside :class:`NotSet`'s as well. + + .. exec_code:: + + from regexfactory import NotSet, Set + + not_abc = NotSet(*"abc") + + is_abc = Set(*"abc") + + print(not_abc.match("x")) + print(is_abc.match("x")) + + """ + + members: list[str] = [] + + def __init__(self, *charsets: CharSet): + self.members = list( + itertools.chain(*(_charset_normalize(arg) for arg in charsets)) + ) + if len(self.members) == 0: + regex = "" + else: + regex = "[^" + "".join(self.members) + "]" + + super().__init__(regex, _precedence=3) From c4604ee6d466f4835be9f9a17845eef851266165 Mon Sep 17 00:00:00 2001 From: Xinyang Chen Date: Thu, 2 May 2024 22:08:14 -0500 Subject: [PATCH 04/17] validate only ones that comes from string, and use less parenthesis --- regexfactory/pattern.py | 40 +++++++++++++++------------------------- 1 file changed, 15 insertions(+), 25 deletions(-) diff --git a/regexfactory/pattern.py b/regexfactory/pattern.py index 75b2bf1..7e0f739 100644 --- a/regexfactory/pattern.py +++ b/regexfactory/pattern.py @@ -55,16 +55,8 @@ class RegexPattern: # -2 Alternation | def __init__(self, regex: str, *, _precedence: int = -10) -> None: - if type(self) is RegexPattern: - # https://stackoverflow.com/questions/19630994/how-to-check-if-a-string-is-a-valid-regex-in-python - try: - re.compile(regex) - except re.error: - raise ValueError(f"invalid regex {regex}") - self.regex = regex - if _precedence is not None: - self._precedence = _precedence + self._precedence = _precedence @staticmethod def from_regex_str(regex: str) -> "RegexPattern": @@ -76,7 +68,11 @@ def from_regex_str(regex: str) -> "RegexPattern": return RegexPattern(regex, _precedence=10) return RegexPattern(regex, _precedence=0) - # TODO: maybe validate this regex? + # https://stackoverflow.com/questions/19630994/how-to-check-if-a-string-is-a-valid-regex-in-python + try: + re.compile(regex) + except re.error: + raise ValueError(f"invalid regex {regex}") return RegexPattern(regex, _precedence=-10) @staticmethod @@ -127,7 +123,15 @@ def __radd__(self, other: ValidPatternType) -> "RegexPattern": def __mul__(self, coefficient: int) -> "RegexPattern": """matches exactly coefficient counts of self""" - return join(*([self.regex] * coefficient)) + from .patterns import Amount + + return Amount(self, coefficient) + + def __or__(self, other) -> "RegexPattern": + """matches exactly coefficient counts of self""" + from .sets import Set + + return Set(self, other) # type: ignore def __eq__(self, other: Any) -> bool: """ @@ -142,20 +146,6 @@ def __hash__(self) -> int: """Hashes the regex string.""" return hash(self.regex) - # @staticmethod - # def get_regex(obj: ValidPatternType, /) -> str: - # """ - # Extracts the regex content from :class:`RegexPattern` or :class:`re.Pattern` objects - # else return the input :class:`str`. - # """ - # if isinstance(obj, RegexPattern): - # return obj.regex - # if isinstance(obj, str): - # return obj - # if isinstance(obj, re.Pattern): - # return obj.pattern - # raise TypeError(f"Can't get regex from {obj.__class__.__qualname__} object.") - def compile( self, *, From dae897bca697443591e4d0a4887260ff50c3c090 Mon Sep 17 00:00:00 2001 From: Xinyang Chen Date: Thu, 2 May 2024 22:12:57 -0500 Subject: [PATCH 05/17] fix docstring and readme --- README.md | 17 ++++++----------- regexfactory/pattern.py | 10 ++++++++-- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 809c2e6..86f05cf 100644 --- a/README.md +++ b/README.md @@ -71,23 +71,18 @@ Or what if you want to match urls in html content? ```python from regexfactory import * - protocol = Amount(Range("a", "z"), 1, or_more=True) -host = Amount(Set(WORD, DIGIT, '.'), 1, or_more=True) -port = Optional(IfBehind(":") + Multi(DIGIT)) +host = Amount(WORD | DIGIT | ".", 1, or_more=True) +port = Optional(":" + Multi(DIGIT)) path = Multi( - RegexPattern('/') + Multi( - NotSet('/', '#', '?', '&', WHITESPACE), - match_zero=True - ), - match_zero=True + "/" + Multi(NotSet("/", "#", "?", "&", WHITESPACE), match_zero=True), + match_zero=True, ) -patt = protocol + RegexPattern("://") + host + port + path - +patt = protocol + "://" + host + port + path sentence = "This is a cool url, https://github.com/GrandMoff100/RegexFactory/ " -print(patt) +print(patt.regex) print(patt.search(sentence)) ``` diff --git a/regexfactory/pattern.py b/regexfactory/pattern.py index 7e0f739..bde943c 100644 --- a/regexfactory/pattern.py +++ b/regexfactory/pattern.py @@ -127,12 +127,18 @@ def __mul__(self, coefficient: int) -> "RegexPattern": return Amount(self, coefficient) - def __or__(self, other) -> "RegexPattern": - """matches exactly coefficient counts of self""" + def __or__(self, other): + """equivalent to Set""" from .sets import Set return Set(self, other) # type: ignore + def __ror__(self, other): + """equivalent to Set""" + from .sets import Set + + return Set(other, self) # type: ignore + def __eq__(self, other: Any) -> bool: """ Returns whether or not two :class:`ValidPatternType`'s have the same regex. From 4db8935f38097a2164d3b5c8520bc504e750aed5 Mon Sep 17 00:00:00 2001 From: Xinyang Chen Date: Mon, 13 May 2024 04:53:09 -0500 Subject: [PATCH 06/17] made tests based on a reference implentation that always groups. also adds many more tests. fixed some bugs; some cases output is now cleaner --- regexfactory/__init__.py | 15 ++- regexfactory/chars.py | 19 ++-- regexfactory/pattern.py | 199 +++++++++++++++++++++++++++++++-------- regexfactory/patterns.py | 1 + regexfactory/sets.py | 44 +++++++-- tests/conftest.py | 8 +- tests/strategies.py | 102 +++++++++++++++++--- tests/test_amount.py | 146 +++++++++++++--------------- tests/test_join.py | 33 ++++--- tests/test_manual.py | 29 ++++++ tests/test_or.py | 61 ++++++++---- tests/test_range.py | 25 ----- tests/test_reference.py | 25 +++++ tests/test_set.py | 48 ++++++++-- tests/utils.py | 43 +++++++++ 15 files changed, 578 insertions(+), 220 deletions(-) create mode 100644 tests/test_manual.py delete mode 100644 tests/test_range.py create mode 100644 tests/test_reference.py create mode 100644 tests/utils.py diff --git a/regexfactory/__init__.py b/regexfactory/__init__.py index 7c1a39d..1e238a2 100644 --- a/regexfactory/__init__.py +++ b/regexfactory/__init__.py @@ -23,10 +23,21 @@ WHITESPACE, WORD, ) -from .pattern import ESCAPED_CHARACTERS, RegexPattern, ValidPatternType, escape, join +from .pattern import ( + ESCAPED_CHARACTERS, + RegexPattern, + ValidPatternType, + amount, + escape, + join, + multi, + optional, + or_, +) from .patterns import ( Amount, Comment, + Concat, Extension, Group, IfAhead, @@ -41,6 +52,6 @@ Optional, Or, ) -from .sets import NotSet, Range, Set +from .sets import ALWAYS, NEVER, NotSet, Range, Set __version__ = "1.0.1" diff --git a/regexfactory/chars.py b/regexfactory/chars.py index 61e30b3..5fa10b2 100644 --- a/regexfactory/chars.py +++ b/regexfactory/chars.py @@ -12,6 +12,7 @@ #: (Dot.) In the default mode, this matches any character except a newline. If the :data:`re.DOTALL` flag has been specified, this matches any character including a newline. ANY = RegexPattern(r".", _precedence=10) +ANY._desc = "ANY" #: (Caret.) Matches the start of the string, and in :data:`re.MULTILINE` mode also matches immediately after each newline. ANCHOR_START = RegexPattern(r"^", _precedence=0) @@ -20,19 +21,25 @@ ANCHOR_END = RegexPattern(r"$", _precedence=0) #: Matches Unicode whitespace characters (which includes :code:`[ \t\n\r\f\v]`, and also many other characters, for example the non-breaking spaces mandated by typography rules in many languages). If the :data:`re.ASCII` flag is used, only :code:`[ \t\n\r\f\v]` is matched. -WHITESPACE = CharClass(r"\s", _precedence=10) +WHITESPACE = CharClass(r"\s") +WHITESPACE._desc = "WHITESPACE" #: Matches any character which is not a whitespace character. This is the opposite of \s. If the :data:`re.ASCII` flag is used this becomes the equivalent of :code:`[^ \t\n\r\f\v]`. -NOTWHITESPACE = CharClass(r"\S", _precedence=10) +NOTWHITESPACE = CharClass(r"\S") +NOTWHITESPACE._desc = "NOTWHITESPACE" #: Matches Unicode word characters; this includes most characters that can be part of a word in any language, as well as numbers and the underscore. If the :data:`re.ASCII` flag is used, only :code:`[a-zA-Z0-9_]` is matched. -WORD = CharClass(r"\w", _precedence=10) +WORD = CharClass(r"\w") +WORD._desc = "WORD" #: Matches any character which is not a word character. This is the opposite of \w. If the :data:`re.ASCII` flag is used this becomes the equivalent of :code:`[^a-zA-Z0-9_]`. If the :data:`re.LOCALE` flag is used, matches characters which are neither alphanumeric in the current locale nor the underscore. -NOTWORD = CharClass(r"\W", _precedence=10) +NOTWORD = CharClass(r"\W") +NOTWORD._desc = "NOTWORD" #: Matches any Unicode decimal digit (that is, any character in Unicode character category [Nd]). This includes :code:`[0-9]`, and also many other digit characters. If the :data:`re.ASCII` flag is used only :code:`[0-9]` is matched. -DIGIT = CharClass(r"\d", _precedence=10) +DIGIT = CharClass(r"\d") +DIGIT._desc = "DIGIT" #: Matches any character which is not a decimal digit. This is the opposite of \d. If the :data:`re.ASCII` flag is used this becomes the equivalent of :code:`[^0-9]`. -NOTDIGIT = CharClass(r"\D", _precedence=10) +NOTDIGIT = CharClass(r"\D") +NOTDIGIT._desc = "NOTDIGIT" diff --git a/regexfactory/pattern.py b/regexfactory/pattern.py index bde943c..7c3f164 100644 --- a/regexfactory/pattern.py +++ b/regexfactory/pattern.py @@ -17,22 +17,8 @@ #: Special characters that need to be escaped to be used without their special meanings. ESCAPED_CHARACTERS = "()[]{}?*+-|^$\\.&~#" - -def join(*patterns: ValidPatternType) -> "RegexPattern": - """Umbrella function for combining :class:`ValidPatternType`'s into a :class:`RegexPattern`.""" - from .patterns import Concat # pylint: disable=import-outside-toplevel - - return Concat(*patterns) - - -def escape(string: str) -> "RegexPattern": - """Escapes special characters in a string to use them without their special meanings.""" - - if len(string) == 0: - return RegexPattern("", _precedence=0) - if len(string) == 1: - return RegexPattern(re.escape(string), _precedence=10) - return RegexPattern(re.escape(string), _precedence=0) +_enable_debug: bool = False +_enable_desc: bool = False class RegexPattern: @@ -41,39 +27,54 @@ class RegexPattern: """ regex: str + _reference_regex: Optional[str] = None + _desc: Optional[str] = None #: The precedence of the pattern. Higher precedence patterns are evaluated first. - # Precedence order here (https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04_08) + # Precedence modified from here (https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04_08) - # 5 Collation-related bracket symbols [==] [::] [..] - # 4 Escaped characters \ # 3 Bracket expression [] # 2 Grouping () # 1 Single-character-ERE duplication * + ? {m,n} - # 0 Concatenation - # -1 Anchoring ^ $ + # 0 Concatenation, Anchoring ^ $ # -2 Alternation | + _precedence: int def __init__(self, regex: str, *, _precedence: int = -10) -> None: self.regex = regex self._precedence = _precedence + @property + def _get_ref(self): + if self._reference_regex is not None: + ans = self._reference_regex + else: + ans = self.regex + return f"(?:{ans})" + + @property + def _get_desc(self): + if self._desc is not None: + return self._desc + return repr(self.regex) + @staticmethod def from_regex_str(regex: str) -> "RegexPattern": - # if regex is self escaping, then it just matches a literal string + """create a RegexPattern from a regex. raises ValueError if regex is invalid.""" if regex == re.escape(regex): - if len(regex) == 0: - return RegexPattern("", _precedence=0) - if len(regex) == 1: - return RegexPattern(regex, _precedence=10) - return RegexPattern(regex, _precedence=0) + return escape(regex) # https://stackoverflow.com/questions/19630994/how-to-check-if-a-string-is-a-valid-regex-in-python try: re.compile(regex) except re.error: raise ValueError(f"invalid regex {regex}") - return RegexPattern(regex, _precedence=-10) + ans = RegexPattern(regex, _precedence=-10) + if _enable_debug: + ans._reference_regex = regex + if _enable_desc: + ans._desc = f"RegexPattern.from_regex_str({regex!r})" + return ans @staticmethod def create(obj: ValidPatternType) -> "RegexPattern": @@ -108,6 +109,8 @@ def inner(pattern: ValidPatternType): def __repr__(self) -> str: raw_regex = f"{self.regex!r}".replace("\\\\", "\\") + if self._desc is not None: + return f"RegexPattern({self._desc}, {raw_regex}, {self._reference_regex})" return f"" def __str__(self) -> str: @@ -127,17 +130,11 @@ def __mul__(self, coefficient: int) -> "RegexPattern": return Amount(self, coefficient) - def __or__(self, other): - """equivalent to Set""" - from .sets import Set - - return Set(self, other) # type: ignore + def __or__(self, other: ValidPatternType): + return or_(self, other) def __ror__(self, other): - """equivalent to Set""" - from .sets import Set - - return Set(other, self) # type: ignore + return or_(other, self) def __eq__(self, other: Any) -> bool: """ @@ -246,3 +243,131 @@ def search( ) -> Optional[re.Match]: """See :meth:`re.Pattern.search`.""" return self.compile(flags=flags).search(content, pos, endpos) + + +def join(*patterns: ValidPatternType) -> RegexPattern: + """Umbrella function for combining :class:`ValidPatternType`'s into a :class:`RegexPattern`.""" + from .patterns import Concat # pylint: disable=import-outside-toplevel + + ps = [RegexPattern.create(p) for p in patterns] + ans = Concat(*ps) + if _enable_debug: + ans._reference_regex = "".join(x._get_ref for x in ps) + if _enable_desc: + ans._desc = "+".join(x._get_desc for x in ps) + return ans + + +def escape(string: str) -> RegexPattern: + """Escapes special characters in a string to use them without their special meanings.""" + ans = _escape(string) + + if _enable_debug: + ans._reference_regex = re.escape(string) + if _enable_desc: + if re.escape(string) == string: + ans._desc = repr(string) + else: + ans._desc = f"escape({repr(string)})" + return ans + + +def _escape(string: str) -> RegexPattern: + if len(string) == 0: + return RegexPattern("", _precedence=0) + if len(string) == 1: + from .sets import CharLiteral # pylint: disable=import-outside-toplevel + + return CharLiteral(re.escape(string)) + return RegexPattern(re.escape(string), _precedence=0) + + +def or_(*args: ValidPatternType) -> RegexPattern: + """equivalent to Set""" + from .patterns import Or + from .sets import NEVER, Set, _is_charset + + args_rx = [RegexPattern.create(x) for x in args] + + if len(args_rx) == 1: + ans = args_rx[0] + else: + all_cs = all(map(_is_charset, args_rx)) + + if all_cs: + ans = Set(*args_rx) # type: ignore + else: + ans = Or(*args_rx) + + if _enable_debug: + if len(args_rx) > 0: + ans._reference_regex = "|".join(x._get_ref for x in args_rx) + else: + ans._reference_regex = NEVER.regex + if _enable_desc: + desc = ",".join(x._get_desc for x in args_rx) + ans._desc = f"or_({desc})" + return ans + + +def amount( + pattern: ValidPatternType, + i: int, + j: Optional[int] = None, + or_more: bool = False, + greedy: bool = True, +) -> RegexPattern: + + from .patterns import Amount + + pt = RegexPattern.create(pattern) + + ans = _amount(pt, i, j, or_more, greedy) + if _enable_debug: + ans._reference_regex = Amount( + RegexPattern(pt._get_ref, _precedence=2).regex, i, j, or_more, greedy + ).regex + if _enable_desc: + pieces = [pt._get_desc, str(i)] + if j is not None: + pieces.append(str(j)) + if or_more: + pieces.append("or_more=True") + if not greedy: + pieces.append("greedy=False") + ans._desc = f"Amount({','.join(pieces)})" + return ans + + +def _amount( + pattern: RegexPattern, i: int, j: Optional[int], or_more: bool, greedy: bool +) -> RegexPattern: + from .patterns import Amount, Multi, Optional + + if i == 1 and j == 1: + return pattern + + if i == 0 and j == 1: + return Optional(pattern, greedy) + + if i == 0 and j is None and or_more: + return Multi(pattern, True, greedy) + + if i == 1 and j is None and or_more: + return Multi(pattern, False, greedy) + + return Amount(pattern, i, j, or_more, greedy) + + +def multi( + pattern: ValidPatternType, + match_zero: bool = False, + greedy: bool = True, +): + if match_zero: + return amount(pattern, 0, or_more=True, greedy=greedy) + return amount(pattern, 1, or_more=True, greedy=greedy) + + +def optional(pattern: ValidPatternType, greedy: bool = True): + return amount(pattern, 0, 1, greedy=greedy) diff --git a/regexfactory/patterns.py b/regexfactory/patterns.py index 00e6598..7536b37 100644 --- a/regexfactory/patterns.py +++ b/regexfactory/patterns.py @@ -87,6 +87,7 @@ def __init__( greedy: bool = True, ) -> None: if j is not None: + assert not or_more amount = f"{i},{j}" elif or_more: amount = f"{i}," diff --git a/regexfactory/sets.py b/regexfactory/sets.py index 4286b2a..fd7deb5 100644 --- a/regexfactory/sets.py +++ b/regexfactory/sets.py @@ -2,12 +2,20 @@ import re import typing as t +from regexfactory import pattern as p from regexfactory.pattern import RegexPattern from regexfactory.patterns import IfNotAhead -CharSet = t.Union[str, "Set", "CharClass", "Range"] +CharSet = t.Union[str, "Set", "CharLiteral", "CharClass", "Range"] -NEVER = IfNotAhead("") +ALWAYS = RegexPattern.from_regex_str("") +ALWAYS._desc = "ALWAYS" +NEVER = IfNotAhead(ALWAYS) +NEVER._desc = "NEVER" + + +def _is_charset(c: RegexPattern) -> t.TypeGuard[CharSet]: + return isinstance(c, (Set, CharLiteral, CharClass, Range)) def _charset_normalize(c: CharSet) -> list[str]: @@ -17,19 +25,31 @@ def _charset_normalize(c: CharSet) -> list[str]: return [c._regex_inner] if isinstance(c, Set): return c.members + if isinstance(c, CharLiteral): + return [c.regex] if isinstance(c, CharClass): return [c.regex] raise TypeError(f"invalid charset: {c!r}") +class CharLiteral(RegexPattern): + """ + matches a single char literal like "A" + """ + + def __init__(self, c: str, _precedence=10): + super().__init__(c, _precedence=_precedence) + + class CharClass(RegexPattern): """ base class for patterns that matches exactly one from a set of characters, for example DIGIT or WORD """ - pass + def __init__(self, c: str, _precedence=10): + super().__init__(c, _precedence=_precedence) class Range(RegexPattern): @@ -66,7 +86,7 @@ def __init__(self, start: str, stop: str) -> None: self.stop = stop self._regex_inner = f"{re.escape(start)}-{re.escape(stop)}" - super().__init__(f"[{self._regex_inner}]") + super().__init__(f"[{self._regex_inner}]", _precedence=3) class Set(RegexPattern): @@ -110,13 +130,13 @@ def __init__(self, *charsets: CharSet) -> None: itertools.chain(*(_charset_normalize(arg) for arg in charsets)) ) if len(self.members) == 0: - regex = str(NEVER) - elif len(self.members) == 1: - regex = self.members[0] + regex = NEVER.regex + prec = NEVER._precedence else: regex = "[" + "".join(self.members) + "]" + prec = 3 - super().__init__(regex, _precedence=3) + super().__init__(regex, _precedence=prec) class NotSet(RegexPattern): @@ -144,8 +164,12 @@ def __init__(self, *charsets: CharSet): itertools.chain(*(_charset_normalize(arg) for arg in charsets)) ) if len(self.members) == 0: - regex = "" + from .chars import NOTWHITESPACE, WHITESPACE + + regex = (WHITESPACE | NOTWHITESPACE).regex + prec = 3 else: regex = "[^" + "".join(self.members) + "]" + prec = 3 - super().__init__(regex, _precedence=3) + super().__init__(regex, _precedence=prec) diff --git a/tests/conftest.py b/tests/conftest.py index e5c6d22..2e063f4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,8 @@ -from hypothesis import settings +from hypothesis import Verbosity, settings + +from regexfactory import pattern # Set default profile to use 500 examples -settings.register_profile("default", max_examples=500) +settings.register_profile("default", max_examples=500, verbosity=Verbosity.normal) + +pattern._enable_desc = True diff --git a/tests/strategies.py b/tests/strategies.py index 74d52c6..25aa6b5 100644 --- a/tests/strategies.py +++ b/tests/strategies.py @@ -1,22 +1,98 @@ +from hypothesis import event from hypothesis import strategies as st -from regexfactory.pattern import ESCAPED_CHARACTERS +import regexfactory as r +from regexfactory import NotSet, Range, RegexPattern, amount, escape, or_ -# Strategy to generate characters that are not used in escapes -non_escape_char = st.characters(blacklist_characters=list(ESCAPED_CHARACTERS)) +@st.composite +def pat_base(draw: st.DrawFn) -> RegexPattern: + v = draw(st.integers(0, 99)) + v = 99 - v -# Strategy to generate text that avoids escaped characters -non_escaped_text = st.text(min_size=1, alphabet=non_escape_char) + def case(weight: int): + nonlocal v + v -= weight + return v < 0 + if case(10): + charsets = [ + r.ANY, + r.ANCHOR_START, + r.ANCHOR_END, + r.WHITESPACE, + r.NOTWHITESPACE, + r.WORD, + r.NOTWORD, + r.DIGIT, + r.NOTDIGIT, + ] + return draw(st.sampled_from(charsets)) -# Strategy to produce either None or a positive integer -optional_step = st.one_of(st.none(), st.integers(min_value=1)) + if case(10): + chars = draw(st.lists(elements=st.characters(codec="utf-8"))) + return or_(*(escape(x) for x in chars)) + if case(5): + chars = draw(st.lists(elements=st.characters(codec="utf-8"))) + return NotSet(*chars) -def build_bounds(lower_bound, step) -> range: - """ - Function to generate a tuple of (lower, upper) in which lower < upper - """ - upper_bound = lower_bound + step - return range(lower_bound, upper_bound + 1) + if case(5): + x = draw(st.characters(codec="utf-8")) + y = draw(st.characters(codec="utf-8")) + if x > y: + x, y = y, x + return Range(x, y) + + return escape(draw(st.text())) + + +@st.composite +def _pat_extend(draw: st.DrawFn, children: st.SearchStrategy[RegexPattern]): + if not draw(st.booleans()): + return draw(children) + draw(children) + + if not draw(st.booleans()): + return draw(children) | draw(children) + + c = draw(children) + + if draw(st.booleans()): + return amount( + c, draw(st.integers(0, 4)), None, draw(st.booleans()), draw(st.booleans()) + ) + + a = draw(st.integers(0, 2)) + b = draw(st.integers(0, 2)) + return amount(c, a, a + b, False, draw(st.booleans())) + + +pat_generic: st.SearchStrategy[RegexPattern] = st.recursive(pat_base(), _pat_extend) + + +always_case = ["", "a", "ab", "aa", ".", " ", "\t", "\n"] + + +@st.composite +def _gencase_random(draw: st.DrawFn, r1: str, r2: str): + # use_true_random prevent shrinks + x = draw(st.randoms(use_true_random=True)).randint(0, 9) + + # use st.text() most of the time since its much faster + if x < 8: + return draw(st.text()) + + event("generating test case using st.from_regex") + if x < 9: + ans = draw(st.from_regex(r1)) + else: + ans = draw(st.from_regex(r2)) + + event("st.from_regex succeeded") + return ans + + +def gencase(r1: str, r2: str) -> st.SearchStrategy[str]: + """generate a str suitable for checking if r1 and r2 behave the same on that str""" + # allow shrinking towards always or text() + return st.sampled_from(always_case) | st.text() | _gencase_random(r1, r2) diff --git a/tests/test_amount.py b/tests/test_amount.py index 7d96735..5d44555 100644 --- a/tests/test_amount.py +++ b/tests/test_amount.py @@ -1,89 +1,71 @@ -from typing import Optional - -import pytest from hypothesis import given from hypothesis import strategies as st -from strategies import build_bounds, optional_step - -from regexfactory import Amount, ValidPatternType - - -def build_amount( - pattern: ValidPatternType, - start: int, - or_more: bool, - greedy: bool, - step: Optional[int], -): - """ - General Amount builder. Note that the `j` parameter is constructed as - the step plus start when step is defined. When step is None we assume - that no upper bound is present. - """ - stop_value = None - if step is not None: - stop_value = start + step - return Amount( - pattern=pattern, i=start, j=stop_value, or_more=or_more, greedy=greedy +from strategies import pat_generic +from utils import check_one + +from regexfactory import amount, join, multi, optional + + +@given(pat_generic, st.integers(0, 4), st.data()) +def test_operator_mul(x, n, data): + check_one(x * n, join(*(x for _ in range(n))), data) + + +# invariants of amount + + +@given(pat_generic, st.integers(0, 4), st.booleans(), st.data()) +def test_amount_fixed(x, n, greedy, data): + check_one( + amount(x, n, n, greedy=greedy), + x * n, + data, + ) + + +@given(pat_generic, st.integers(0, 4), st.integers(0, 4), st.booleans(), st.data()) +def test_amount_bounded(x, n, m, greedy, data): + check_one( + amount(x, n, n + m, greedy=greedy), + x * n + (optional(x, greedy=greedy)) * m, + data, ) -@pytest.mark.patterns -@given(st.text(min_size=1), st.integers(min_value=1)) -def test_amount_single_count(word, count): - """ - Test to ensure that when `or_more=False` and no upper bound is - provided the regex will be of the form word{count}. - """ - actual = Amount(word, i=count, or_more=False) - assert actual.regex == "{word}{{{count}}}".format(word=word, count=str(count)) - - -@pytest.mark.patterns -@given( - st.text(min_size=1), - st.builds( - build_bounds, - lower_bound=st.integers(min_value=1), - step=st.integers(min_value=1), - ), -) -def test_amount_lower_upper(word, bound: range): - """ - Test to ensure that if a lower and upper bound are provided then the - regex of the resulting `Amount` will be of the form {word}{lower,upper}. - """ - actual = Amount(word, bound.start, bound.stop) - expected = "{word}{{{lower},{upper}}}".format( - word=word, lower=str(bound.start), upper=str(bound.stop) +@given(pat_generic, st.integers(0, 4), st.booleans(), st.data()) +def test_amount_or_more1(x, n, greedy, data): + check_one( + amount(x, n, or_more=True, greedy=greedy), + x * n + amount(x, 0, or_more=True, greedy=greedy), + data, ) - assert actual.regex == expected - - -@pytest.mark.patterns -@given(st.text(min_size=1), st.integers(min_value=1)) -def test_amount_or_more(word, count): - """ - Test to ensure that when `or_more=True` and no upper bound is - provided the regex will be of the form word{count,}. - """ - actual = Amount(word, count, or_more=True) - assert actual.regex == "{word}{{{count},}}".format(word=word, count=str(count)) - - -@pytest.mark.patterns -@given( - st.builds( - build_amount, - pattern=st.text(min_size=1), - start=st.integers(min_value=1), - or_more=st.booleans(), - greedy=st.just(False), - step=optional_step, + + +@given(pat_generic, st.booleans(), st.data()) +def test_amount_or_more2(x, greedy, data): + check_one( + amount(x, 0, or_more=True, greedy=greedy), + optional(x, greedy=greedy) + amount(x, 0, or_more=True, greedy=greedy), + data, + ) + + +# multi is consistent with + and * + + +@given(pat_generic, st.data()) +def test_multi1(x, data): + check_one( + multi(x), + f"(?:{x.regex})+", + data, + ) + + +@given(pat_generic, st.data()) +def test_multi2(x, data): + check_one( + multi(x, match_zero=True), + f"(?:{x.regex})*", + data, ) -) -def test_amount_non_greedy(amt): - """ - Test to ensure that instances of Amount with greedy as False will end with "?" - """ - assert amt.regex.endswith("?") diff --git a/tests/test_join.py b/tests/test_join.py index 727cdf6..31014f1 100644 --- a/tests/test_join.py +++ b/tests/test_join.py @@ -1,18 +1,23 @@ -import pytest -from hypothesis import example, given +from hypothesis import given from hypothesis import strategies as st -from strategies import non_escaped_text +from strategies import pat_generic +from utils import check_one -from regexfactory.pattern import join +from regexfactory import escape, join, pattern +pattern._enable_desc = True -@pytest.mark.pattern -@given(st.lists(elements=non_escaped_text, min_size=1, max_size=10, unique=True)) -@example(words=["0", "1"]) -def test_join(words: list): - """ - Tests to capture that the join function concatenates the expressions and - each word in the list is found in the larger regex. - """ - joined_regex = join(*words) - assert joined_regex.regex == "".join(words) + +@given(pat_generic, pat_generic, st.data()) +def test_operator_add(x, y, data): + check_one(x + y, join(x, y), data) + + +@given(pat_generic, pat_generic, pat_generic, st.data()) +def test_join_assoc(x, y, z, data): + check_one(x + (y + z), (x + y) + z, data) + + +@given(st.text(), st.text(), st.data()) +def test_sum_escape(x, y, data): + check_one(escape(x + y), escape(x) + escape(y), data) diff --git a/tests/test_manual.py b/tests/test_manual.py new file mode 100644 index 0000000..dfeedc0 --- /dev/null +++ b/tests/test_manual.py @@ -0,0 +1,29 @@ +from hypothesis import given +from hypothesis import strategies as st + +from regexfactory import escape, optional + + +def check(a, b: bool): + assert (a is not None) == b + + +@given(st.text(), st.text()) +def test_anchor_start(a: str, b: str): + check(("^" + escape(a)).search(b), b.startswith(a)) + + +@given(st.text(), st.text()) +def test_anchor_end(a: str, b: str): + check((escape(a) + "$").search(b), b.endswith(a)) + + +@given(st.text(), st.text()) +def test_optional(a: str, b: str): + x = optional(escape(a)).search(a + b) + assert x is not None + assert x.group() == a + + x = optional(escape(a), greedy=False).search(a + b) + assert x is not None + assert x.group() == "" diff --git a/tests/test_or.py b/tests/test_or.py index b1b6579..cfa557f 100644 --- a/tests/test_or.py +++ b/tests/test_or.py @@ -1,26 +1,47 @@ -import re - import pytest -from hypothesis import example, given +from hypothesis import given from hypothesis import strategies as st -from strategies import non_escaped_text +from strategies import pat_generic +from utils import check_one, check_regex + +from regexfactory import RegexPattern, amount, escape, join, multi, optional, or_ + + +@given(pat_generic, pat_generic, st.data()) +def test_operator_or(x, y, data): + check_one(x | y, or_(x, y), data) + + +@given(pat_generic, pat_generic, pat_generic, st.data()) +def test_or_assoc(x, y, z, data): + check_one(x | (y | z), (x | y) | z, data) + + +@given(pat_generic, pat_generic, pat_generic, st.data()) +def test_join_or1(x, y, z, data): + check_one( + (x | y) + z, + (x + z) | (y + z), + data, + ) + + +def test_join_or2b(): + # above does not pass the other way. ex: + with pytest.raises(RuntimeError): + x = or_("ax", "a") + y = escape("xb") + z = escape("bc") -from regexfactory import Or + check_regex(x + (y | z), (x + y) | (x + z), "axbc") -@pytest.mark.patterns -@given( - st.lists( - non_escaped_text, - min_size=1, - max_size=10, +@given(pat_generic, pat_generic, pat_generic, st.data()) +def test_join_or2a(x: RegexPattern, y, z, data): + # should work if x is only allowed to match one thing + x = RegexPattern(f"(?>{x.regex})") + check_one( + x + (y | z), + (x + y) | (x + z), + data, ) -) -@example(arr=["0", "0"]) -def test_matching_or(arr: list): - actual = Or(*arr) - if len(arr) == 1: - assert isinstance(actual.match(arr[0]), re.Match) - else: - for value in arr: - assert isinstance(actual.match(value), re.Match) diff --git a/tests/test_range.py b/tests/test_range.py deleted file mode 100644 index b969b61..0000000 --- a/tests/test_range.py +++ /dev/null @@ -1,25 +0,0 @@ -import pytest - -from regexfactory import Range - - -@pytest.mark.patterns -def test_numeric_range(): - start = "0" - end = "9" - assert Range(start, end).regex == "[0-9]" - - -@pytest.mark.patterns -@pytest.mark.parametrize( - "start, stop, expected", - [ - ("0", "9", "[0-9]"), - ("a", "f", "[a-f]"), - ("r", "q", "[r-q]"), - ("A", "Z", "[A-Z]"), - ], -) -def test_range_parameters(start, stop, expected): - actual = Range(start=start, stop=stop) - assert actual.regex == expected diff --git a/tests/test_reference.py b/tests/test_reference.py new file mode 100644 index 0000000..d135862 --- /dev/null +++ b/tests/test_reference.py @@ -0,0 +1,25 @@ +from hypothesis import HealthCheck, assume, given, settings +from hypothesis import strategies as st +from strategies import gencase, pat_generic +from utils import check_regex + +from regexfactory import RegexPattern, pattern + +pattern._enable_debug = True +pattern._enable_desc = True + + +# checks that ._reference_regex behaves the same as .regex +@given(pat_generic, st.data()) +@settings( + max_examples=10000, suppress_health_check=[HealthCheck.too_slow], deadline=None +) +def test_reference(pat: RegexPattern, data: st.DataObject): + assume(pat._reference_regex is not None) + assert pat._reference_regex is not None + assume(pat.regex != pat._reference_regex) + check_regex( + pat.regex, + pat._reference_regex, + data.draw(gencase(pat.regex, pat._reference_regex)), + ) diff --git a/tests/test_set.py b/tests/test_set.py index d400c8e..34f3897 100644 --- a/tests/test_set.py +++ b/tests/test_set.py @@ -1,16 +1,46 @@ import re -import pytest from hypothesis import given from hypothesis import strategies as st -from strategies import non_escape_char +from utils import check_one -from regexfactory import Set +import regexfactory as r +from regexfactory import IfNotAhead, NotSet, Range, Set, or_, pattern +pattern._enable_desc = True -@pytest.mark.patterns -@given(st.lists(elements=non_escape_char, min_size=1)) -def test_set(chars: list): - actual = Set(*chars) - for value in chars: - assert isinstance(actual.match(value), re.Match) + +@given(st.data()) +def test_range(data): + check_one( + Range("a", "e"), + Set("abcde"), + data, + ) + + +@given(st.text(), st.data()) +def test_set(chars, data): + check_one( + Set(*chars), + or_(*(re.escape(x) for x in chars)), + data, + ) + + +@given(st.text(), st.data()) +def test_notset(chars, data): + check_one( + NotSet(*chars), + IfNotAhead(Set(*chars)) + NotSet(), + data, + ) + + +@given(st.data()) +def test_any(data): + check_one( + or_(r.ANY, r.WHITESPACE), + NotSet(), + data, + ) diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000..e564fdb --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,43 @@ +import re + +from hypothesis import strategies as st + +from regexfactory import RegexPattern, ValidPatternType + + +def check_regex(p1: ValidPatternType, p2: ValidPatternType, v: str): + r1 = p1 if isinstance(p1, str) else RegexPattern.create(p1).regex + r2 = p2 if isinstance(p2, str) else RegexPattern.create(p2).regex + + """checks that r1 and r2 does the same on v""" + if r1 == r2: + return + + x = re.compile(r1).search(v) + y = re.compile(r2).search(v) + + try: + assert (x is None) == (y is None) + + if x is not None: + assert y is not None + assert x.start() == y.start() + assert x.end() == y.end() + except Exception as e: + raise RuntimeError( + f"string {v!r} results in:\n" + f"r1: {r1!r}\n" + f"=> {x!r}\n" + f"r2: {r2!r}\n" + f"=> {y!r}\n" + ) from e + + +def check_one(v1: ValidPatternType, v2: ValidPatternType, data: st.DataObject): + from strategies import gencase + + r1 = RegexPattern.create(v1).regex + r2 = RegexPattern.create(v2).regex + if r1 == r2: + return + check_regex(r1, r2, data.draw(gencase(r1, r2))) From c22a6435d4050bcc2610b52c1ebcdefbedaea51b Mon Sep 17 00:00:00 2001 From: Xinyang Chen Date: Tue, 14 May 2024 11:02:29 -0500 Subject: [PATCH 07/17] better way to write this --- regexfactory/sets.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/regexfactory/sets.py b/regexfactory/sets.py index fd7deb5..e7e4f8f 100644 --- a/regexfactory/sets.py +++ b/regexfactory/sets.py @@ -166,8 +166,9 @@ def __init__(self, *charsets: CharSet): if len(self.members) == 0: from .chars import NOTWHITESPACE, WHITESPACE - regex = (WHITESPACE | NOTWHITESPACE).regex - prec = 3 + any_sing = WHITESPACE | NOTWHITESPACE + regex = any_sing.regex + prec = any_sing._precedence else: regex = "[^" + "".join(self.members) + "]" prec = 3 From 5d51627d26508d994a9fe4686b1f5014472f5986 Mon Sep 17 00:00:00 2001 From: Xinyang Chen Date: Tue, 14 May 2024 11:04:02 -0500 Subject: [PATCH 08/17] remove some useless stuff --- tests/test_join.py | 4 +--- tests/test_reference.py | 1 - tests/test_set.py | 4 +--- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/tests/test_join.py b/tests/test_join.py index 31014f1..18e83be 100644 --- a/tests/test_join.py +++ b/tests/test_join.py @@ -3,9 +3,7 @@ from strategies import pat_generic from utils import check_one -from regexfactory import escape, join, pattern - -pattern._enable_desc = True +from regexfactory import escape, join @given(pat_generic, pat_generic, st.data()) diff --git a/tests/test_reference.py b/tests/test_reference.py index d135862..98e0962 100644 --- a/tests/test_reference.py +++ b/tests/test_reference.py @@ -6,7 +6,6 @@ from regexfactory import RegexPattern, pattern pattern._enable_debug = True -pattern._enable_desc = True # checks that ._reference_regex behaves the same as .regex diff --git a/tests/test_set.py b/tests/test_set.py index 34f3897..8de1455 100644 --- a/tests/test_set.py +++ b/tests/test_set.py @@ -5,9 +5,7 @@ from utils import check_one import regexfactory as r -from regexfactory import IfNotAhead, NotSet, Range, Set, or_, pattern - -pattern._enable_desc = True +from regexfactory import IfNotAhead, NotSet, Range, Set, or_ @given(st.data()) From a3b40919dae8b0107e226ce9b64c7dbeff8f074c Mon Sep 17 00:00:00 2001 From: Xinyang Chen Date: Tue, 14 May 2024 12:15:08 -0500 Subject: [PATCH 09/17] less parens --- regexfactory/pattern.py | 6 ++++++ tests/test_escape.py | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 tests/test_escape.py diff --git a/regexfactory/pattern.py b/regexfactory/pattern.py index 7c3f164..0cf4346 100644 --- a/regexfactory/pattern.py +++ b/regexfactory/pattern.py @@ -61,8 +61,14 @@ def _get_desc(self): @staticmethod def from_regex_str(regex: str) -> "RegexPattern": """create a RegexPattern from a regex. raises ValueError if regex is invalid.""" + + # attempt to treat regex as a literal string if regex == re.escape(regex): return escape(regex) + parts = regex.split(r"\\\\") + guess = "\\".join(x.replace("\\", "") for x in parts) + if regex == re.escape(guess): + return escape(guess) # https://stackoverflow.com/questions/19630994/how-to-check-if-a-string-is-a-valid-regex-in-python try: diff --git a/tests/test_escape.py b/tests/test_escape.py new file mode 100644 index 0000000..c38f40d --- /dev/null +++ b/tests/test_escape.py @@ -0,0 +1,18 @@ +import re + +from hypothesis import given +from hypothesis import strategies as st +from strategies import pat_generic +from utils import check_one + +from regexfactory import RegexPattern, escape + + +@given(pat_generic, st.data()) +def test_from_regex_str(x, data): + check_one(x, RegexPattern.from_regex_str(x.regex), data) + + +@given(st.text(), st.data()) +def test_escape_str(x, data): + check_one(escape(x), re.escape(x), data) From 539a6fd9e026b8fbf59de4d22208f1b2540a8687 Mon Sep 17 00:00:00 2001 From: Xinyang Chen Date: Tue, 14 May 2024 12:20:06 -0500 Subject: [PATCH 10/17] better diagnostics on invalid regex --- regexfactory/pattern.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/regexfactory/pattern.py b/regexfactory/pattern.py index 0cf4346..97bf3c9 100644 --- a/regexfactory/pattern.py +++ b/regexfactory/pattern.py @@ -73,8 +73,8 @@ def from_regex_str(regex: str) -> "RegexPattern": # https://stackoverflow.com/questions/19630994/how-to-check-if-a-string-is-a-valid-regex-in-python try: re.compile(regex) - except re.error: - raise ValueError(f"invalid regex {regex}") + except re.error as e: + raise ValueError(f"invalid regex {regex}") from e ans = RegexPattern(regex, _precedence=-10) if _enable_debug: ans._reference_regex = regex From f74df0d59b233e704ef35abdfe43ddeee4007cb6 Mon Sep 17 00:00:00 2001 From: Xinyang Chen Date: Tue, 14 May 2024 14:31:42 -0500 Subject: [PATCH 11/17] change signature of amount + add docstring --- regexfactory/pattern.py | 31 +++++++++++++++++++++++++++---- tests/strategies.py | 7 +++++-- tests/test_amount.py | 11 ++++++++++- 3 files changed, 42 insertions(+), 7 deletions(-) diff --git a/regexfactory/pattern.py b/regexfactory/pattern.py index 97bf3c9..bab3869 100644 --- a/regexfactory/pattern.py +++ b/regexfactory/pattern.py @@ -9,7 +9,7 @@ import re import sys -from typing import Any, Iterator, List, Optional, Tuple, Union +from typing import Any, Iterator, List, Literal, Optional, Tuple, Union, overload #: ValidPatternType = Union[re.Pattern, str, "RegexPattern"] @@ -132,9 +132,8 @@ def __radd__(self, other: ValidPatternType) -> "RegexPattern": def __mul__(self, coefficient: int) -> "RegexPattern": """matches exactly coefficient counts of self""" - from .patterns import Amount - return Amount(self, coefficient) + return amount(self, coefficient) def __or__(self, other: ValidPatternType): return or_(self, other) @@ -289,7 +288,6 @@ def _escape(string: str) -> RegexPattern: def or_(*args: ValidPatternType) -> RegexPattern: - """equivalent to Set""" from .patterns import Or from .sets import NEVER, Set, _is_charset @@ -320,9 +318,24 @@ def amount( pattern: ValidPatternType, i: int, j: Optional[int] = None, + *, or_more: bool = False, greedy: bool = True, ) -> RegexPattern: + """ + (1) amount(pattern, i, j): + matches between i and j, inclusive, copies of pattern + (2) amount(pattern, i): + matches exactly i copies of pattern + (3) amount(pattern, i, or_more = True): + matches at least i copies of pattern + + if greedy is True, match as long as possible; othewise match as short as possible. + backtracking is always enabled, regardless of the setting of greedy. + """ + + if j is not None and or_more: + raise ValueError() from .patterns import Amount @@ -349,6 +362,13 @@ def _amount( pattern: RegexPattern, i: int, j: Optional[int], or_more: bool, greedy: bool ) -> RegexPattern: from .patterns import Amount, Multi, Optional + from .sets import ALWAYS + + if j is None and not or_more: + j = i + + if i == 0 and j == 0: + return ALWAYS if i == 1 and j == 1: return pattern @@ -362,6 +382,9 @@ def _amount( if i == 1 and j is None and or_more: return Multi(pattern, False, greedy) + if i == j: + j = None + return Amount(pattern, i, j, or_more, greedy) diff --git a/tests/strategies.py b/tests/strategies.py index 25aa6b5..35ce969 100644 --- a/tests/strategies.py +++ b/tests/strategies.py @@ -59,12 +59,15 @@ def _pat_extend(draw: st.DrawFn, children: st.SearchStrategy[RegexPattern]): if draw(st.booleans()): return amount( - c, draw(st.integers(0, 4)), None, draw(st.booleans()), draw(st.booleans()) + c, + draw(st.integers(0, 4)), + or_more=draw(st.booleans()), + greedy=draw(st.booleans()), ) a = draw(st.integers(0, 2)) b = draw(st.integers(0, 2)) - return amount(c, a, a + b, False, draw(st.booleans())) + return amount(c, a, a + b, or_more=False, greedy=draw(st.booleans())) pat_generic: st.SearchStrategy[RegexPattern] = st.recursive(pat_base(), _pat_extend) diff --git a/tests/test_amount.py b/tests/test_amount.py index 5d44555..659d116 100644 --- a/tests/test_amount.py +++ b/tests/test_amount.py @@ -15,7 +15,7 @@ def test_operator_mul(x, n, data): @given(pat_generic, st.integers(0, 4), st.booleans(), st.data()) -def test_amount_fixed(x, n, greedy, data): +def test_amount_fixed1(x, n, greedy, data): check_one( amount(x, n, n, greedy=greedy), x * n, @@ -23,6 +23,15 @@ def test_amount_fixed(x, n, greedy, data): ) +@given(pat_generic, st.integers(0, 4), st.booleans(), st.data()) +def test_amount_fixed2(x, n, greedy, data): + check_one( + amount(x, n, greedy=greedy), + x * n, + data, + ) + + @given(pat_generic, st.integers(0, 4), st.integers(0, 4), st.booleans(), st.data()) def test_amount_bounded(x, n, m, greedy, data): check_one( From 5a5099bf5c87813ae8762443697ecb83d39f76ed Mon Sep 17 00:00:00 2001 From: Xinyang Chen Date: Wed, 15 May 2024 10:42:11 -0500 Subject: [PATCH 12/17] fix and update readme --- README.md | 54 ++++++++++++++++++++---------------------------------- 1 file changed, 20 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 86f05cf..244c42a 100644 --- a/README.md +++ b/README.md @@ -9,20 +9,18 @@ Dynamically construct python regex patterns. Say you want a regex pattern to match the initials of someones name. ```python -import re -from regexfactory import Amount, Range - +from regexfactory import * -pattern = Amount(Range("A", "Z"), 2, 3) +pattern = amount(Range("A", "Z"), 2, 3) -matches = pattern.findall( - "My initials are BDP. Valorie's are VO" -) +matches = pattern.findall("My initials are BDP. Valorie's are VO") +print(pattern.regex) print(matches) ``` -```bash +``` +[A-Z]{2,3} ['BDP', 'VO'] ``` @@ -31,36 +29,24 @@ print(matches) Or how matching both uppercase and lowercase hex strings in a sentence. ```python -import re from regexfactory import * -pattern = Optional("#") + Or( - Amount( - Set( - Range("0", "9"), - Range("a", "f") - ), - 6 - ), - Amount( - Set( - Range("0", "9"), - Range("A", "F") - ), - 6 - ), - +pattern = optional("#") + or_( + (Range("0", "9") | Range("a", "f")) * 6, + (Range("0", "9") | Range("A", "F")) * 6, ) sentence = """ My favorite color is #000000. I also like 5fb8a0. My second favorite color is #FF21FF. """ +print(pattern.regex) matches = pattern.findall(sentence) print(matches) ``` -```bash +``` +(?:#)?(?:[0-9a-f]{6}|[0-9A-F]{6}) ['#000000', '5fb8a0', '#FF21FF'] ``` @@ -71,11 +57,11 @@ Or what if you want to match urls in html content? ```python from regexfactory import * -protocol = Amount(Range("a", "z"), 1, or_more=True) -host = Amount(WORD | DIGIT | ".", 1, or_more=True) -port = Optional(":" + Multi(DIGIT)) -path = Multi( - "/" + Multi(NotSet("/", "#", "?", "&", WHITESPACE), match_zero=True), +protocol = amount(Range("a", "z"), 1, or_more=True) +host = amount(WORD | DIGIT | r"\.", 1, or_more=True) +port = optional(":" + multi(DIGIT)) +path = multi( + "/" + multi(NotSet("/", "#", "?", "&", WHITESPACE), match_zero=True), match_zero=True, ) patt = protocol + "://" + host + port + path @@ -87,9 +73,9 @@ print(patt.regex) print(patt.search(sentence)) ``` -```bash -[a-z]{1,}://[\w\d.]{1,}(?:\d{1,})?(/([^/#?&\s]{0,})){0,} - +``` +[a-z]+://[\w\d\.]+(?::\d+)?(?:/[^/\#\?\&\s]*)* + ``` ## The Pitch From a985e4c61db7eb2ba3edaa7f3e3978afea451650 Mon Sep 17 00:00:00 2001 From: Xinyang Chen Date: Wed, 15 May 2024 10:51:06 -0500 Subject: [PATCH 13/17] rename always to empty --- regexfactory/__init__.py | 2 +- regexfactory/pattern.py | 8 +++++--- regexfactory/sets.py | 10 ++++++---- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/regexfactory/__init__.py b/regexfactory/__init__.py index 1e238a2..0d236f0 100644 --- a/regexfactory/__init__.py +++ b/regexfactory/__init__.py @@ -52,6 +52,6 @@ Optional, Or, ) -from .sets import ALWAYS, NEVER, NotSet, Range, Set +from .sets import EMPTY, NEVER, NotSet, Range, Set __version__ = "1.0.1" diff --git a/regexfactory/pattern.py b/regexfactory/pattern.py index bab3869..642cbc2 100644 --- a/regexfactory/pattern.py +++ b/regexfactory/pattern.py @@ -279,7 +279,9 @@ def escape(string: str) -> RegexPattern: def _escape(string: str) -> RegexPattern: if len(string) == 0: - return RegexPattern("", _precedence=0) + from .sets import EMPTY + + return EMPTY if len(string) == 1: from .sets import CharLiteral # pylint: disable=import-outside-toplevel @@ -362,13 +364,13 @@ def _amount( pattern: RegexPattern, i: int, j: Optional[int], or_more: bool, greedy: bool ) -> RegexPattern: from .patterns import Amount, Multi, Optional - from .sets import ALWAYS + from .sets import EMPTY if j is None and not or_more: j = i if i == 0 and j == 0: - return ALWAYS + return EMPTY if i == 1 and j == 1: return pattern diff --git a/regexfactory/sets.py b/regexfactory/sets.py index e7e4f8f..55f72b4 100644 --- a/regexfactory/sets.py +++ b/regexfactory/sets.py @@ -2,15 +2,17 @@ import re import typing as t -from regexfactory import pattern as p from regexfactory.pattern import RegexPattern from regexfactory.patterns import IfNotAhead CharSet = t.Union[str, "Set", "CharLiteral", "CharClass", "Range"] -ALWAYS = RegexPattern.from_regex_str("") -ALWAYS._desc = "ALWAYS" -NEVER = IfNotAhead(ALWAYS) +# matches nothing, always succeeds +EMPTY = RegexPattern("", _precedence=0) +EMPTY._desc = "EMPTY" + +# always fails +NEVER = IfNotAhead(EMPTY) NEVER._desc = "NEVER" From 44de802257cddb4faf7c8c48a37ffcd3d969be47 Mon Sep 17 00:00:00 2001 From: Xinyang Chen Date: Fri, 17 May 2024 11:17:52 -0500 Subject: [PATCH 14/17] pylint --- .envrc | 1 + regexfactory/pattern.py | 28 ++++++++++++++++++++++------ regexfactory/patterns.py | 7 +++++-- regexfactory/sets.py | 18 ++++++++++++++++-- setup.cfg | 2 +- tests/utils.py | 2 +- 6 files changed, 46 insertions(+), 12 deletions(-) diff --git a/.envrc b/.envrc index 501f70e..f49b722 100644 --- a/.envrc +++ b/.envrc @@ -1 +1,2 @@ +PATH_add $(nix build --impure --expr "(import ).default.python.withPackages (ps: [ps.pip ps.pyparsing ps.hypothesis ps.pytest ps.pytest-cov ps.setuptools ps.ipython ps.pylint])" --no-link --print-out-paths)/bin path_add PYTHONPATH $(pwd)/ diff --git a/regexfactory/pattern.py b/regexfactory/pattern.py index 642cbc2..da2c27d 100644 --- a/regexfactory/pattern.py +++ b/regexfactory/pattern.py @@ -9,7 +9,7 @@ import re import sys -from typing import Any, Iterator, List, Literal, Optional, Tuple, Union, overload +from typing import Any, Iterator, List, Optional, Tuple, Union #: ValidPatternType = Union[re.Pattern, str, "RegexPattern"] @@ -84,6 +84,11 @@ def from_regex_str(regex: str) -> "RegexPattern": @staticmethod def create(obj: ValidPatternType) -> "RegexPattern": + """ + creats a RegexPattern from a str or RegexPattern. + if obj is a RegexPattern, its returned. + otherwise obj is treated as a regex. + """ if isinstance(obj, RegexPattern): return obj if isinstance(obj, str): @@ -96,7 +101,7 @@ def create(obj: ValidPatternType) -> "RegexPattern": def _ensure_precedence( pattern: ValidPatternType, precedence: int ) -> "RegexPattern": - from .patterns import Group # pylint: disable=import-outside-toplevel + from .patterns import Group p = RegexPattern.create(pattern) @@ -252,7 +257,7 @@ def search( def join(*patterns: ValidPatternType) -> RegexPattern: """Umbrella function for combining :class:`ValidPatternType`'s into a :class:`RegexPattern`.""" - from .patterns import Concat # pylint: disable=import-outside-toplevel + from .patterns import Concat ps = [RegexPattern.create(p) for p in patterns] ans = Concat(*ps) @@ -283,13 +288,16 @@ def _escape(string: str) -> RegexPattern: return EMPTY if len(string) == 1: - from .sets import CharLiteral # pylint: disable=import-outside-toplevel + from .sets import CharLiteral return CharLiteral(re.escape(string)) return RegexPattern(re.escape(string), _precedence=0) def or_(*args: ValidPatternType) -> RegexPattern: + """ + matches any one of args. args is tried from left to right. + """ from .patterns import Or from .sets import NEVER, Set, _is_charset @@ -363,7 +371,8 @@ def amount( def _amount( pattern: RegexPattern, i: int, j: Optional[int], or_more: bool, greedy: bool ) -> RegexPattern: - from .patterns import Amount, Multi, Optional + from .patterns import Amount, Multi + from .patterns import Optional as POpt from .sets import EMPTY if j is None and not or_more: @@ -376,7 +385,7 @@ def _amount( return pattern if i == 0 and j == 1: - return Optional(pattern, greedy) + return POpt(pattern, greedy) if i == 0 and j is None and or_more: return Multi(pattern, True, greedy) @@ -395,10 +404,17 @@ def multi( match_zero: bool = False, greedy: bool = True, ): + """ + maches one or more counts of pattern. + if match_zero=True, match zero or more instead. + """ if match_zero: return amount(pattern, 0, or_more=True, greedy=greedy) return amount(pattern, 1, or_more=True, greedy=greedy) def optional(pattern: ValidPatternType, greedy: bool = True): + """ + maches zero or one counts of pattern. + """ return amount(pattern, 0, 1, greedy=greedy) diff --git a/regexfactory/patterns.py b/regexfactory/patterns.py index 7536b37..66b7ce0 100644 --- a/regexfactory/patterns.py +++ b/regexfactory/patterns.py @@ -12,6 +12,10 @@ class Concat(RegexPattern): + """ + a concatation of patterns. + """ + def __init__(self, *patterns: ValidPatternType) -> None: super().__init__( "".join(map(self._ensure_precedence_fn(0), patterns)), @@ -78,6 +82,7 @@ class Amount(RegexPattern): """ + # pylint: disable=too-many-arguments def __init__( self, pattern: ValidPatternType, @@ -94,7 +99,6 @@ def __init__( else: amount = f"{i}" regex = ( - # TODO: should this be 1? self._ensure_precedence(pattern, 2).regex + "{" + amount @@ -119,7 +123,6 @@ def __init__( suffix = "*" if match_zero else "+" if greedy is False: suffix += "?" - # TODO: should this be 1? regex = self._ensure_precedence(pattern, 2).regex super().__init__(regex + suffix, _precedence=1) diff --git a/regexfactory/sets.py b/regexfactory/sets.py index 55f72b4..0a46a00 100644 --- a/regexfactory/sets.py +++ b/regexfactory/sets.py @@ -1,9 +1,23 @@ +""" +Charset Subclasses +************************ + +Charset is any pattern that matches exactly single character, out of a set. + +for example, :code:`'a'`, :code:`'[ab]'` or :code:`r'[a\\w]'` + +Charset gets special support. for example +:code:`or_("a", "b")` returns `` +""" + +# pylint: disable=cyclic-import + import itertools import re import typing as t -from regexfactory.pattern import RegexPattern -from regexfactory.patterns import IfNotAhead +from .pattern import RegexPattern +from .patterns import IfNotAhead CharSet = t.Union[str, "Set", "CharLiteral", "CharClass", "Range"] diff --git a/setup.cfg b/setup.cfg index c2e4b6b..723143c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -47,4 +47,4 @@ per-file-ignores = __init__.py:F401, conf.py:E402 profile = black [pylint.messages_control] -disable = too-few-public-methods, too-many-arguments, line-too-long +disable = too-few-public-methods, too-many-arguments, line-too-long, import-outside-toplevel, protected-access diff --git a/tests/utils.py b/tests/utils.py index e564fdb..90b790b 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -9,7 +9,7 @@ def check_regex(p1: ValidPatternType, p2: ValidPatternType, v: str): r1 = p1 if isinstance(p1, str) else RegexPattern.create(p1).regex r2 = p2 if isinstance(p2, str) else RegexPattern.create(p2).regex - """checks that r1 and r2 does the same on v""" + # checks that r1 and r2 does the same on v if r1 == r2: return From 8cfb795e14cf2a02cdc2abca82a3e2e01041d0a1 Mon Sep 17 00:00:00 2001 From: Xinyang Chen Date: Fri, 17 May 2024 11:23:30 -0500 Subject: [PATCH 15/17] remove unusedc imports --- tests/test_or.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_or.py b/tests/test_or.py index cfa557f..782d98e 100644 --- a/tests/test_or.py +++ b/tests/test_or.py @@ -4,7 +4,7 @@ from strategies import pat_generic from utils import check_one, check_regex -from regexfactory import RegexPattern, amount, escape, join, multi, optional, or_ +from regexfactory import RegexPattern, escape, or_ @given(pat_generic, pat_generic, st.data()) From 9a19ec8ae68c295fdee6637ad625a31154483d35 Mon Sep 17 00:00:00 2001 From: Xinyang Chen Date: Tue, 21 May 2024 19:24:31 -0500 Subject: [PATCH 16/17] getitem --- regexfactory/pattern.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/regexfactory/pattern.py b/regexfactory/pattern.py index da2c27d..d963e8c 100644 --- a/regexfactory/pattern.py +++ b/regexfactory/pattern.py @@ -140,6 +140,26 @@ def __mul__(self, coefficient: int) -> "RegexPattern": return amount(self, coefficient) + def __getitem__(self, arg: slice) -> "RegexPattern": + if isinstance(arg, slice): + assert arg.step is None + if isinstance(arg.start, int): + start = arg.start + else: + assert arg.start is None + start = 0 + if isinstance(arg.stop, int): + end = arg.stop + or_more = False + else: + assert arg.stop is None + end = None + or_more = True + + return amount(self, start, end, or_more=or_more) + + raise ValueError(f"{arg}") + def __or__(self, other: ValidPatternType): return or_(self, other) From 799d5cc66638c0a2244d298b08c7c5a5b2012a2a Mon Sep 17 00:00:00 2001 From: Xinyang Chen Date: Tue, 28 May 2024 20:50:14 -0500 Subject: [PATCH 17/17] experimental --- regexfactory/pattern.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regexfactory/pattern.py b/regexfactory/pattern.py index d963e8c..12d3de1 100644 --- a/regexfactory/pattern.py +++ b/regexfactory/pattern.py @@ -125,7 +125,7 @@ def __repr__(self) -> str: return f"" def __str__(self) -> str: - return self.regex + return self._ensure_precedence(self, 2).regex def __add__(self, other: ValidPatternType) -> "RegexPattern": """Adds two :class:`ValidPatternType`'s together, into a :class:`RegexPattern`"""