From 5ec6553ddf3003405c26528c24c013c1322b081a Mon Sep 17 00:00:00 2001 From: Duncan Grisby Date: Tue, 3 Sep 2024 11:12:55 +0100 Subject: [PATCH] Extract better substrings Find substrings in more cases, and allow a single regex to register multiple substrings. --- src/esmre.py | 126 ++++++++++++++++++++++++--------- test/test_esmre.py | 172 ++++++++++++++++++++++----------------------- 2 files changed, 177 insertions(+), 121 deletions(-) diff --git a/src/esmre.py b/src/esmre.py index 51f6950..eb5ad8f 100644 --- a/src/esmre.py +++ b/src/esmre.py @@ -3,6 +3,7 @@ # esmre.py - clue-indexed regular expressions module # Copyright (C) 2007-2008 Tideway Systems Limited. +# Copyright (C) 2021-2024 BMC Software. # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -22,16 +23,57 @@ import esm import threading +class InBackslashState: + def __init__(self, parent_state): + self.parent_state = parent_state + + def process_byte(self, ch): + if not ch.isalnum(): + try: + self.parent_state.append_to_current_hint(ch) + except AttributeError: + # Parent does not have append_to_current_hint + pass + + return self.parent_state + + if ch.isdigit(): + return InBackslashNumberState(self.parent_state) + + if ch == "N": + return InNamedUnicodeState(self.parent_state) + + try: + self.parent_state.bank_current_hint_with_last_byte() + except AttributeError: + pass + + return self.parent_state + + +class InBackslashNumberState: + def __init__(self, parent_state): + self.parent_state = parent_state + + def process_byte(self, ch): + if not ch.isdigit(): + return self.parent_state + + return self + -class InBackslashState(object): +class InNamedUnicodeState: def __init__(self, parent_state): self.parent_state = parent_state def process_byte(self, ch): + if ch == "{": + return InBracesState(self.parent_state) + return self.parent_state -class InClassState(object): +class InClassState: def __init__(self, parent_state): self.parent_state = parent_state @@ -46,7 +88,7 @@ def process_byte(self, ch): return self -class InBracesState(object): +class InBracesState: def __init__(self, parent_state): self.parent_state = parent_state @@ -58,10 +100,13 @@ def process_byte(self, ch): return self -class CollectingState(object): +class CollectingState: def __init__(self): self.hints = [""] + def finish(self): + pass + def process_byte(self, ch): self.update_hints(ch) return self.next_state(ch) @@ -77,9 +122,6 @@ def bank_current_hint_and_forget_last_byte(self): self.hints.append("") - def forget_all_hints(self): - self.hints = [""] - def append_to_current_hint(self, ch): self.hints[-1] += ch @@ -87,11 +129,11 @@ def update_hints(self, ch): if ch in "?*{": self.bank_current_hint_and_forget_last_byte() - elif ch in "+.^$([\\": + elif ch in "+.^$([": self.bank_current_hint_with_last_byte() - elif ch == "|": - self.forget_all_hints() + elif ch in "|\\": + pass else: self.append_to_current_hint(ch) @@ -120,11 +162,21 @@ def alternation_state(self): class RootState(CollectingState): + def __init__(self): + CollectingState.__init__(self) + self.alternate_hints = [] + + def finish(self): + self.alternate_hints.append(self.hints) + self.hints = [""] + def alternation_state(self): - raise StopIteration + self.alternate_hints.append(self.hints) + self.hints = [""] + return self -class StartOfGroupState(object): +class StartOfGroupState: def __init__(self, parent_state): self.parent_state = parent_state @@ -162,18 +214,22 @@ def alternation_state(self): return self -class StartOfExtensionGroupState(object): +class StartOfExtensionGroupState: def __init__(self, parent_state): self.parent_state = parent_state def process_byte(self, ch): if ch == "P": return MaybeStartOfNamedGroupState(self.parent_state) + + elif ch == ":": + return InGroupState(self.parent_state) + else: return IgnoredGroupState(self.parent_state).process_byte(ch) -class MaybeStartOfNamedGroupState(object): +class MaybeStartOfNamedGroupState: def __init__(self, parent_state): self.parent_state = parent_state @@ -184,7 +240,7 @@ def process_byte(self, ch): return IgnoredGroupState(self.parent_state) -class InNamedGroupNameState(object): +class InNamedGroupNameState: def __init__(self, parent_state): self.parent_state = parent_state @@ -207,40 +263,40 @@ def hints(regex): for ch in regex: state = state.process_byte(ch) + state.finish() + except StopIteration: pass - def flattened(hints): - for item in hints: + all_hints = state.alternate_hints + + def flattened(l): + for item in l: if isinstance(item, list): for i in flattened(item): yield i else: yield item - return [hint for hint in flattened(state.hints) if hint] - - -def shortlist(hints): - if not hints: - return [] + def best(hints): + return max((hint for hint in flattened(hints)), key=len) - best = "" + result = {best(hints).lower() for hints in all_hints} - for hint in hints: - if len(hint) > len(best): - best = hint + if all(result): + return result - return [best] + return set() -class Index(object): +class Index: def __init__(self): self.esm = esm.Index() self.hintless_objects = list() self.fixed = False self.lock = threading.Lock() + def enter(self, regex, obj): self.lock.acquire() try: @@ -248,17 +304,18 @@ def enter(self, regex, obj): if self.fixed: raise TypeError("enter() cannot be called after query()") - keywords = shortlist(hints(regex)) + keywords = hints(regex) if not keywords: self.hintless_objects.append(obj) - for hint in shortlist(hints(regex)): - self.esm.enter(hint.lower(), obj) + for hint in keywords: + self.esm.enter(hint, obj) finally: self.lock.release() + def query(self, string): self.lock.acquire() try: @@ -270,6 +327,5 @@ def query(self, string): finally: self.lock.release() - return self.hintless_objects + [ - obj for (_, obj) in self.esm.query(string.lower()) - ] + return self.hintless_objects + \ + [obj for (_, obj) in self.esm.query(string.lower())] diff --git a/test/test_esmre.py b/test/test_esmre.py index 1460a84..414e574 100644 --- a/test/test_esmre.py +++ b/test/test_esmre.py @@ -3,6 +3,7 @@ # esmre_tests.py - tests for esmre module # Copyright (C) 2007-2008 Tideway Systems Limited. +# Copyright (C) 2021-2024 BMC Software. # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -22,136 +23,140 @@ import unittest import esmre - class HintExtractionTests(unittest.TestCase): def checkHints(self, expected_hints, regex): - self.assertEqual(set(expected_hints), set(esmre.hints(regex))) + self.assertEqual(expected_hints, esmre.hints(regex)) def testSimpleString(self): - self.checkHints(["yarr"], r"yarr") + self.checkHints({"yarr"}, r"yarr") def testSkipsOptionalCharacter(self): - self.checkHints(["dubloon"], r"dubloons?") + self.checkHints({"dubloon"}, r"dubloons?") def testStartsNewStringAfterOptionalCharacter(self): - self.checkHints(["ship", "shape"], r"ship ?shape") + self.checkHints({"shape"}, r"ship ?shape") def testSkipsOptionalRepeatedCharacter(self): - self.checkHints(["bristol", "fasion"], r"bristol *fasion") + self.checkHints({"bristol"}, r"bristol *fasion") def testIncludesRepeatedCharacterButStartsNewHint(self): - self.checkHints(["ava", "st me harties"], r"ava+st me harties") + self.checkHints({"st me harties"}, + r"ava+st me harties") def testSkipsGroupsWithAlternation(self): - self.checkHints( - ["Hoist the ", ", ye ", "!"], - r"Hoist the (mizzen mast|main brace), " - r"ye (landlubbers|scurvy dogs)!", - ) + self.checkHints({"hoist the "}, + r"Hoist the (mizzen mast|main brace), " + r"ye (landlubbers|scurvy dogs)!") def testSkipsAny(self): - self.checkHints( - ["After 10 paces, ", " marks the spot"], - r"After 10 paces, . marks the spot", - ) + self.checkHints({"after 10 paces, "}, + r"After 10 paces, . marks the spot") def testSkipsOneOrMoreAny(self): - self.checkHints(["Hard to ", "!"], r"Hard to .+!") + self.checkHints({"hard to "}, + r"Hard to .+!") def testSkipsNestedGroups(self): - self.checkHints( - ["Squark!", " Pieces of ", "!"], r"Squark!( Pieces of (.+)!)" - ) + self.checkHints({" pieces of "}, + r"Squark!( Pieces of (.+)!)") def testSkipsCharacterClass(self): - self.checkHints(["r"], r"[ya]a*r+") + self.checkHints({"r"}, + r"[ya]a*r+") def testRightBracketDoesNotCloseGroupIfInClass(self): - self.checkHints([":=", "X"], r":=([)D])X") + self.checkHints({":="}, + r":=([)D])X") def testSkipsBackslashMetacharacters(self): - self.checkHints(["Cap'n", " ", " Beard"], r"Cap'n\b ([\S] Beard)") + self.checkHints({" beard"}, + r"Cap'n\b ([\S] Beard)") def testBackslashBracketDoesNotCloseGroup(self): - self.checkHints([":=", "X"], r":=(\)|D)X") + self.checkHints({":="}, + r":=(\)|D)X") def testBackslashSquareBracketDoesNotCloseClass(self): - self.checkHints([":=", "X"], r":=[)D\]]X") + self.checkHints({":="}, + r":=[)D\]]X") def testSkipsMetacharactersAfterGroups(self): - self.checkHints( - ["Yo ", "ho ", " and a bottle of rum"], - r"Yo (ho )+ and a bottle of rum", - ) + self.checkHints({" and a bottle of rum"}, + r"Yo (ho )+ and a bottle of rum") def testSkipsRepetionBraces(self): - self.checkHints(["A", ", me harties"], r"Ar{2-10}, me harties") + self.checkHints({", me harties"}, + r"Ar{2-10}, me harties") - def testAlternationCausesEmptyResult(self): - self.checkHints([], r"rum|grog") + def testAlternation(self): + self.checkHints({"rum", "grog"}, r"rum|grog") def testSkipMatchBeginning(self): - self.checkHints(["The black perl"], r"^The black perl") + self.checkHints({"the black perl"}, r"^The black perl") def testSkipMatchEnd(self): - self.checkHints(["Davey Jones' Locker"], r"Davey Jones' Locker$") + self.checkHints({"davey jones' locker"}, r"Davey Jones' Locker$") def testOnlyGroupGivesEmptyResult(self): - self.checkHints([], r"(rum|grog)") + self.checkHints(set(), r"(rum|grog)") def testGetsHintsFromGroups(self): - self.checkHints(["/"], r"([0-3][0-9]/[0-1][0-9]/[1-2][0-9]{3})") + self.checkHints({"/"}, r"([0-3][0-9]/[0-1][0-9]/[1-2][0-9]{3})") def testSkipsOptionalGroups(self): - self.checkHints(["Shiver me timbers!"], r"Shiver me timbers!( Arrr!)?") + self.checkHints({"shiver me timbers!"}, + r"Shiver me timbers!( Arrr!)?") def testSkipsMostExtensionGroups(self): for regex in [ - # set flag - r"(?i)(?L)(?m)(?s)(?u)(?x)", - # non-grouping paren - r"(?:foo)", - # previous named group - r"(?P=foo)", - # comment - r"(?#foo)", - # lookahead - r"(?=foo)", - # negative lookahead - r"(?!foo)", - # lookbehind - r"(?<=foo)", - # negative lookbehind - r"(?[0-3][0-9]/[0-1][0-9]/[1-2][0-9]{3})" - ) + {"/"}, r"(?P[0-3][0-9]/[0-1][0-9]/[1-2][0-9]{3})") + def testNonGrouping(self): + self.checkHints({"foo"}, r"(?:foo)") -class ShortlistTests(unittest.TestCase): - def checkShortlist(self, expected_shortlist, hints): - self.assertEqual(expected_shortlist, esmre.shortlist(hints)) + def testEscapedSlash(self): + self.checkHints({r"foo\bar"}, r"foo\\bar") - def testShortlistIsEmptyForEmptyCandidates(self): - self.checkShortlist([], []) + def testEscapedDot(self): + self.checkHints({r"foo.bar"}, r"foo\.bar") - def testShortlistIsOnlyCandidate(self): - self.checkShortlist(["Blue Beard"], ["Blue Beard"]) + def testOptionalGroupAfter(self): + self.checkHints({"short"}, r"short(longer)?") - def testShorlistSelectsLongestCandidate(self): - self.checkShortlist(["Black Beard"], ["Black Beard", "Blue Beard"]) + def testOptionalGroupBefore(self): + self.checkHints({"short"}, r"(longer)?short") - def testShorlistSelectsLongestCandidateAtEnd(self): - self.checkShortlist( - ["Yellow Beard"], ["Black Beard", "Blue Beard", "Yellow Beard"] - ) + def testOptionalGroupMiddle(self): + self.checkHints({"short"}, r"tiny(longer)?short") class IndexTests(unittest.TestCase): @@ -161,25 +166,20 @@ def setUp(self): self.index.enter(r"\bway\W+haye?\b", "sea shanty") def testSingleQuery(self): - self.assertEqual( - ["savoy opera"], - self.index.query("I am the very model of a modern Major-General."), - ) + self.assertEqual(["savoy opera"], self.index.query( + "I am the very model of a modern Major-General.")) def testCannotEnterAfterQuery(self): self.index.query("blah") self.assertRaises(TypeError, self.index.enter, "foo", "bar") def testCaseInsensitive(self): - self.assertEqual( - ["sea shanty"], self.index.query("Way, hay up she rises,") - ) - self.assertEqual( - ["sea shanty"], - self.index.query("To my way haye, blow the man down,"), - ) - - def testAlwaysReportsOpjectForHintlessExpressions(self): + self.assertEqual(["sea shanty"], self.index.query( + "Way, hay up she rises,")) + self.assertEqual(["sea shanty"], self.index.query( + "To my way haye, blow the man down,")) + + def testAlwaysReportsObjectForHintlessExpressions(self): self.index.enter(r"(\d+\s)*(paces|yards)", "distance") self.assertTrue("distance" in self.index.query("'til morning"))