diff --git a/CHANGES.md b/CHANGES.md index cb5eced42f7c..a6508476cc3c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -80,6 +80,7 @@ * (Java) JmsIO migrated to `jakarta.jms` (JMS 3.1) and ActiveMQ 6.2.5. User code implementing `JmsIO.MessageMapper`, `valueMapper`, `topicNameMapper`, or providing a `ConnectionFactory` must update imports from `javax.jms.*` to `jakarta.jms.*`. The module now requires Java 17 at runtime ([#38729](https://github.com/apache/beam/issues/38729)). +* (Python) `Regex.find_all` with `group=Regex.ALL` now returns every group, as `(match, group1, group2, ...)`, instead of only group 1, and with `outputEmpty=False` it drops a match only when the whole match is empty, matching the `group=0` behavior ([#40399](https://github.com/apache/beam/pull/40399)). ## Deprecations diff --git a/sdks/python/apache_beam/transforms/util.py b/sdks/python/apache_beam/transforms/util.py index d99ebb456b56..b7885544dbfb 100644 --- a/sdks/python/apache_beam/transforms/util.py +++ b/sdks/python/apache_beam/transforms/util.py @@ -2219,28 +2219,29 @@ def _process(element): @staticmethod @typehints.with_input_types(str) - @typehints.with_output_types(Union[list[str], list[tuple[str, str]]]) + @typehints.with_output_types(Union[list[str], list[tuple[str, ...]]]) @ptransform_fn def find_all(pcoll, regex, group=0, outputEmpty=True): """ Returns the matches if a portion of the line matches the Regex. By default, list of group 0 will return with empty items. To get all groups, pass the `Regex.ALL` flag in the `group` parameter which returns all the groups in - the tuple format. + the tuple format. When using `Regex.ALL`, each item is formatted as a + tuple: `(match, group1, group2, ...)`. Args: regex: the regular expression string or (re.compile) pattern. group: (optional) name of the group, it can be integer or a string value. - outputEmpty: (optional) Should empty be output. True to output empties - and false if not. + outputEmpty: (optional) Whether to allow matches with empty groups. + If false, these outputs will be silently dropped. """ regex = Regex._regex_compile(regex) def _process(element): matches = regex.finditer(element) if group == Regex.ALL: - yield [(m.group(), m.groups()[0]) for m in matches - if outputEmpty or m.groups()[0]] + yield [(m.group(), *m.groups()) for m in matches + if outputEmpty or m.group()] else: yield [m.group(group) for m in matches if outputEmpty or m.group(group)] diff --git a/sdks/python/apache_beam/transforms/util_test.py b/sdks/python/apache_beam/transforms/util_test.py index e8e7d762b819..bba09245a0fb 100644 --- a/sdks/python/apache_beam/transforms/util_test.py +++ b/sdks/python/apache_beam/transforms/util_test.py @@ -2714,9 +2714,36 @@ def test_find_all_groups(self): pcol | 'all non empty groups' >> util.Regex.find_all( 'a(b*)', util.Regex.ALL, outputEmpty=False), equal_to([[('ab', 'b'), ('ab', 'b'), ('ab', 'b')], - [('abb', 'bb'), ('abbb', 'bbb')]]), + [('abb', 'bb'), ('a', ''), ('abbb', 'bbb')]]), label='CheckAllNonEmptyGroups') + def test_find_all_groups_multiple_and_no_groups(self): + with TestPipeline() as p: + pcol = (p | beam.Create(['a=1 b=2 c='])) + assert_that( + pcol + | 'two groups' >> util.Regex.find_all(r'(\w)=(\d?)', util.Regex.ALL), + equal_to([[('a=1', 'a', '1'), ('b=2', 'b', '2'), ('c=', 'c', '')]]), + label='CheckTwoGroups') + + assert_that( + pcol | 'two groups non empty match' >> util.Regex.find_all( + r'(\w)=(\d?)', util.Regex.ALL, outputEmpty=False), + equal_to([[('a=1', 'a', '1'), ('b=2', 'b', '2'), ('c=', 'c', '')]]), + label='CheckTwoGroupsNonEmptyMatch') + + digits = (p | 'create digits' >> beam.Create(['1 2'])) + assert_that( + digits | 'no groups' >> util.Regex.find_all(r'\d*', util.Regex.ALL), + equal_to([[('1', ), ('', ), ('2', ), ('', )]]), + label='CheckNoGroups') + + assert_that( + digits | 'no groups non empty' >> util.Regex.find_all( + r'\d*', util.Regex.ALL, outputEmpty=False), + equal_to([[('1', ), ('2', )]]), + label='CheckNoGroupsNonEmpty') + def test_find_kv(self): with TestPipeline() as p: pcol = (p | beam.Create(['a b c d']))