From 0f35e2516b700e867ca5f3d377d7acaf9aaddfe2 Mon Sep 17 00:00:00 2001 From: Anish Mehta Date: Sun, 4 Oct 2026 19:07:24 +0530 Subject: [PATCH 1/8] [Python] Return every group from Regex.find_all with Regex.ALL --- sdks/python/apache_beam/transforms/util.py | 6 +++--- sdks/python/apache_beam/transforms/util_test.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/sdks/python/apache_beam/transforms/util.py b/sdks/python/apache_beam/transforms/util.py index 30f8dc01c458..ec63015c553e 100644 --- a/sdks/python/apache_beam/transforms/util.py +++ b/sdks/python/apache_beam/transforms/util.py @@ -2214,7 +2214,7 @@ def _process(element): @staticmethod @typehints.with_input_types(str) - @typehints.with_output_types(Union[list[str], list[tuple[str, str]]]) + @typehints.with_output_types(Union[list[str], list[tuple[str, ...]]]) @ptransform_fn def find_all(pcoll, regex, group=0, outputEmpty=True): """ @@ -2234,8 +2234,8 @@ def find_all(pcoll, regex, group=0, outputEmpty=True): def _process(element): matches = regex.finditer(element) if group == Regex.ALL: - yield [(m.group(), m.groups()[0]) for m in matches - if outputEmpty or m.groups()[0]] + yield [(m.group(), *m.groups()) for m in matches + if outputEmpty or any(m.groups() or (m.group(), ))] else: yield [m.group(group) for m in matches if outputEmpty or m.group(group)] diff --git a/sdks/python/apache_beam/transforms/util_test.py b/sdks/python/apache_beam/transforms/util_test.py index 509f2f2c8975..5f26d992dc13 100644 --- a/sdks/python/apache_beam/transforms/util_test.py +++ b/sdks/python/apache_beam/transforms/util_test.py @@ -2678,6 +2678,21 @@ def test_find_all_groups(self): [('abb', 'bb'), ('abbb', 'bbb')]]), label='CheckAllNonEmptyGroups') + def test_find_all_groups_multiple_and_no_groups(self): + with TestPipeline() as p: + pcol = (p | beam.Create(['a=1 b=2 c='])) + assert_that( + pcol + | 'two groups' >> util.Regex.find_all(r'(\w)=(\d?)', util.Regex.ALL), + equal_to([[('a=1', 'a', '1'), ('b=2', 'b', '2'), ('c=', 'c', '')]]), + label='CheckTwoGroups') + + assert_that( + pcol | 'no groups' >> util.Regex.find_all( + r'\w=\d', util.Regex.ALL, outputEmpty=False), + equal_to([[('a=1', ), ('b=2', )]]), + label='CheckNoGroups') + def test_find_kv(self): with TestPipeline() as p: pcol = (p | beam.Create(['a b c d'])) From 9d249b901cc8523de4ceaf66d4f05e9aeec29c53 Mon Sep 17 00:00:00 2001 From: Anish Mehta <145196704+anishmehta24@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:57:41 +0530 Subject: [PATCH 2/8] Update sdks/python/apache_beam/transforms/util.py Co-authored-by: Danny McCormick --- sdks/python/apache_beam/transforms/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/transforms/util.py b/sdks/python/apache_beam/transforms/util.py index ec63015c553e..efb39a4979f3 100644 --- a/sdks/python/apache_beam/transforms/util.py +++ b/sdks/python/apache_beam/transforms/util.py @@ -2235,7 +2235,7 @@ def _process(element): matches = regex.finditer(element) if group == Regex.ALL: yield [(m.group(), *m.groups()) for m in matches - if outputEmpty or any(m.groups() or (m.group(), ))] + if outputEmpty or all(m.groups() or (m.group(), ))] else: yield [m.group(group) for m in matches if outputEmpty or m.group(group)] From 823064d13b8d0e220a04d1d4452ffaa3d92bcc95 Mon Sep 17 00:00:00 2001 From: Anish Mehta Date: Mon, 5 Oct 2026 23:00:00 +0530 Subject: [PATCH 3/8] Test outputEmpty=False with two groups and with empty matches --- .../python/apache_beam/transforms/util_test.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/sdks/python/apache_beam/transforms/util_test.py b/sdks/python/apache_beam/transforms/util_test.py index 5f26d992dc13..0b98047b1483 100644 --- a/sdks/python/apache_beam/transforms/util_test.py +++ b/sdks/python/apache_beam/transforms/util_test.py @@ -2688,11 +2688,23 @@ def test_find_all_groups_multiple_and_no_groups(self): label='CheckTwoGroups') assert_that( - pcol | 'no groups' >> util.Regex.find_all( - r'\w=\d', util.Regex.ALL, outputEmpty=False), - equal_to([[('a=1', ), ('b=2', )]]), + pcol | 'two non empty groups' >> util.Regex.find_all( + r'(\w)=(\d?)', util.Regex.ALL, outputEmpty=False), + equal_to([[('a=1', 'a', '1'), ('b=2', 'b', '2')]]), + label='CheckTwoNonEmptyGroups') + + digits = (p | 'create digits' >> beam.Create(['1 2'])) + assert_that( + digits | 'no groups' >> util.Regex.find_all(r'\d*', util.Regex.ALL), + equal_to([[('1', ), ('', ), ('2', ), ('', )]]), label='CheckNoGroups') + assert_that( + digits | 'no groups non empty' >> util.Regex.find_all( + r'\d*', util.Regex.ALL, outputEmpty=False), + equal_to([[('1', ), ('2', )]]), + label='CheckNoGroupsNonEmpty') + def test_find_kv(self): with TestPipeline() as p: pcol = (p | beam.Create(['a b c d'])) From a4aa3ae5c8f49a23526ef214f32ef7e17996d3c9 Mon Sep 17 00:00:00 2001 From: Anish Mehta Date: Tue, 6 Oct 2026 02:07:50 +0530 Subject: [PATCH 4/8] Gate Regex.ALL outputEmpty on the full match and note the change in CHANGES.md --- CHANGES.md | 2 +- sdks/python/apache_beam/transforms/util.py | 6 ++++-- sdks/python/apache_beam/transforms/util_test.py | 8 ++++---- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 788ffd08154f..cc3dfdb27ead 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -74,7 +74,7 @@ ## Breaking Changes -* X behavior was changed ([#X](https://github.com/apache/beam/issues/X)). +* (Python) `Regex.find_all` with `group=Regex.ALL` now returns every group, as `(match, group1, group2, ...)`, instead of only group 1, and with `outputEmpty=False` it drops a match only when the whole match is empty, the same as `group=0` ([#40399](https://github.com/apache/beam/pull/40399)). ## Deprecations diff --git a/sdks/python/apache_beam/transforms/util.py b/sdks/python/apache_beam/transforms/util.py index efb39a4979f3..fad0c6e43727 100644 --- a/sdks/python/apache_beam/transforms/util.py +++ b/sdks/python/apache_beam/transforms/util.py @@ -2221,7 +2221,9 @@ def find_all(pcoll, regex, group=0, outputEmpty=True): Returns the matches if a portion of the line matches the Regex. By default, list of group 0 will return with empty items. To get all groups, pass the `Regex.ALL` flag in the `group` parameter which returns all the groups in - the tuple format. + the tuple format. With `Regex.ALL` each item is + `(match, group1, group2, ...)`, and `outputEmpty=False` drops an item only + when the whole match is empty, the same as for group 0. Args: regex: the regular expression string or (re.compile) pattern. @@ -2235,7 +2237,7 @@ def _process(element): matches = regex.finditer(element) if group == Regex.ALL: yield [(m.group(), *m.groups()) for m in matches - if outputEmpty or all(m.groups() or (m.group(), ))] + if outputEmpty or m.group()] else: yield [m.group(group) for m in matches if outputEmpty or m.group(group)] diff --git a/sdks/python/apache_beam/transforms/util_test.py b/sdks/python/apache_beam/transforms/util_test.py index 0b98047b1483..2c2a901d7c49 100644 --- a/sdks/python/apache_beam/transforms/util_test.py +++ b/sdks/python/apache_beam/transforms/util_test.py @@ -2675,7 +2675,7 @@ def test_find_all_groups(self): pcol | 'all non empty groups' >> util.Regex.find_all( 'a(b*)', util.Regex.ALL, outputEmpty=False), equal_to([[('ab', 'b'), ('ab', 'b'), ('ab', 'b')], - [('abb', 'bb'), ('abbb', 'bbb')]]), + [('abb', 'bb'), ('a', ''), ('abbb', 'bbb')]]), label='CheckAllNonEmptyGroups') def test_find_all_groups_multiple_and_no_groups(self): @@ -2688,10 +2688,10 @@ def test_find_all_groups_multiple_and_no_groups(self): label='CheckTwoGroups') assert_that( - pcol | 'two non empty groups' >> util.Regex.find_all( + pcol | 'two groups non empty match' >> util.Regex.find_all( r'(\w)=(\d?)', util.Regex.ALL, outputEmpty=False), - equal_to([[('a=1', 'a', '1'), ('b=2', 'b', '2')]]), - label='CheckTwoNonEmptyGroups') + equal_to([[('a=1', 'a', '1'), ('b=2', 'b', '2'), ('c=', 'c', '')]]), + label='CheckTwoGroupsNonEmptyMatch') digits = (p | 'create digits' >> beam.Create(['1 2'])) assert_that( From 9d304ac2dd2d2fbe9b0c3c0ca14ce16d10eb064f Mon Sep 17 00:00:00 2001 From: Anish Mehta <145196704+anishmehta24@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:36:45 +0530 Subject: [PATCH 5/8] Update sdks/python/apache_beam/transforms/util.py Co-authored-by: Danny McCormick --- sdks/python/apache_beam/transforms/util.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sdks/python/apache_beam/transforms/util.py b/sdks/python/apache_beam/transforms/util.py index fad0c6e43727..cd6709b86cf5 100644 --- a/sdks/python/apache_beam/transforms/util.py +++ b/sdks/python/apache_beam/transforms/util.py @@ -2221,9 +2221,8 @@ def find_all(pcoll, regex, group=0, outputEmpty=True): Returns the matches if a portion of the line matches the Regex. By default, list of group 0 will return with empty items. To get all groups, pass the `Regex.ALL` flag in the `group` parameter which returns all the groups in - the tuple format. With `Regex.ALL` each item is - `(match, group1, group2, ...)`, and `outputEmpty=False` drops an item only - when the whole match is empty, the same as for group 0. + the tuple format. When using `Regex.ALL`, each item is formatted as a + tuple: `(match, group1, group2, ...)`. Args: regex: the regular expression string or (re.compile) pattern. From 68ac885275b2deeba698d40a950a46004734b311 Mon Sep 17 00:00:00 2001 From: Anish Mehta <145196704+anishmehta24@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:40:56 +0530 Subject: [PATCH 6/8] Apply suggestion from @damccorm Co-authored-by: Danny McCormick --- sdks/python/apache_beam/transforms/util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/transforms/util.py b/sdks/python/apache_beam/transforms/util.py index cd6709b86cf5..edbf2b1f6fc3 100644 --- a/sdks/python/apache_beam/transforms/util.py +++ b/sdks/python/apache_beam/transforms/util.py @@ -2228,7 +2228,8 @@ def find_all(pcoll, regex, group=0, outputEmpty=True): regex: the regular expression string or (re.compile) pattern. group: (optional) name of the group, it can be integer or a string value. outputEmpty: (optional) Should empty be output. True to output empties - and false if not. + outputEmpty: (optional) Whether to allow matches with empty groups. + If false, these outputs will be silently dropped. """ regex = Regex._regex_compile(regex) From fd2706a5641014037ebed9c1debebb238a899c8a Mon Sep 17 00:00:00 2001 From: Anish Mehta Date: Tue, 6 Oct 2026 20:41:30 +0530 Subject: [PATCH 7/8] Drop the leftover outputEmpty line and fix the docstring indent --- sdks/python/apache_beam/transforms/util.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sdks/python/apache_beam/transforms/util.py b/sdks/python/apache_beam/transforms/util.py index edbf2b1f6fc3..dd10f4511c02 100644 --- a/sdks/python/apache_beam/transforms/util.py +++ b/sdks/python/apache_beam/transforms/util.py @@ -2227,9 +2227,8 @@ def find_all(pcoll, regex, group=0, outputEmpty=True): Args: regex: the regular expression string or (re.compile) pattern. group: (optional) name of the group, it can be integer or a string value. - outputEmpty: (optional) Should empty be output. True to output empties - outputEmpty: (optional) Whether to allow matches with empty groups. - If false, these outputs will be silently dropped. + outputEmpty: (optional) Whether to allow matches with empty groups. + If false, these outputs will be silently dropped. """ regex = Regex._regex_compile(regex) From 7ca7540303578a6161ca2f50f6f4ba2baaafcdfb Mon Sep 17 00:00:00 2001 From: Anish Mehta <145196704+anishmehta24@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:42:26 +0530 Subject: [PATCH 8/8] Update CHANGES.md Co-authored-by: Danny McCormick --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index cc3dfdb27ead..11a60903fdf6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -74,7 +74,7 @@ ## Breaking Changes -* (Python) `Regex.find_all` with `group=Regex.ALL` now returns every group, as `(match, group1, group2, ...)`, instead of only group 1, and with `outputEmpty=False` it drops a match only when the whole match is empty, the same as `group=0` ([#40399](https://github.com/apache/beam/pull/40399)). +* (Python) `Regex.find_all` with `group=Regex.ALL` now returns every group, as `(match, group1, group2, ...)`, instead of only group 1, and with `outputEmpty=False` it drops a match only when the whole match is empty, matching the `group=0` behavior ([#40399](https://github.com/apache/beam/pull/40399)). ## Deprecations