diff --git a/test_thefuzz.py b/test_thefuzz.py index f816a69..f12a222 100644 --- a/test_thefuzz.py +++ b/test_thefuzz.py @@ -1,5 +1,6 @@ import unittest import re +import warnings import pycodestyle from thefuzz import fuzz @@ -505,6 +506,27 @@ def test_dedupe(self): result = process.dedupe(contains_dupes) self.assertEqual(result, deduped_list) + def test_dedupe_with_empty_processed_item(self): + """dedupe() should not crash when an item is reduced to an empty + string by the default processor. Such an item has no fuzzy + duplicates (every comparison scores 0), so it is kept as its own + cluster instead of raising ValueError from max() of an empty + sequence. See issue #94. + """ + # '###' is reduced to '' by utils.full_process, so extractBests + # returns no matches for it. + self.assertEqual(utils.full_process('###'), '') + + contains_dupes = ['###', 'apple', 'apple pie'] + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + result = process.dedupe(contains_dupes) + + # The empty-processed item is preserved rather than dropped or crashing. + self.assertIn('###', result) + # The genuine duplicates ('apple', 'apple pie') still collapse to one. + self.assertEqual(len(result), 2) + def test_simplematch(self): basic_string = 'a, b' match_strings = ['a, b'] diff --git a/thefuzz/process.py b/thefuzz/process.py index fcf7e1e..7d8fe66 100644 --- a/thefuzz/process.py +++ b/thefuzz/process.py @@ -438,6 +438,14 @@ def dedupe( deduped = set() for item in contains_dupes: matches = extractBests(item, contains_dupes, scorer=scorer, score_cutoff=threshold, limit=None) + if not matches: + # The item's processed form is empty (e.g. it contains only + # characters the processor strips), so every comparison scores 0 + # and nothing meets the threshold - not even the item itself. + # It has no fuzzy duplicates, so keep it as its own cluster + # instead of crashing on max() of an empty sequence. + deduped.add(item) + continue deduped.add(max(matches, key=lambda x: (len(x[0]), x[0]))[0]) return list(deduped) if len(deduped) != len(contains_dupes) else contains_dupes