Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions test_thefuzz.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import unittest
import re
import warnings
import pycodestyle

from thefuzz import fuzz
Expand Down Expand Up @@ -505,6 +506,27 @@ def test_dedupe(self):
result = process.dedupe(contains_dupes)
self.assertEqual(result, deduped_list)

def test_dedupe_with_empty_processed_item(self):
"""dedupe() should not crash when an item is reduced to an empty
string by the default processor. Such an item has no fuzzy
duplicates (every comparison scores 0), so it is kept as its own
cluster instead of raising ValueError from max() of an empty
sequence. See issue #94.
"""
# '###' is reduced to '' by utils.full_process, so extractBests
# returns no matches for it.
self.assertEqual(utils.full_process('###'), '')

contains_dupes = ['###', 'apple', 'apple pie']
with warnings.catch_warnings():
warnings.simplefilter('ignore')
result = process.dedupe(contains_dupes)

# The empty-processed item is preserved rather than dropped or crashing.
self.assertIn('###', result)
# The genuine duplicates ('apple', 'apple pie') still collapse to one.
self.assertEqual(len(result), 2)

def test_simplematch(self):
basic_string = 'a, b'
match_strings = ['a, b']
Expand Down
8 changes: 8 additions & 0 deletions thefuzz/process.py
Original file line number Diff line number Diff line change
Expand Up @@ -438,6 +438,14 @@ def dedupe(
deduped = set()
for item in contains_dupes:
matches = extractBests(item, contains_dupes, scorer=scorer, score_cutoff=threshold, limit=None)
if not matches:
# The item's processed form is empty (e.g. it contains only
# characters the processor strips), so every comparison scores 0
# and nothing meets the threshold - not even the item itself.
# It has no fuzzy duplicates, so keep it as its own cluster
# instead of crashing on max() of an empty sequence.
deduped.add(item)
continue
deduped.add(max(matches, key=lambda x: (len(x[0]), x[0]))[0])

return list(deduped) if len(deduped) != len(contains_dupes) else contains_dupes