From d14f3d1e2dee9d94254b409571da395797746edf Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 21:06:07 +0000 Subject: [PATCH 1/2] Vectorize reciprocal ANI averaging in add_avani (issue #308) add_avani looped over every Ndb row with iterrows() into a string-keyed dict. On a single large primary cluster (e.g. 28k same-species genomes) that is ~811M rows, taking hours and far too much memory. Encode each (reference, querry) pair as an integer and look up reverse pairs in bulk instead. Output is identical, including duplicate and NaN handling and the KeyError on a missing reverse pair; clustering a 2,000-genome primary cluster goes from 133s to 6s. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013UpcPFwkh6QMusnD6ApThL --- CHANGELOG.md | 9 +++++++++ drep/d_cluster/external.py | 36 +++++++++++++++++++++++++----------- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5a068f..3eabc69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project (attempts to) adhere to [Semantic Versioning](http://semver.org/). +## [Unreleased] + +### Fixed +- Averaging reciprocal ANI values before secondary clustering is now vectorized + instead of looping over every row of Ndb (issue #308). Results are identical, + but it's ~25-30x faster on large primary clusters (a 2,000-genome primary + cluster went from 133s to 6s to cluster) and no longer builds a string-keyed + dictionary with an entry for every comparison + ## [4.0.1] - 2026-09-03 **You can now hand raw CheckM2 output straight to `--genomeInfo`.** No awk, no diff --git a/drep/d_cluster/external.py b/drep/d_cluster/external.py index 036e8af..8fd9616 100644 --- a/drep/d_cluster/external.py +++ b/drep/d_cluster/external.py @@ -717,17 +717,31 @@ def add_avani(db): db: dataframe ''' - logging.debug('making dictionary for average_ani') - combo2value = {} - for i, row in db.iterrows(): - combo2value["{0}-vs-{1}".format(row['querry'], row['reference'])] \ - = row['ani'] - - logging.debug('list comprehension for average_ani') - db['av_ani'] = [np.mean([combo2value["{0}-vs-{1}".format(q, r)], - combo2value["{0}-vs-{1}".format(r, q)]]) if r != q else 1\ - for q, r in zip(db['querry'].tolist(), - db['reference'].tolist())] + # Vectorized: this used to iterrows() into a string-keyed dict, which took + # hours and hundreds of GB once a single primary cluster held tens of + # thousands of genomes (issue #308). Each (reference, querry) pair is encoded + # as one integer so its reverse can be looked up in bulk. + logging.debug('averaging reciprocal ANI values') + ref = db['reference'].values + qry = db['querry'].values + names = pd.Index(pd.unique(np.concatenate([ref, qry]))) + n = len(names) + r = names.get_indexer(ref).astype(np.int64) + q = names.get_indexer(qry).astype(np.int64) + + # When a pair appears more than once the last value wins, as before + ani = pd.Series(db['ani'].values, index=r * n + q) + ani = ani[~ani.index.duplicated(keep='last')] + forward = ani.reindex(r * n + q).values + reverse = ani.reindex(q * n + r).values + + same = r == q + missing = ~np.isin(q * n + r, ani.index.values) & ~same + if missing.any(): + i = np.argmax(missing) + raise KeyError("{0}-vs-{1}".format(ref[i], qry[i])) + + db['av_ani'] = np.where(same, 1, (forward + reverse) / 2) logging.debug('averageing done') From e5e1aee5852b38c9c002e885513bbeb81299136a Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 21:22:56 +0000 Subject: [PATCH 2/2] Release 4.0.2 Bump the version and date the changelog. The fix is the vectorized reciprocal ANI averaging that makes large single-species primary clusters cluster ~25-30x faster (issue #308). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013UpcPFwkh6QMusnD6ApThL --- CHANGELOG.md | 2 +- drep/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3eabc69..912b9b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project (attempts to) adhere to [Semantic Versioning](http://semver.org/). -## [Unreleased] +## [4.0.2] - 2026-09-30 ### Fixed - Averaging reciprocal ANI values before secondary clustering is now vectorized diff --git a/drep/VERSION b/drep/VERSION index 1454f6e..4d54dad 100644 --- a/drep/VERSION +++ b/drep/VERSION @@ -1 +1 @@ -4.0.1 +4.0.2