From ae9e3b37f46ae7e5d79bdd8c03538e54411c8629 Mon Sep 17 00:00:00 2001 From: LCK <35679285+lckdl@users.noreply.github.com> Date: Wed, 19 Jul 2023 22:36:00 +0800 Subject: [PATCH] speed up to_hash --- .../transformers/data_processing/slimpajama/dedup/to_hash.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modelzoo/transformers/data_processing/slimpajama/dedup/to_hash.py b/modelzoo/transformers/data_processing/slimpajama/dedup/to_hash.py index fcf308ed..c17d687d 100644 --- a/modelzoo/transformers/data_processing/slimpajama/dedup/to_hash.py +++ b/modelzoo/transformers/data_processing/slimpajama/dedup/to_hash.py @@ -66,7 +66,7 @@ def to_minhash(chunks): output_name = f"{dataset_name}/{file_name}" m = MinHash(num_perm=128) - [m.update(x.encode('utf8')) for x in get_features(text, width)] + m.update_batch(map(lambda x: x.encode('utf8'), get_features(text, width))) buckets.append( {"file_name": output_name, "doc_id": doc_id, "hash": m,} )