From d67959162e254897ab432947e175651e75d27d83 Mon Sep 17 00:00:00 2001 From: Taksh Date: Wed, 24 Jun 2026 21:06:47 +0530 Subject: [PATCH] Fix open issues: typos, leaks, Hearst pattern, and scorer pairing. - Fix constructor typo in CandidateGenerator (#601) - Truncate cache filename hashes for eCryptfs/encfs (#590) - Rename empty TF-IDF counter variable (#593) - Fix which_look_like Hearst pattern hypernym (#595) - Close file handles in data_util and evaluate_ner (#599, #597) - Fix PerClassScorer span pairing (#592) Co-authored-by: Cursor --- scispacy/candidate_generation.py | 6 ++--- scispacy/data_util.py | 43 ++++++++++++++------------------ scispacy/file_cache.py | 4 +-- scispacy/hearst_patterns.py | 2 +- scispacy/per_class_scorer.py | 4 +-- scispacy/train_utils.py | 3 ++- 6 files changed, 29 insertions(+), 33 deletions(-) diff --git a/scispacy/candidate_generation.py b/scispacy/candidate_generation.py index d4b42c06..d42877ac 100644 --- a/scispacy/candidate_generation.py +++ b/scispacy/candidate_generation.py @@ -244,7 +244,7 @@ def __init__( [ann_index, tfidf_vectorizer, ann_concept_aliases_list, kb] ): raise ValueError( - "You cannot pass both a name argument and other constuctor arguments." + "You cannot pass both a name argument and other constructor arguments." ) # Set the name to the default, after we have checked @@ -466,11 +466,11 @@ def create_tfidf_ann_index( empty_tfidfs_boolean_flags = numpy.array( concept_alias_tfidfs.sum(axis=1) != 0 ).reshape(-1) - number_of_non_empty_tfidfs = sum(empty_tfidfs_boolean_flags == False) # noqa: E712 + number_of_empty_tfidfs = sum(empty_tfidfs_boolean_flags == False) # noqa: E712 total_number_of_tfidfs = numpy.size(concept_alias_tfidfs, 0) print( - f"Deleting {number_of_non_empty_tfidfs}/{total_number_of_tfidfs} aliases because their tfidf is empty" + f"Deleting {number_of_empty_tfidfs}/{total_number_of_tfidfs} aliases because their tfidf is empty" ) # remove empty tfidf vectors, otherwise nmslib will crash concept_aliases = [ diff --git a/scispacy/data_util.py b/scispacy/data_util.py index 87cd2064..6e6f5637 100644 --- a/scispacy/data_util.py +++ b/scispacy/data_util.py @@ -183,18 +183,12 @@ def _cleanup_dir(dir_path: str): corpus = os.path.join(resolved_directory_path, expected_names[0]) examples = med_mentions_example_iterator(corpus) - train_ids = { - x.strip() - for x in open(os.path.join(resolved_directory_path, expected_names[4])) - } - dev_ids = { - x.strip() - for x in open(os.path.join(resolved_directory_path, expected_names[2])) - } - test_ids = { - x.strip() - for x in open(os.path.join(resolved_directory_path, expected_names[3])) - } + with open(os.path.join(resolved_directory_path, expected_names[4])) as train_file: + train_ids = {x.strip() for x in train_file} + with open(os.path.join(resolved_directory_path, expected_names[2])) as dev_file: + dev_ids = {x.strip() for x in dev_file} + with open(os.path.join(resolved_directory_path, expected_names[3])) as test_file: + test_ids = {x.strip() for x in test_file} train_examples = [] dev_examples = [] @@ -299,19 +293,20 @@ def read_ner_from_tsv(filename: str) -> List[SpacyNerExample]: """ spacy_format_data = [] examples: List[Tuple[str, str]] = [] - for line in open(cached_path(filename)): - line = line.strip() - if line.startswith("-DOCSTART-"): - continue - # We have reached the end of a sentence. - if not line: - if not examples: + with open(cached_path(filename)) as tsv_file: + for line in tsv_file: + line = line.strip() + if line.startswith("-DOCSTART-"): continue - spacy_format_data.append(_handle_sentence(examples)) - examples = [] - else: - word, entity = line.split("\t") - examples.append((word, entity)) + # We have reached the end of a sentence. + if not line: + if not examples: + continue + spacy_format_data.append(_handle_sentence(examples)) + examples = [] + else: + word, entity = line.split("\t") + examples.append((word, entity)) if examples: spacy_format_data.append(_handle_sentence(examples)) diff --git a/scispacy/file_cache.py b/scispacy/file_cache.py index 9ff99180..42391643 100644 --- a/scispacy/file_cache.py +++ b/scispacy/file_cache.py @@ -60,12 +60,12 @@ def url_to_filename(url: str, etag: Optional[str] = None) -> str: last_part = url.split("/")[-1] url_bytes = url.encode("utf-8") url_hash = sha256(url_bytes) - filename = url_hash.hexdigest() + filename = url_hash.hexdigest()[:16] if etag: etag_bytes = etag.encode("utf-8") etag_hash = sha256(etag_bytes) - filename += "." + etag_hash.hexdigest() + filename += "." + etag_hash.hexdigest()[:16] filename += "." + last_part return filename diff --git a/scispacy/hearst_patterns.py b/scispacy/hearst_patterns.py index 992b7a43..c28b16a7 100644 --- a/scispacy/hearst_patterns.py +++ b/scispacy/hearst_patterns.py @@ -492,7 +492,7 @@ {"LEMMA": "which"}, {"LEMMA": "look"}, {"LEMMA": "like"}, - hyponym, + hypernym, ], "position": "last", }, diff --git a/scispacy/per_class_scorer.py b/scispacy/per_class_scorer.py index 57993b0b..ade6af8f 100644 --- a/scispacy/per_class_scorer.py +++ b/scispacy/per_class_scorer.py @@ -18,9 +18,9 @@ def __call__( gold_spans = copy.copy(gold_spans) predicted_spans = copy.copy(predicted_spans) untyped_gold_spans = {(x[0], x[1]) for x in gold_spans} - untyped_predicted_spans = {(x[0], x[1]) for x in predicted_spans} - for untyped_span, span in zip(untyped_predicted_spans, predicted_spans): + for span in predicted_spans: + untyped_span = (span[0], span[1]) if span in gold_spans: self._true_positives[span[2]] += 1 gold_spans.remove(span) diff --git a/scispacy/train_utils.py b/scispacy/train_utils.py index f02fd188..5ee826d0 100644 --- a/scispacy/train_utils.py +++ b/scispacy/train_utils.py @@ -26,7 +26,8 @@ def evaluate_ner( metrics = scorer.get_metric() if dump_path is not None: - json.dump(metrics, open(dump_path, "a+")) + with open(dump_path, "a+") as metrics_file: + json.dump(metrics, metrics_file) for name, metric in metrics.items(): if "overall" in name or "untyped" in name or verbose: print(f"{name}: \t\t {metric}")