diff --git a/scispacy/candidate_generation.py b/scispacy/candidate_generation.py index d4b42c0..d42877a 100644 --- a/scispacy/candidate_generation.py +++ b/scispacy/candidate_generation.py @@ -244,7 +244,7 @@ def __init__( [ann_index, tfidf_vectorizer, ann_concept_aliases_list, kb] ): raise ValueError( - "You cannot pass both a name argument and other constuctor arguments." + "You cannot pass both a name argument and other constructor arguments." ) # Set the name to the default, after we have checked @@ -466,11 +466,11 @@ def create_tfidf_ann_index( empty_tfidfs_boolean_flags = numpy.array( concept_alias_tfidfs.sum(axis=1) != 0 ).reshape(-1) - number_of_non_empty_tfidfs = sum(empty_tfidfs_boolean_flags == False) # noqa: E712 + number_of_empty_tfidfs = sum(empty_tfidfs_boolean_flags == False) # noqa: E712 total_number_of_tfidfs = numpy.size(concept_alias_tfidfs, 0) print( - f"Deleting {number_of_non_empty_tfidfs}/{total_number_of_tfidfs} aliases because their tfidf is empty" + f"Deleting {number_of_empty_tfidfs}/{total_number_of_tfidfs} aliases because their tfidf is empty" ) # remove empty tfidf vectors, otherwise nmslib will crash concept_aliases = [ diff --git a/scispacy/data_util.py b/scispacy/data_util.py index 87cd206..6e6f563 100644 --- a/scispacy/data_util.py +++ b/scispacy/data_util.py @@ -183,18 +183,12 @@ def _cleanup_dir(dir_path: str): corpus = os.path.join(resolved_directory_path, expected_names[0]) examples = med_mentions_example_iterator(corpus) - train_ids = { - x.strip() - for x in open(os.path.join(resolved_directory_path, expected_names[4])) - } - dev_ids = { - x.strip() - for x in open(os.path.join(resolved_directory_path, expected_names[2])) - } - test_ids = { - x.strip() - for x in open(os.path.join(resolved_directory_path, expected_names[3])) - } + with open(os.path.join(resolved_directory_path, expected_names[4])) as train_file: + train_ids = {x.strip() for x in train_file} + with open(os.path.join(resolved_directory_path, expected_names[2])) as dev_file: + dev_ids = {x.strip() for x in dev_file} + with open(os.path.join(resolved_directory_path, expected_names[3])) as test_file: + test_ids = {x.strip() for x in test_file} train_examples = [] dev_examples = [] @@ -299,19 +293,20 @@ def read_ner_from_tsv(filename: str) -> List[SpacyNerExample]: """ spacy_format_data = [] examples: List[Tuple[str, str]] = [] - for line in open(cached_path(filename)): - line = line.strip() - if line.startswith("-DOCSTART-"): - continue - # We have reached the end of a sentence. - if not line: - if not examples: + with open(cached_path(filename)) as tsv_file: + for line in tsv_file: + line = line.strip() + if line.startswith("-DOCSTART-"): continue - spacy_format_data.append(_handle_sentence(examples)) - examples = [] - else: - word, entity = line.split("\t") - examples.append((word, entity)) + # We have reached the end of a sentence. + if not line: + if not examples: + continue + spacy_format_data.append(_handle_sentence(examples)) + examples = [] + else: + word, entity = line.split("\t") + examples.append((word, entity)) if examples: spacy_format_data.append(_handle_sentence(examples)) diff --git a/scispacy/file_cache.py b/scispacy/file_cache.py index 9ff9918..4239164 100644 --- a/scispacy/file_cache.py +++ b/scispacy/file_cache.py @@ -60,12 +60,12 @@ def url_to_filename(url: str, etag: Optional[str] = None) -> str: last_part = url.split("/")[-1] url_bytes = url.encode("utf-8") url_hash = sha256(url_bytes) - filename = url_hash.hexdigest() + filename = url_hash.hexdigest()[:16] if etag: etag_bytes = etag.encode("utf-8") etag_hash = sha256(etag_bytes) - filename += "." + etag_hash.hexdigest() + filename += "." + etag_hash.hexdigest()[:16] filename += "." + last_part return filename diff --git a/scispacy/hearst_patterns.py b/scispacy/hearst_patterns.py index 992b7a4..c28b16a 100644 --- a/scispacy/hearst_patterns.py +++ b/scispacy/hearst_patterns.py @@ -492,7 +492,7 @@ {"LEMMA": "which"}, {"LEMMA": "look"}, {"LEMMA": "like"}, - hyponym, + hypernym, ], "position": "last", }, diff --git a/scispacy/per_class_scorer.py b/scispacy/per_class_scorer.py index 57993b0..ade6af8 100644 --- a/scispacy/per_class_scorer.py +++ b/scispacy/per_class_scorer.py @@ -18,9 +18,9 @@ def __call__( gold_spans = copy.copy(gold_spans) predicted_spans = copy.copy(predicted_spans) untyped_gold_spans = {(x[0], x[1]) for x in gold_spans} - untyped_predicted_spans = {(x[0], x[1]) for x in predicted_spans} - for untyped_span, span in zip(untyped_predicted_spans, predicted_spans): + for span in predicted_spans: + untyped_span = (span[0], span[1]) if span in gold_spans: self._true_positives[span[2]] += 1 gold_spans.remove(span) diff --git a/scispacy/train_utils.py b/scispacy/train_utils.py index f02fd18..5ee826d 100644 --- a/scispacy/train_utils.py +++ b/scispacy/train_utils.py @@ -26,7 +26,8 @@ def evaluate_ner( metrics = scorer.get_metric() if dump_path is not None: - json.dump(metrics, open(dump_path, "a+")) + with open(dump_path, "a+") as metrics_file: + json.dump(metrics, metrics_file) for name, metric in metrics.items(): if "overall" in name or "untyped" in name or verbose: print(f"{name}: \t\t {metric}")