diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 3d3491b..6c2a52a 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -17,9 +17,11 @@ jobs: python-version: "3.12" - name: The pilot scripts compile run: python -m py_compile pilot/*.py + - name: Offline unit tests + run: python -m unittest discover -s tests -v - name: No accession number is committed run: | - if git grep -nE "PXD[0-9]{6}|GSE[0-9]+"; then + if git grep -nE "PXD[0-9]{6}|GSE[0-9]+|MTBLS[0-9]+|ST[0-9]{6}|PRJ(NA|EB|DB)[0-9]+"; then echo "Accession numbers found. Commit counts only (see the policy in README.md)." exit 1 fi diff --git a/README.md b/README.md index dd3466e..65d7d58 100644 --- a/README.md +++ b/README.md @@ -112,3 +112,5 @@ the repository. Europe PMC adds papers over time, so a re-run may read a differe ## Licence Apache License 2.0 (see [LICENSE](LICENSE)), as for Foundation0's other repositories. + +Draft work for #2: [implementation and remaining checks](notes/extra-archives.md). diff --git a/notes/extra-archives.md b/notes/extra-archives.md new file mode 100644 index 0000000..adadb13 --- /dev/null +++ b/notes/extra-archives.md @@ -0,0 +1,39 @@ +# Three additional archive adapters + +Status: draft extractors and saved-response classifiers. Synthetic tests pass; +the three 50-paper pilots and live schema checks remain undone. Refs #2. + +`pilot/archive_adapters.py` extracts MetaboLights, Metabolomics Workbench and +BioProject accessions, including BioProject's NCBI, ENA and DDBJ prefixes. It +deduplicates within text and rejects identifiers embedded in longer tokens. +Its `request_url` function gives a read-only API URL; it does not make requests. + +The response classifier requires an exact matching identifier in successful +metadata. It calls this `public_metadata`, which does not establish that every +data file is public. Failure, malformed data, a schema mismatch or an absent +record is `unknown`. There is deliberately no private label without a documented +explicit private-status response. An empty search or HTTP 401 is insufficient. + +```sh +python3 -m unittest discover -s tests -v +python3 pilot/archive_adapters.py /private/tmp/archive-responses.json +``` + +Each private input row has `archive`, `accession`, `status` (integer HTTP status), +and `body` (response text). Use one observation per accession. Keep response +timestamps and source URLs beside that file. The tool prints aggregate counts +only and refuses an input path within the checkout. Do not commit input records. + +Before connecting a network runner, verify each URL and schema against a public +control and a documented unavailable control. Use a real contact address, +throttling and a private cache. Then select 50 papers per archive with fixed +queries and dates, record full-text failures, and publish only counts. A reader +must inspect any proposed private case before it is included in a report. + +Sources: [MetaboLights download guide](https://www.ebi.ac.uk/training/online/courses/metabolights-quick-tour/getting-data-from-metabolights/), +[MetaboLights API](https://www.ebi.ac.uk/metabolights/ws/api/spec.html), +[Workbench REST API](https://www.metabolomicsworkbench.org/tools/MWRestAPIv1.0.pdf), +[BioProject accession prefixes](https://support.nlm.nih.gov/knowledgebase/article/KA-03434/en-us), +and [NCBI EFetch documentation](https://www.ncbi.nlm.nih.gov/books/NBK25499/#chapter4.EFetch). +The MetaboLights API schema was inaccessible to the browser during this review; +its response shape here is provisional and fails to unknown if it differs. diff --git a/pilot/archive_adapters.py b/pilot/archive_adapters.py new file mode 100644 index 0000000..00be487 --- /dev/null +++ b/pilot/archive_adapters.py @@ -0,0 +1,80 @@ +"""Extract three accession families and classify saved API responses conservatively.""" +import argparse +from collections import Counter +import json +from pathlib import Path +import re +import urllib.parse +import xml.etree.ElementTree as ET + +PATTERNS = { + "MetaboLights": re.compile(r"(?'), "public_metadata") + self.assertEqual(m.classify("BioProject", acc, 200, 'not found'), "unknown") + + def test_aggregate_omits_ids_and_rejects_duplicates(self): + record = {"archive":"MetaboLights", "accession":"MTBLS"+"12", "status":404, "body":"{}"} + self.assertEqual(m.aggregate([record]), [{"archive":"MetaboLights", "status":"unknown", "count":1}]) + with self.assertRaises(ValueError): m.aggregate([record, record]) + +if __name__ == "__main__": unittest.main()