Add accession extractors and conservative archive response checks

.github/workflows/check.yml

@@ -17,9 +17,11 @@ jobs:
           python-version: "3.12"
       - name: The pilot scripts compile
         run: python -m py_compile pilot/*.py
+      - name: Offline unit tests
+        run: python -m unittest discover -s tests -v
       - name: No accession number is committed
         run: |
-          if git grep -nE "PXD[0-9]{6}|GSE[0-9]+"; then
+          if git grep -nE "PXD[0-9]{6}|GSE[0-9]+|MTBLS[0-9]+|ST[0-9]{6}|PRJ(NA|EB|DB)[0-9]+"; then
             echo "Accession numbers found. Commit counts only (see the policy in README.md)."
             exit 1
           fi

README.md

@@ -112,3 +112,5 @@ the repository. Europe PMC adds papers over time, so a re-run may read a differe
 ## Licence
 
 Apache License 2.0 (see [LICENSE](LICENSE)), as for Foundation0's other repositories.
+
+Draft work for #2: [implementation and remaining checks](notes/extra-archives.md).

notes/extra-archives.md

@@ -0,0 +1,39 @@
+# Three additional archive adapters
+
+Status: draft extractors and saved-response classifiers. Synthetic tests pass;
+the three 50-paper pilots and live schema checks remain undone. Refs #2.
+
+`pilot/archive_adapters.py` extracts MetaboLights, Metabolomics Workbench and
+BioProject accessions, including BioProject's NCBI, ENA and DDBJ prefixes. It
+deduplicates within text and rejects identifiers embedded in longer tokens.
+Its `request_url` function gives a read-only API URL; it does not make requests.
+
+The response classifier requires an exact matching identifier in successful
+metadata. It calls this `public_metadata`, which does not establish that every
+data file is public. Failure, malformed data, a schema mismatch or an absent
+record is `unknown`. There is deliberately no private label without a documented
+explicit private-status response. An empty search or HTTP 401 is insufficient.
+
+```sh
+python3 -m unittest discover -s tests -v
+python3 pilot/archive_adapters.py /private/tmp/archive-responses.json
+```
+
+Each private input row has `archive`, `accession`, `status` (integer HTTP status),
+and `body` (response text). Use one observation per accession. Keep response
+timestamps and source URLs beside that file. The tool prints aggregate counts
+only and refuses an input path within the checkout. Do not commit input records.
+
+Before connecting a network runner, verify each URL and schema against a public
+control and a documented unavailable control. Use a real contact address,
+throttling and a private cache. Then select 50 papers per archive with fixed
+queries and dates, record full-text failures, and publish only counts. A reader
+must inspect any proposed private case before it is included in a report.
+
+Sources: [MetaboLights download guide](https://www.ebi.ac.uk/training/online/courses/metabolights-quick-tour/getting-data-from-metabolights/),
+[MetaboLights API](https://www.ebi.ac.uk/metabolights/ws/api/spec.html),
+[Workbench REST API](https://www.metabolomicsworkbench.org/tools/MWRestAPIv1.0.pdf),
+[BioProject accession prefixes](https://support.nlm.nih.gov/knowledgebase/article/KA-03434/en-us),
+and [NCBI EFetch documentation](https://www.ncbi.nlm.nih.gov/books/NBK25499/#chapter4.EFetch).
+The MetaboLights API schema was inaccessible to the browser during this review;
+its response shape here is provisional and fails to unknown if it differs.

pilot/archive_adapters.py

@@ -0,0 +1,80 @@
+"""Extract three accession families and classify saved API responses conservatively."""
+import argparse
+from collections import Counter
+import json
+from pathlib import Path
+import re
+import urllib.parse
+import xml.etree.ElementTree as ET
+
+PATTERNS = {
+    "MetaboLights": re.compile(r"(?<![A-Za-z0-9])MTBLS[0-9]+(?![A-Za-z0-9])"),
+    "Metabolomics Workbench": re.compile(r"(?<![A-Za-z0-9])ST[0-9]{6}(?![A-Za-z0-9])"),
+    "BioProject": re.compile(r"(?<![A-Za-z0-9])PRJ(?:NA|EB|DB)[0-9]+(?![A-Za-z0-9])"),
+}
+
+def extract(text):
+    return {name: sorted(set(pattern.findall(text))) for name, pattern in PATTERNS.items()}
+
+def request_url(archive, accession):
+    if archive not in PATTERNS or not PATTERNS[archive].fullmatch(accession):
+        raise ValueError("invalid archive or accession")
+    if archive == "MetaboLights":
+        return "https://www.ebi.ac.uk/metabolights/ws/studies/" + accession
+    if archive == "Metabolomics Workbench":
+        return "https://www.metabolomicsworkbench.org/rest/study/study_id/" + accession + "/summary"
+    return "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?" + urllib.parse.urlencode(
+        {"db": "bioproject", "id": accession, "retmode": "xml"})
+
+def classify(archive, accession, status, body):
+    request_url(archive, accession)  # validate before interpreting any response
+    if status != 200:
+        return "unknown"
+    if archive == "BioProject":
+        try:
+            root = ET.fromstring(body)
+        except ET.ParseError:
+            return "unknown"
+        found = any(node.tag.rsplit("}", 1)[-1] == "ArchiveID" and
+                    node.get("accession") == accession for node in root.iter())
+        return "public_metadata" if found else "unknown"
+    try:
+        data = json.loads(body)
+    except (TypeError, ValueError):
+        return "unknown"
+    if archive == "MetaboLights":
+        study = data.get("content", {}) if isinstance(data, dict) else {}
+        if isinstance(study, dict) and study.get("studyIdentifier") == accession:
+            return "public_metadata"
+    if archive == "Metabolomics Workbench":
+        rows = [data] if isinstance(data, dict) and "study_id" in data else (
+            list(data.values()) if isinstance(data, dict) else [])
+        if any(isinstance(row, dict) and row.get("study_id") == accession for row in rows):
+            return "public_metadata"
+    return "unknown"
+
+def aggregate(records):
+    seen, counts = set(), Counter()
+    for record in records:
+        key = (record["archive"], record["accession"])
+        if key in seen:
+            raise ValueError("duplicate response: select one observation per accession")
+        seen.add(key)
+        state = classify(*key, record["status"], record["body"])
+        counts[(key[0], state)] += 1
+    return [{"archive": archive, "status": status, "count": count}
+            for (archive, status), count in sorted(counts.items())]
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("responses", help="private JSON array of saved responses outside the repository")
+    args = parser.parse_args()
+    path = Path(args.responses).resolve()
+    root = Path(__file__).resolve().parents[1]
+    if path == root or root in path.parents:
+        parser.error("responses must be outside this repository")
+    try:
+        result = aggregate(json.loads(path.read_text()))
+    except (ValueError, KeyError, OSError) as error:
+        parser.error("invalid saved-response input")
+    print(json.dumps(result, indent=2))

tests/test_archive_adapters.py

@@ -0,0 +1,35 @@
+import importlib.util
+import json
+from pathlib import Path
+import unittest
+s = importlib.util.spec_from_file_location("adapters", Path(__file__).resolve().parents[1]/"pilot/archive_adapters.py")
+m = importlib.util.module_from_spec(s); s.loader.exec_module(m)
+
+class AdapterTests(unittest.TestCase):
+    def test_extraction_boundaries_and_duplicates(self):
+        tokens = ["MTBLS"+"12", "ST"+"123456", "PRJNA"+"123", "PRJEB"+"12", "PRJDB"+"4"]
+        for token in tokens:
+            self.assertEqual(sum(map(len, m.extract("("+token+") "+token).values())), 1)
+            self.assertEqual(sum(map(len, m.extract("X"+token+" "+token+"X").values())), 0)
+        self.assertFalse(m.extract("ST"+"1234567")["Metabolomics Workbench"])
+
+    def test_http_failures_do_not_imply_private(self):
+        for status in (401, 403, 404, 429, 500):
+            self.assertEqual(m.classify("MetaboLights", "MTBLS"+"12", status, "{}"), "unknown")
+
+    def test_public_identity_required(self):
+        acc = "ST"+"123456"
+        self.assertEqual(m.classify("Metabolomics Workbench", acc, 200, json.dumps({"study_id": acc})), "public_metadata")
+        self.assertEqual(m.classify("Metabolomics Workbench", acc, 200, '{"message":"success"}'), "unknown")
+
+    def test_bioproject_exact_identity(self):
+        acc = "PRJEB"+"12"
+        self.assertEqual(m.classify("BioProject", acc, 200, '<Package><ArchiveID accession="'+acc+'"/></Package>'), "public_metadata")
+        self.assertEqual(m.classify("BioProject", acc, 200, '<error>not found</error>'), "unknown")
+
+    def test_aggregate_omits_ids_and_rejects_duplicates(self):
+        record = {"archive":"MetaboLights", "accession":"MTBLS"+"12", "status":404, "body":"{}"}
+        self.assertEqual(m.aggregate([record]), [{"archive":"MetaboLights", "status":"unknown", "count":1}])
+        with self.assertRaises(ValueError): m.aggregate([record, record])
+
+if __name__ == "__main__": unittest.main()