diff --git a/README.md b/README.md index c74d108..0baa27f 100644 --- a/README.md +++ b/README.md @@ -55,10 +55,12 @@ score and an evidence trail, never a false certainty. | `matilde_openneuro_dataset_info` | Metadata for an OpenNeuro dataset (title, authors, modalities, subjects, tasks, size) | | `matilde_openneuro_search` | List OpenNeuro dataset IDs to discover brain-imaging datasets | | `matilde_openneuro_list_files` | List a dataset's files with sizes and direct download URLs | +| `matilde_fetch_fulltext` | Resolve the best **legal open-access** full text for a DOI (PDF or OA landing) via OpenAlex/Unpaywall/arXiv — for grounding a claim against the source, not just its metadata. Never routes around a paywall | -No API keys required — Crossref, OpenAlex, and DataCite are free and -unauthenticated. Optionally set `MATILDE_CONTACT_EMAIL` to join the providers' -polite pools. +No API keys required — Crossref, OpenAlex, DataCite, and the open-access +providers (OpenAlex, Unpaywall, arXiv) are free. Optionally set +`MATILDE_CONTACT_EMAIL` to join the providers' polite pools and enable the +Unpaywall lookup (which widens open-access coverage). ## Try it diff --git a/hermes-skill/SKILL.md b/hermes-skill/SKILL.md index 9ba7707..02e02d2 100644 --- a/hermes-skill/SKILL.md +++ b/hermes-skill/SKILL.md @@ -82,11 +82,26 @@ For a fast retraction-only check by DOI: matilde_check_retraction(doi="10.1016/...") ``` +To read the actual paper — to ground a claim against what the source really +says — resolve its **legal open-access** full text by DOI: + +``` +matilde_fetch_fulltext(doi="10.7554/eLife.00013") +``` + +It returns the best open-access location (a direct PDF where one exists, else an +OA landing page) from OpenAlex / Unpaywall / arXiv, with `is_oa`, `oa_status`, +and license. If no open-access copy exists, `is_oa` is false and no URL is +returned — it surfaces only legal OA sources and will not route around a +paywall. Set `MATILDE_CONTACT_EMAIL` to widen coverage via Unpaywall. When the +result gives a URL, fetch and read it with your general research/file tools. + **Interpreting axes honestly.** A `verified` verdict means *checked against authoritative metadata* — existence, identity, retraction status. It does **not** mean the cited passage actually supports the claim it is attached to. That deeper -check (claim-support grounding) is not yet automated; when it matters, read the -source and confirm the passage yourself, and say that you did. +check (claim-support grounding) is not yet automated; when it matters, pull the +source with `matilde_fetch_fulltext`, read the passage, confirm it yourself, and +say that you did. ### Phase 4: Synthesize diff --git a/matilde_plugin/__init__.py b/matilde_plugin/__init__.py index 524ea44..ecd3f98 100644 --- a/matilde_plugin/__init__.py +++ b/matilde_plugin/__init__.py @@ -54,6 +54,7 @@ OPENNEURO_INFO_SCHEMA = _tools_mod.OPENNEURO_INFO_SCHEMA OPENNEURO_SEARCH_SCHEMA = _tools_mod.OPENNEURO_SEARCH_SCHEMA OPENNEURO_FILES_SCHEMA = _tools_mod.OPENNEURO_FILES_SCHEMA +FETCH_FULLTEXT_SCHEMA = _tools_mod.FETCH_FULLTEXT_SCHEMA _check_available = _tools_mod._check_available _handle_verify_citation = _tools_mod._handle_verify_citation _handle_verify_bibliography = _tools_mod._handle_verify_bibliography @@ -61,6 +62,7 @@ _handle_openneuro_dataset_info = _tools_mod._handle_openneuro_dataset_info _handle_openneuro_search = _tools_mod._handle_openneuro_search _handle_openneuro_list_files = _tools_mod._handle_openneuro_list_files +_handle_fetch_fulltext = _tools_mod._handle_fetch_fulltext # --------------------------------------------------------------------------- # Tool registry — (tool_name, schema, handler, emoji) @@ -72,6 +74,7 @@ ("matilde_openneuro_dataset_info", OPENNEURO_INFO_SCHEMA, _handle_openneuro_dataset_info, "🧠"), ("matilde_openneuro_search", OPENNEURO_SEARCH_SCHEMA, _handle_openneuro_search, "🔎"), ("matilde_openneuro_list_files", OPENNEURO_FILES_SCHEMA, _handle_openneuro_list_files, "🗂"), + ("matilde_fetch_fulltext", FETCH_FULLTEXT_SCHEMA, _handle_fetch_fulltext, "📄"), ) diff --git a/matilde_plugin/engine/fulltext.py b/matilde_plugin/engine/fulltext.py new file mode 100644 index 0000000..bedf683 --- /dev/null +++ b/matilde_plugin/engine/fulltext.py @@ -0,0 +1,164 @@ +"""Open-access full-text locator. + +Given a DOI, resolve the best *legal* open-access location for the work — a +direct PDF where one exists, otherwise an OA landing page. Sources, in priority +order, are all free and unauthenticated (or email-gated, never key-gated): + + 1. **OpenAlex** — ``open_access`` + ``best_oa_location`` (no email needed). + 2. **Unpaywall** — queried only when a contact email is supplied (its API + requires ``?email=``). Often finds a publisher/repository OA copy OpenAlex + hasn't indexed yet. + 3. **arXiv synthesis** — an arXiv-registered DOI (``10.48550/arXiv.``) + always has a legal PDF at ``arxiv.org/pdf/``, even if the providers miss. + +This locator only ever returns open-access sources. A paywalled work resolves to +``is_oa=False`` with no URL — it deliberately does **not** route around a +paywall. The point is to give a citation-verifier the *content* it can legally +read to ground a claim, not to bypass access controls. + +Design mirrors ``citations``: all network I/O is injected via a ``fetch`` callable +``(url) -> parsed JSON dict`` (raising ``LookupError`` on 404), so every path is +unit-testable without a network. ``default_fetch`` is the stdlib production default. +""" +from __future__ import annotations + +import dataclasses +import re +import urllib.parse +from dataclasses import dataclass, field +from typing import Callable, Optional + +from .citations import OPENALEX_BASE, _normalize_doi, default_fetch + +FetchFn = Callable[..., dict] + +UNPAYWALL_BASE = "https://api.unpaywall.org/v2" +# arXiv-registered DOIs look like 10.48550/arXiv.1706.03762 (case-insensitive). +_ARXIV_DOI = re.compile(r"^10\.48550/arxiv\.(.+)$", re.I) + + +# --------------------------------------------------------------------------- +# Data model +# --------------------------------------------------------------------------- + +@dataclass +class FullTextResult: + """The resolved best open-access location for a DOI (or a closed verdict).""" + + doi: str + is_oa: bool = False + oa_status: str = "unknown" # gold | green | hybrid | bronze | closed | unknown + best_url: str = "" # best PDF if available, else best landing page + pdf_url: str = "" + landing_url: str = "" + source: str = "" # which provider produced the chosen location + license: str = "" + candidates: list = field(default_factory=list) + + def to_dict(self) -> dict: + return dataclasses.asdict(self) + + +# --------------------------------------------------------------------------- +# Provider parsers (each returns a normalized loc dict, or None on miss) +# --------------------------------------------------------------------------- + +def _try_openalex(doi: str, fetch: FetchFn) -> Optional[dict]: + try: + work = fetch(f"{OPENALEX_BASE}/works/doi:{doi}") + except Exception: + return None + oa = work.get("open_access") or {} + best = work.get("best_oa_location") or {} + # Only ``pdf_url`` is a guaranteed direct PDF. ``oa_url`` is the "best OA URL" + # but is often a landing page — treat it as a landing fallback, never a PDF. + return { + "is_oa": bool(oa.get("is_oa")), + "oa_status": oa.get("oa_status") or "", + "pdf_url": best.get("pdf_url") or "", + "landing_url": best.get("landing_page_url") or oa.get("oa_url") or "", + "license": best.get("license") or "", + "version": best.get("version") or "", + "host_type": (best.get("source") or {}).get("type") or "", + } + + +def _try_unpaywall(doi: str, fetch: FetchFn, email: str) -> Optional[dict]: + email_q = urllib.parse.quote(email) + try: + data = fetch(f"{UNPAYWALL_BASE}/{doi}?email={email_q}") + except Exception: + return None + best = data.get("best_oa_location") or {} + return { + "is_oa": bool(data.get("is_oa")), + "oa_status": data.get("oa_status") or "", + "pdf_url": best.get("url_for_pdf") or "", + "landing_url": best.get("url") or "", + "license": best.get("license") or "", + "version": best.get("version") or "", + "host_type": best.get("host_type") or "", + } + + +def _arxiv_pdf(doi: str) -> str: + m = _ARXIV_DOI.match(doi) + return f"https://arxiv.org/pdf/{m.group(1)}" if m else "" + + +# --------------------------------------------------------------------------- +# Assembly +# --------------------------------------------------------------------------- + +def _from_loc(doi: str, loc: dict, source: str) -> FullTextResult: + pdf = loc.get("pdf_url") or "" + landing = loc.get("landing_url") or "" + best = pdf or landing + candidate = { + "url": best, "pdf_url": pdf, "landing_url": landing, + "license": loc.get("license") or "", "version": loc.get("version") or "", + "host_type": loc.get("host_type") or "", "source": source, + } + return FullTextResult( + doi=doi, is_oa=True, oa_status=loc.get("oa_status") or "unknown", + best_url=best, pdf_url=pdf, landing_url=landing, source=source, + license=loc.get("license") or "", candidates=[candidate], + ) + + +def find_open_access(doi: str, fetch: Optional[FetchFn] = None, *, + email: Optional[str] = None) -> FullTextResult: + """Resolve the best legal open-access location for *doi*. + + Returns a :class:`FullTextResult`. ``is_oa=False`` means no open-access copy + was found — the work is paywalled and this locator will not route around it. + Pass *email* to enable the Unpaywall lookup (its API requires a contact + address); without it, only OpenAlex + arXiv are consulted. + """ + fetch = fetch or default_fetch + doi = _normalize_doi(doi) + result = FullTextResult(doi=doi) + if not doi: + return result + + # 1. OpenAlex (primary, no email needed) + oa = _try_openalex(doi, fetch) + if oa and oa["is_oa"] and (oa["pdf_url"] or oa["landing_url"]): + return _from_loc(doi, oa, "openalex") + if oa is not None: + result.is_oa = oa["is_oa"] + result.oa_status = oa["oa_status"] or "closed" + + # 2. Unpaywall (only with a contact email) + if email: + up = _try_unpaywall(doi, fetch, email) + if up and up["is_oa"] and (up["pdf_url"] or up["landing_url"]): + return _from_loc(doi, up, "unpaywall") + + # 3. arXiv synthesis — a registered arXiv DOI always has a legal PDF + arx = _arxiv_pdf(doi) + if arx: + return _from_loc(doi, {"pdf_url": arx, "oa_status": "green", + "host_type": "repository"}, "arxiv") + + return result diff --git a/matilde_plugin/plugin.yaml b/matilde_plugin/plugin.yaml index 7af2f8d..c4af2c6 100644 --- a/matilde_plugin/plugin.yaml +++ b/matilde_plugin/plugin.yaml @@ -11,7 +11,7 @@ name: "matilde" version: "0.1.0" -description: "Matilde — academic research assistant: verifiable citations (Crossref/OpenAlex/DataCite + retraction) and OpenNeuro/BIDS dataset discovery." +description: "Matilde — academic research assistant: verifiable citations (Crossref/OpenAlex/DataCite + retraction), legal open-access full-text retrieval (OpenAlex/Unpaywall/arXiv), and OpenNeuro/BIDS dataset discovery." author: "NimbleCoAI" @@ -28,3 +28,4 @@ provides_tools: - "matilde_openneuro_dataset_info" - "matilde_openneuro_search" - "matilde_openneuro_list_files" + - "matilde_fetch_fulltext" diff --git a/matilde_plugin/tools.py b/matilde_plugin/tools.py index 2c39120..2356427 100644 --- a/matilde_plugin/tools.py +++ b/matilde_plugin/tools.py @@ -370,3 +370,55 @@ def _handle_openneuro_list_files(args: dict, **kwargs: Any) -> str: ) except Exception as exc: return _tool_error(f"openneuro_list_files failed: {type(exc).__name__}: {exc}") + + +# --------------------------------------------------------------------------- +# Tool: open-access full-text locator +# --------------------------------------------------------------------------- + +FETCH_FULLTEXT_SCHEMA = { + "name": "matilde_fetch_fulltext", + "description": ( + "Find the best *legal open-access* full-text location for a paper by DOI " + "— a direct PDF where one exists, else an OA landing page — using OpenAlex, " + "Unpaywall, and arXiv. Use this to retrieve the actual paper content so a " + "claim can be grounded against the source, not just its metadata. Returns " + "is_oa, oa_status (gold/green/hybrid/bronze/closed), pdf_url, landing_url, " + "license, and the provider. If no open-access copy exists, is_oa is false " + "and no URL is returned — this tool only surfaces legal OA sources and will " + "not route around a paywall. Set MATILDE_CONTACT_EMAIL (or pass 'email') to " + "enable the Unpaywall lookup, which widens coverage." + ), + "parameters": { + "type": "object", + "properties": { + "doi": {"type": "string", "description": "DOI of the work (bare, or as a doi.org URL)."}, + "email": {"type": "string", "description": "Contact email to enable the Unpaywall lookup (optional; falls back to MATILDE_CONTACT_EMAIL)."}, + }, + "required": ["doi"], + }, +} + + +def _handle_fetch_fulltext(args: dict, **kwargs: Any) -> str: + doi = str(args.get("doi", "")).strip() + if not doi: + return _tool_error("'doi' is required (bare or doi.org URL).") + try: + import os + from .engine.fulltext import find_open_access + email = (str(args.get("email", "")).strip() + or os.environ.get("MATILDE_CONTACT_EMAIL", "").strip() + or None) + res = find_open_access(doi, email=email) + payload = res.to_dict() + if res.is_oa: + payload["message"] = (f"Open access ({res.oa_status}) via {res.source}: " + f"{res.best_url}") + else: + payload["message"] = (f"No open-access copy found for {res.doi} " + f"(status: {res.oa_status}). Matilde returns only " + f"legal OA sources.") + return _tool_result(payload) + except Exception as exc: + return _tool_error(f"fetch_fulltext failed: {type(exc).__name__}: {exc}") diff --git a/tests/test_fulltext.py b/tests/test_fulltext.py new file mode 100644 index 0000000..b86861f --- /dev/null +++ b/tests/test_fulltext.py @@ -0,0 +1,187 @@ +"""Tests for the open-access full-text locator (engine/fulltext.py). + +Like the citations tests, all network I/O is injected — a fake ``fetch`` maps +request URLs to canned JSON, so the suite is deterministic and offline. The +locator resolves the best *legal* open-access location for a DOI via OpenAlex +(no key) + Unpaywall (needs a contact email) + an arXiv synthesis fallback. It +never returns a paywalled/pirated source — closed access returns is_oa=False. +""" +from __future__ import annotations + +import json +import os +import sys + +import pytest + +sys.path.insert(0, os.path.normpath(os.path.join(os.path.dirname(__file__), ".."))) + +from matilde_plugin.engine.fulltext import ( # noqa: E402 + FullTextResult, + find_open_access, +) + + +# --------------------------------------------------------------------------- +# Canned provider responses +# --------------------------------------------------------------------------- + +# OpenAlex work that is green-OA with a repository PDF. +OPENALEX_OA = { + "doi": "https://doi.org/10.1234/oa-paper", + "open_access": {"is_oa": True, "oa_status": "green", + "oa_url": "https://repo.example.org/oa-paper.pdf"}, + "best_oa_location": { + "pdf_url": "https://repo.example.org/oa-paper.pdf", + "landing_page_url": "https://repo.example.org/oa-paper", + "license": "cc-by", + "version": "publishedVersion", + "source": {"type": "repository"}, + }, +} + +# OpenAlex work with NO open-access location (paywalled). +OPENALEX_CLOSED = { + "doi": "https://doi.org/10.1234/closed-paper", + "open_access": {"is_oa": False, "oa_status": "closed", "oa_url": None}, + "best_oa_location": None, +} + +# Unpaywall says a paper OpenAlex thinks is closed actually has a gold PDF. +UNPAYWALL_OA = { + "doi": "10.1234/closed-paper", + "is_oa": True, + "oa_status": "gold", + "best_oa_location": { + "url": "https://publisher.example.com/closed-paper", + "url_for_pdf": "https://publisher.example.com/closed-paper.pdf", + "license": "cc-by", + "version": "publishedVersion", + "host_type": "publisher", + }, +} + + +def make_fetch(routes: dict): + """Return a fake fetch that maps a substring -> canned JSON (LookupError if + no route matches, mirroring default_fetch's 404 behavior).""" + calls = [] + + def _fetch(url, timeout=20.0): + calls.append(url) + for needle, payload in routes.items(): + if needle in url: + return payload + raise LookupError(url) + + _fetch.calls = calls + return _fetch + + +# --------------------------------------------------------------------------- +# OpenAlex (primary, no email needed) +# --------------------------------------------------------------------------- + +def test_openalex_oa_with_pdf_is_resolved(): + fetch = make_fetch({"api.openalex.org": OPENALEX_OA}) + res = find_open_access("10.1234/oa-paper", fetch=fetch) + assert isinstance(res, FullTextResult) + assert res.is_oa is True + assert res.pdf_url == "https://repo.example.org/oa-paper.pdf" + assert res.best_url == "https://repo.example.org/oa-paper.pdf" + assert res.oa_status == "green" + assert res.source == "openalex" + assert res.license == "cc-by" + + +def test_oa_without_direct_pdf_does_not_mislabel_landing_as_pdf(): + # OA work whose best_oa_location has NO pdf_url — only an oa_url landing page. + # pdf_url must stay empty (don't claim a landing page is a PDF); best_url + # falls back to the landing page so the agent still gets somewhere legal. + work = { + "doi": "https://doi.org/10.7554/elife.x", + "open_access": {"is_oa": True, "oa_status": "gold", + "oa_url": "https://elifesciences.org/articles/x"}, + "best_oa_location": { + "pdf_url": None, + "landing_page_url": "https://elifesciences.org/articles/x", + "license": "cc-by", "version": "publishedVersion", + "source": {"type": "publisher"}, + }, + } + fetch = make_fetch({"api.openalex.org": work}) + res = find_open_access("10.7554/elife.x", fetch=fetch) + assert res.is_oa is True + assert res.pdf_url == "" + assert res.landing_url == "https://elifesciences.org/articles/x" + assert res.best_url == "https://elifesciences.org/articles/x" + + +def test_closed_access_returns_not_oa(): + fetch = make_fetch({"api.openalex.org": OPENALEX_CLOSED}) + res = find_open_access("10.1234/closed-paper", fetch=fetch) + assert res.is_oa is False + assert res.best_url == "" + assert res.pdf_url == "" + assert res.oa_status == "closed" + assert res.candidates == [] + + +# --------------------------------------------------------------------------- +# Unpaywall fallback (needs email) +# --------------------------------------------------------------------------- + +def test_unpaywall_fills_gap_when_openalex_closed(): + fetch = make_fetch({ + "api.openalex.org": OPENALEX_CLOSED, + "api.unpaywall.org": UNPAYWALL_OA, + }) + res = find_open_access("10.1234/closed-paper", fetch=fetch, + email="researcher@example.com") + assert res.is_oa is True + assert res.pdf_url == "https://publisher.example.com/closed-paper.pdf" + assert res.source == "unpaywall" + assert res.oa_status == "gold" + + +def test_unpaywall_skipped_without_email(): + # No email => Unpaywall must not be queried (its API requires one). + fetch = make_fetch({ + "api.openalex.org": OPENALEX_CLOSED, + "api.unpaywall.org": UNPAYWALL_OA, + }) + res = find_open_access("10.1234/closed-paper", fetch=fetch, email=None) + assert res.is_oa is False + assert not any("unpaywall" in u for u in fetch.calls) + + +# --------------------------------------------------------------------------- +# arXiv synthesis fallback +# --------------------------------------------------------------------------- + +def test_arxiv_doi_synthesizes_pdf_url_when_providers_miss(): + # OpenAlex 404s on this DOI; the arXiv DOI shape still yields a legal PDF. + fetch = make_fetch({}) # everything LookupErrors + res = find_open_access("10.48550/arXiv.1706.03762", fetch=fetch) + assert res.is_oa is True + assert res.pdf_url == "https://arxiv.org/pdf/1706.03762" + assert res.source == "arxiv" + + +# --------------------------------------------------------------------------- +# DOI normalization +# --------------------------------------------------------------------------- + +def test_doi_url_is_normalized_before_query(): + fetch = make_fetch({"api.openalex.org": OPENALEX_OA}) + find_open_access("https://doi.org/10.1234/oa-paper", fetch=fetch) + assert any("doi:10.1234/oa-paper" in u for u in fetch.calls) + + +def test_to_dict_is_json_serializable(): + fetch = make_fetch({"api.openalex.org": OPENALEX_OA}) + res = find_open_access("10.1234/oa-paper", fetch=fetch) + d = res.to_dict() + json.dumps(d) # must not raise + assert d["is_oa"] is True + assert d["pdf_url"] == "https://repo.example.org/oa-paper.pdf" diff --git a/tests/test_fulltext_tool.py b/tests/test_fulltext_tool.py new file mode 100644 index 0000000..5755662 --- /dev/null +++ b/tests/test_fulltext_tool.py @@ -0,0 +1,58 @@ +"""Tests for the matilde_fetch_fulltext tool handler (envelope + validation). + +The locator engine itself is covered in test_fulltext.py with injected I/O. +Here we test the thin handler: it validates input, calls the engine, and wraps +the result in the standard ``{success, ...}`` JSON envelope, never raising. +""" +from __future__ import annotations + +import json +import os +import sys + +sys.path.insert(0, os.path.normpath(os.path.join(os.path.dirname(__file__), ".."))) + +import matilde_plugin.engine.fulltext as ft # noqa: E402 +from matilde_plugin.tools import _handle_fetch_fulltext # noqa: E402 + + +def test_handler_requires_doi(): + out = json.loads(_handle_fetch_fulltext({})) + assert out["success"] is False + assert "doi" in out["error"].lower() + + +def test_handler_envelopes_open_access_result(monkeypatch): + fake = ft.FullTextResult( + doi="10.1234/x", is_oa=True, oa_status="green", + best_url="https://repo/x.pdf", pdf_url="https://repo/x.pdf", + source="openalex", license="cc-by", + ) + monkeypatch.setattr(ft, "find_open_access", lambda *a, **k: fake) + + out = json.loads(_handle_fetch_fulltext({"doi": "10.1234/x"})) + assert out["success"] is True + assert out["is_oa"] is True + assert out["pdf_url"] == "https://repo/x.pdf" + assert out["source"] == "openalex" + assert "message" in out + + +def test_handler_reports_closed_access_plainly(monkeypatch): + fake = ft.FullTextResult(doi="10.1234/closed", is_oa=False, oa_status="closed") + monkeypatch.setattr(ft, "find_open_access", lambda *a, **k: fake) + + out = json.loads(_handle_fetch_fulltext({"doi": "10.1234/closed"})) + assert out["success"] is True # a clean "no OA copy" is a successful answer + assert out["is_oa"] is False + assert out["pdf_url"] == "" + + +def test_handler_never_raises_on_engine_error(monkeypatch): + def boom(*a, **k): + raise RuntimeError("network down") + monkeypatch.setattr(ft, "find_open_access", boom) + + out = json.loads(_handle_fetch_fulltext({"doi": "10.1234/x"})) + assert out["success"] is False + assert "fetch_fulltext failed" in out["error"] diff --git a/tests/test_plugin_tools.py b/tests/test_plugin_tools.py index 26a6167..1482974 100644 --- a/tests/test_plugin_tools.py +++ b/tests/test_plugin_tools.py @@ -34,6 +34,7 @@ def test_plugin_registers_expected_tools(): "matilde_openneuro_dataset_info", "matilde_openneuro_search", "matilde_openneuro_list_files", + "matilde_fetch_fulltext", ]