From de652e4d180e12c6187cde988b3f1db27d56d68e Mon Sep 17 00:00:00 2001 From: Juniper Bevensee Date: Thu, 18 Jun 2026 12:42:39 +1200 Subject: [PATCH] feat(fulltext): provider-neutral external resolver hook (opt-in, last resort) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an optional extension point to matilde_fetch_fulltext: when MATILDE_FULLTEXT_RESOLVER_URL (or a resolver_url arg) is set, an external full-text resolver is consulted — but ONLY after every legal open-access lookup (OpenAlex/Unpaywall/arXiv) has missed. A legal OA copy always wins, so a configured resolver is never hit when an open-access copy exists. The package ships no such service and names no provider. The operator supplies one out of band against the contract: GET {resolver_url}/resolve?doi= -> {"pdf_url": "..."} Honesty by construction: a resolver hit sets is_oa=false and source="external-resolver", and the tool message flags it as NOT open access. The public repo stays open-access-only and provider-neutral; any specific resolver (and its access/legal posture) lives entirely with the operator who configures the URL. Unset => behaviour never engages. Built test-first: 4 engine tests (used-on-miss, skipped-when-unset, never-consulted-when-OA-exists, miss-falls-through) + 2 handler tests (env forwarding, honest message). Full suite 93 passed, 6 skipped. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 17 +++++++++ matilde_plugin/engine/fulltext.py | 51 +++++++++++++++++++++++---- matilde_plugin/tools.py | 25 ++++++++++---- tests/test_fulltext.py | 57 +++++++++++++++++++++++++++++++ tests/test_fulltext_tool.py | 29 ++++++++++++++++ 5 files changed, 165 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 0baa27f..685da64 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,23 @@ providers (OpenAlex, Unpaywall, arXiv) are free. Optionally set `MATILDE_CONTACT_EMAIL` to join the providers' polite pools and enable the Unpaywall lookup (which widens open-access coverage). +### Optional: an external full-text resolver + +`matilde_fetch_fulltext` has a provider-neutral extension point. If you set +`MATILDE_FULLTEXT_RESOLVER_URL`, then **only when every open-access lookup has +missed**, Matilde will ask that endpoint for a full-text URL: + +``` +GET {MATILDE_FULLTEXT_RESOLVER_URL}/resolve?doi= -> {"pdf_url": "..."} +``` + +This package ships no such service and names no provider — you supply one out of +band (an institutional/library proxy, a licensed full-text API, a self-hosted +service). A result obtained this way is reported with `is_oa: false` and +`source: "external-resolver"`: it is full text, **not** open access, and you are +responsible for having the right to access it. Leave the variable unset and the +behaviour never engages. + ## Try it ```python diff --git a/matilde_plugin/engine/fulltext.py b/matilde_plugin/engine/fulltext.py index bedf683..225b3e8 100644 --- a/matilde_plugin/engine/fulltext.py +++ b/matilde_plugin/engine/fulltext.py @@ -106,6 +106,25 @@ def _arxiv_pdf(doi: str) -> str: return f"https://arxiv.org/pdf/{m.group(1)}" if m else "" +def _try_external_resolver(doi: str, fetch: FetchFn, resolver_url: str) -> str: + """Ask a configured external full-text resolver for a PDF URL. + + This is a provider-neutral extension point: ``resolver_url`` points at a + self-hosted or third-party service implementing the contract + ``GET {resolver_url}/resolve?doi={doi} -> {"pdf_url": "...", ...}``. This + package ships no such service and names no provider — the operator supplies + one out of band (an institutional proxy, a paid API, anything). Consulted + only after every legal open-access lookup has missed. + """ + base = resolver_url.rstrip("/") + doi_q = urllib.parse.quote(doi, safe="") + try: + data = fetch(f"{base}/resolve?doi={doi_q}") + except Exception: + return "" + return (data or {}).get("pdf_url") or "" + + # --------------------------------------------------------------------------- # Assembly # --------------------------------------------------------------------------- @@ -127,13 +146,20 @@ def _from_loc(doi: str, loc: dict, source: str) -> FullTextResult: def find_open_access(doi: str, fetch: Optional[FetchFn] = None, *, - email: Optional[str] = None) -> FullTextResult: - """Resolve the best legal open-access location for *doi*. - - Returns a :class:`FullTextResult`. ``is_oa=False`` means no open-access copy - was found — the work is paywalled and this locator will not route around it. - Pass *email* to enable the Unpaywall lookup (its API requires a contact - address); without it, only OpenAlex + arXiv are consulted. + email: Optional[str] = None, + resolver_url: Optional[str] = None) -> FullTextResult: + """Resolve the best full-text location for *doi*, preferring legal open access. + + Returns a :class:`FullTextResult`. Legal open-access sources are tried first + (OpenAlex, then Unpaywall if *email* is given, then an arXiv-DOI synthesis). + ``is_oa=False`` means no open-access copy was found. + + If — and only if — every OA lookup misses and *resolver_url* is configured, + a provider-neutral external resolver is consulted as a last resort. A hit + there populates the full-text URL but keeps ``is_oa=False`` and + ``source="external-resolver"``: it is full text, not open access, and the + result says so. A legal OA copy always wins, so a configured resolver is + never consulted when an open-access copy exists. """ fetch = fetch or default_fetch doi = _normalize_doi(doi) @@ -161,4 +187,15 @@ def find_open_access(doi: str, fetch: Optional[FetchFn] = None, *, return _from_loc(doi, {"pdf_url": arx, "oa_status": "green", "host_type": "repository"}, "arxiv") + # 4. External resolver (opt-in, last resort) — full text, NOT open access. + if resolver_url: + pdf = _try_external_resolver(doi, fetch, resolver_url) + if pdf: + result.pdf_url = pdf + result.best_url = pdf + result.source = "external-resolver" + result.candidates = [{"url": pdf, "pdf_url": pdf, "landing_url": "", + "license": "", "version": "", "host_type": "", + "source": "external-resolver"}] + return result diff --git a/matilde_plugin/tools.py b/matilde_plugin/tools.py index 2356427..83da41d 100644 --- a/matilde_plugin/tools.py +++ b/matilde_plugin/tools.py @@ -384,16 +384,20 @@ def _handle_openneuro_list_files(args: dict, **kwargs: Any) -> str: "Unpaywall, and arXiv. Use this to retrieve the actual paper content so a " "claim can be grounded against the source, not just its metadata. Returns " "is_oa, oa_status (gold/green/hybrid/bronze/closed), pdf_url, landing_url, " - "license, and the provider. If no open-access copy exists, is_oa is false " - "and no URL is returned — this tool only surfaces legal OA sources and will " - "not route around a paywall. Set MATILDE_CONTACT_EMAIL (or pass 'email') to " - "enable the Unpaywall lookup, which widens coverage." + "license, and the provider. If no open-access copy exists, is_oa is false. " + "Set MATILDE_CONTACT_EMAIL (or pass 'email') to enable the Unpaywall lookup, " + "which widens coverage. If an external full-text resolver is configured " + "(MATILDE_FULLTEXT_RESOLVER_URL or 'resolver_url'), it is consulted only as " + "a last resort after every open-access lookup misses; such a result is " + "flagged is_oa=false with source 'external-resolver' — it is full text, not " + "open access, so confirm you have the right to access it." ), "parameters": { "type": "object", "properties": { "doi": {"type": "string", "description": "DOI of the work (bare, or as a doi.org URL)."}, "email": {"type": "string", "description": "Contact email to enable the Unpaywall lookup (optional; falls back to MATILDE_CONTACT_EMAIL)."}, + "resolver_url": {"type": "string", "description": "Base URL of an external full-text resolver to use as a last resort (optional; falls back to MATILDE_FULLTEXT_RESOLVER_URL). Off unless configured."}, }, "required": ["doi"], }, @@ -410,15 +414,22 @@ def _handle_fetch_fulltext(args: dict, **kwargs: Any) -> str: email = (str(args.get("email", "")).strip() or os.environ.get("MATILDE_CONTACT_EMAIL", "").strip() or None) - res = find_open_access(doi, email=email) + resolver_url = (str(args.get("resolver_url", "")).strip() + or os.environ.get("MATILDE_FULLTEXT_RESOLVER_URL", "").strip() + or None) + res = find_open_access(doi, email=email, resolver_url=resolver_url) payload = res.to_dict() if res.is_oa: payload["message"] = (f"Open access ({res.oa_status}) via {res.source}: " f"{res.best_url}") + elif res.best_url: + # Full text via a configured external resolver — NOT open access. + payload["message"] = (f"Full text via {res.source} (NOT open access — " + f"verify you have the right to access it): " + f"{res.best_url}") else: payload["message"] = (f"No open-access copy found for {res.doi} " - f"(status: {res.oa_status}). Matilde returns only " - f"legal OA sources.") + f"(status: {res.oa_status}).") return _tool_result(payload) except Exception as exc: return _tool_error(f"fetch_fulltext failed: {type(exc).__name__}: {exc}") diff --git a/tests/test_fulltext.py b/tests/test_fulltext.py index b86861f..fd3f60a 100644 --- a/tests/test_fulltext.py +++ b/tests/test_fulltext.py @@ -168,6 +168,63 @@ def test_arxiv_doi_synthesizes_pdf_url_when_providers_miss(): assert res.source == "arxiv" +# --------------------------------------------------------------------------- +# External resolver hook (provider-neutral; opt-in via resolver_url) +# --------------------------------------------------------------------------- + +# A configured external resolver answers with a full-text URL for a paywalled +# work. Its OA status is unknown/none — the result must NOT claim open access. +EXTERNAL_HIT = {"pdf_url": "https://resolver.example.net/files/closed-paper.pdf"} + + +def test_external_resolver_used_when_oa_misses(): + fetch = make_fetch({ + "api.openalex.org": OPENALEX_CLOSED, + "resolver.example.net": EXTERNAL_HIT, + }) + res = find_open_access("10.1234/closed-paper", fetch=fetch, + resolver_url="https://resolver.example.net") + assert res.best_url == "https://resolver.example.net/files/closed-paper.pdf" + assert res.pdf_url == "https://resolver.example.net/files/closed-paper.pdf" + assert res.source == "external-resolver" + # Honesty: an external resolver is NOT open access — don't mislabel it. + assert res.is_oa is False + + +def test_external_resolver_skipped_when_unset(): + fetch = make_fetch({ + "api.openalex.org": OPENALEX_CLOSED, + "resolver.example.net": EXTERNAL_HIT, + }) + res = find_open_access("10.1234/closed-paper", fetch=fetch, resolver_url=None) + assert res.best_url == "" + assert not any("resolver.example.net" in u for u in fetch.calls) + + +def test_external_resolver_not_consulted_when_legal_oa_exists(): + # The legal OA copy wins; the external resolver (maybe Sci-Hub) is never hit. + fetch = make_fetch({ + "api.openalex.org": OPENALEX_OA, + "resolver.example.net": EXTERNAL_HIT, + }) + res = find_open_access("10.1234/oa-paper", fetch=fetch, + resolver_url="https://resolver.example.net") + assert res.source == "openalex" + assert res.is_oa is True + assert not any("resolver.example.net" in u for u in fetch.calls) + + +def test_external_resolver_miss_falls_through_to_closed(): + fetch = make_fetch({ + "api.openalex.org": OPENALEX_CLOSED, + "resolver.example.net": {"pdf_url": None}, + }) + res = find_open_access("10.1234/closed-paper", fetch=fetch, + resolver_url="https://resolver.example.net") + assert res.best_url == "" + assert res.is_oa is False + + # --------------------------------------------------------------------------- # DOI normalization # --------------------------------------------------------------------------- diff --git a/tests/test_fulltext_tool.py b/tests/test_fulltext_tool.py index 5755662..9c080c9 100644 --- a/tests/test_fulltext_tool.py +++ b/tests/test_fulltext_tool.py @@ -56,3 +56,32 @@ def boom(*a, **k): out = json.loads(_handle_fetch_fulltext({"doi": "10.1234/x"})) assert out["success"] is False assert "fetch_fulltext failed" in out["error"] + + +def test_handler_forwards_resolver_url_from_env(monkeypatch): + captured = {} + + def capture(doi, **kwargs): + captured.update(kwargs) + return ft.FullTextResult(doi=doi) + + monkeypatch.setattr(ft, "find_open_access", capture) + monkeypatch.setenv("MATILDE_FULLTEXT_RESOLVER_URL", "https://resolver.example.net") + + _handle_fetch_fulltext({"doi": "10.1234/x"}) + assert captured.get("resolver_url") == "https://resolver.example.net" + + +def test_handler_message_flags_external_as_not_open_access(monkeypatch): + fake = ft.FullTextResult( + doi="10.1234/x", is_oa=False, oa_status="closed", + best_url="https://resolver/x.pdf", pdf_url="https://resolver/x.pdf", + source="external-resolver", + ) + monkeypatch.setattr(ft, "find_open_access", lambda *a, **k: fake) + + out = json.loads(_handle_fetch_fulltext({"doi": "10.1234/x"})) + assert out["success"] is True + assert out["pdf_url"] == "https://resolver/x.pdf" + assert out["is_oa"] is False + assert "not open access" in out["message"].lower()