diff --git a/pyproject.toml b/pyproject.toml index fa734d1..dfd7fd4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,13 @@ build-backend = "setuptools.build_meta" where = ["src"] [tool.setuptools.package-data] -scansci_pdf = ["data/*.json", "data/*.dat", "_core/*.pyd", "_core/*.so"] +scansci_pdf = [ + "data/*.json", + "data/*.dat", + "templates/*.html", + "_core/*.pyd", + "_core/*.so", +] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/skill/SKILL.md b/skill/SKILL.md index 3c8c7ff..2a92d64 100644 --- a/skill/SKILL.md +++ b/skill/SKILL.md @@ -26,7 +26,7 @@ scansci-pdf 是一个 MCP 服务器，提供 21 个工具，覆盖学术论文 |------|----------|------| | 按 DOI/arXiv ID 下载单篇论文 | `scansci_pdf_download` | 支持 5 种下载策略（fastest/scihub_only/...） | | 批量下载多篇论文 | `scansci_pdf_batch_download` | 并发下载，默认 10 线程 | -| 关键词/作者搜索论文 | `scansci_pdf_search` | 基于 OpenAlex，支持关键词、作者名、作者ID | +| 高级文献检索 | `scansci_pdf_search` | 10 个学术 API，支持布尔/字段表达式、结构化过滤与检索 provenance | | 导出引文 | `scansci_pdf_citation` | BibTeX / RIS / EndNote 三种格式 | | 导入 .bib 文件并下载 | `scansci_pdf_import_bib` | 自动提取 DOI 并批量下载 | | 推送到 Zotero | `scansci_pdf_zotero_push` | 需先下载论文到缓存 | @@ -82,21 +82,81 @@ scansci-pdf 是一个 MCP 服务器，提供 21 个工具，覆盖学术论文 | 工具 | 描述 | 关键参数 | |------|------|----------| -| `scansci_pdf_search` | 关键词/作者搜索论文（OpenAlex） | `query`（关键词搜索）、`author`（按作者名）、`author_id`（按作者ID）、`limit`（默认 10）、`year_from`、`year_to`、`sort` | +| `scansci_pdf_search` | 10 个学术 API 的可审计检索，支持高级布尔/字段表达式 | `query`、`sources`、`query_mode`、`limit`、`offset`、日期/类型/OA/被引等过滤 | | `scansci_pdf_parse_list` | 解析论文列表文件 | `file_path`（必需，.md/.txt/.bib） | -**参数约束：** -- `query`: 关键词搜索。留空时可配合 `author` 或 `author_id` 按作者搜索 -- `author`: 作者名（如 "Fang Jingyun"），自动解析为 OpenAlex 作者 ID，支持中英文姓名顺序互换 -- `author_id`: OpenAlex 作者 ID（如 "A5102961214"），跳过姓名解析直接搜索 -- `sort`: `"cited_by_count"`（被引最多）、`"publication_date"`（最新）、省略为相关性排序 -- `year_from` / `year_to`: 整数年份，如 `2020` +**统一高级检索式：** + +- 布尔与分组：`AND`、`OR`、`NOT`、括号、双引号短语、`*` 通配符。 +- 通用字段：`title:`、`author:`、`abstract:`、`journal:`、`doi:`、`category:`、`mesh:`、`fulltext:`。 +- 示例：`title:"climate change" AND (author:Smith OR mesh:"global warming") NOT review`。 +- 只选择单一来源时可传入该来源的原生检索式。跨库时，服务会翻译通用字段；无法精确映射的字段会降级为文本搜索，并在 `retrieval.warnings` 中明确报告。 + +**参数说明：** + +| 参数 | 说明 | +|------|------| +| `query` | 普通关键词、DOI、arXiv ID 或高级表达式。仅作者检索或 bioRxiv/medRxiv 日期浏览时可留空 | +| `sources` | 可选子集：`pubmed`、`pmc`、`biorxiv`、`medrxiv`、`arxiv`、`openalex`、`crossref`、`semantic_scholar`、`core`、`unpaywall`；`["all"]` 仅用于明确要求的广泛检索 | +| `query_mode` | `auto`（默认自动识别）、`plain` 或 `advanced` | +| `exact` | 在来源支持时请求精确匹配 | +| `limit` / `offset` | 最终返回 1-100 条；offset 是各来源内部的零基分页位置，不是合并结果的全局偏移；page/token 来源会返回 warning | +| `year_from` / `year_to` | 兼容整数年份过滤 | +| `date_from` / `date_to` | 更精确的 ISO 日期 `YYYY-MM-DD` | +| `sort` | `relevance`、`publication_date`、`updated_date`、`cited_by_count` | +| `author` / `author_id` | 作者名或精确 OpenAlex Author ID | +| `publication_types` | 来源原生文献类型列表，如 `Review`、`JournalArticle`、`journal-article`、`article` | +| `fields_of_study` | Semantic Scholar 学科列表；其他库按主题文本映射 | +| `venue` / `category` | 期刊/会议；或 arXiv、bioRxiv、medRxiv 类别 | +| `open_access_only` | 仅保留已确认 OA 的结果 | +| `has_abstract` | `true` 要求摘要，`false` 排除带摘要记录，省略则不限 | +| `min_citations` / `language` | 最低已知被引数；语言代码/名称 | +| `recent_days` | 1-365，用于 PubMed 或 bioRxiv/medRxiv 最近记录检索；其他来源会提示改用日期范围 | +| `enrich_open_access` | 用 Unpaywall 串行补充前 10 个 DOI；必须先把配置 `email` 设为真实邮箱 | + +**自动路由：** + +- DOI：Crossref + Semantic Scholar + Unpaywall。 +- arXiv ID：arXiv + Semantic Scholar + OpenAlex。 +- 一般主题：PubMed + OpenAlex + Semantic Scholar + Crossref。 +- bioRxiv/medRxiv 没有关键词 API；主题检索应走 OpenAlex/Semantic Scholar，直接 API 只支持 DOI、日期范围、最近天数和 category。 +- Unpaywall 是 DOI 查询/OA 丰富接口，不是可靠的关键词发现库。 +- PMC 是全文库；PubMed 主要提供题录与摘要元数据。 **返回值（search）：** + ```json -{"results": [{"title": "...", "doi": "...", "authors": [...], "year": 2024, "cited_by_count": 42, "abstract": "..."}]} +{ + "results": [ + { + "title": "...", + "identifier": "...", + "doi": "...", + "authors": ["..."], + "year": 2024, + "cited_by_count": 42, + "is_oa": true, + "source": "openalex+unpaywall" + } + ], + "retrieval": { + "query_mode": "advanced", + "scope": "bounded_search", + "sources_requested": ["openalex", "pubmed"], + "access_date": "2026-07-19", + "total_by_source": {"openalex": 120, "pubmed": 85}, + "warnings": [], + "errors": {}, + "partial": false, + "source_provenance": [ + {"source": "openalex", "endpoint": "...", "parameters": {}, "retrieved": 10} + ] + } +} ``` +所有 API key/token 在 provenance 与配置读取中均脱敏。空结果、来源错误、参数降级或分页限制不会静默隐藏。 + ### 引文管理 | 工具 | 描述 | 关键参数 | diff --git a/src/scansci_pdf/__init__.py b/src/scansci_pdf/__init__.py index b6c0152..7ca465d 100644 --- a/src/scansci_pdf/__init__.py +++ b/src/scansci_pdf/__init__.py @@ -52,6 +52,7 @@ def _patched_load_default_certs(self, *args, **kwargs): "download", "batch_download", "search_papers", + "search_papers_detailed", "load_config", "update_config", "get_config_safe", @@ -59,4 +60,5 @@ def _patched_load_default_certs(self, *args, **kwargs): from .sources import download, batch_download from .search import search_papers +from .advanced_search import search_papers_detailed from .config import load_config, update_config, get_config_safe diff --git a/src/scansci_pdf/advanced_search.py b/src/scansci_pdf/advanced_search.py new file mode 100644 index 0000000..4a3a80d --- /dev/null +++ b/src/scansci_pdf/advanced_search.py @@ -0,0 +1,246 @@ +"""Advanced multi-database literature search orchestration.""" + +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime, timezone +from typing import Any, Callable + +from .config import load_config +from .search_provider_arxiv import search_arxiv +from .search_providers_access import ( + enrich_unpaywall, + search_core, + search_unpaywall, +) +from .search_providers_biomedical import ( + search_biorxiv, + search_medrxiv, + search_pmc, + search_pubmed, +) +from .search_providers_index import ( + search_crossref, + search_openalex, + search_semantic_scholar, +) +from .search_query import ( + SearchSpec, + SourceReport, + deduplicate_results, + sort_results, +) + +Provider = Callable[[SearchSpec, dict[str, Any]], SourceReport] + +_PROVIDERS: dict[str, Provider] = { + "pubmed": search_pubmed, + "pmc": search_pmc, + "biorxiv": search_biorxiv, + "medrxiv": search_medrxiv, + "arxiv": search_arxiv, + "openalex": search_openalex, + "crossref": search_crossref, + "semantic_scholar": search_semantic_scholar, + "core": search_core, + "unpaywall": search_unpaywall, +} + + +def _route_sources(spec: SearchSpec) -> list[str]: + if spec.sources: + return spec.sources + identifier_type, _ = spec.identifier + if identifier_type == "doi": + return ["crossref", "semantic_scholar", "unpaywall"] + if identifier_type == "arxiv": + return ["arxiv", "semantic_scholar", "openalex"] + # Broad default follows paper-lookup's comprehensive-search contract. + return ["pubmed", "openalex", "semantic_scholar", "crossref"] + + +def _validate_retrieval(spec: SearchSpec, sources: list[str]) -> None: + if spec.effective_query: + return + if spec.author_id: + return + browse_sources = {"biorxiv", "medrxiv"} + if ( + set(sources).issubset(browse_sources) + and (spec.date_from or spec.date_to or spec.recent_days) + ): + return + raise ValueError( + "query is required unless author/author_id is set, or bioRxiv/medRxiv " + "is browsed with date_from/date_to/recent_days" + ) + + +def search_papers_detailed( + query: str = "", + limit: int = 10, + year_from: int | None = None, + year_to: int | None = None, + sort: str | None = None, + *, + sources: list[str] | str | None = None, + query_mode: str = "auto", + exact: bool = False, + offset: int = 0, + date_from: str | None = None, + date_to: str | None = None, + author: str | None = None, + author_id: str | None = None, + publication_types: list[str] | str | None = None, + fields_of_study: list[str] | str | None = None, + venue: str | None = None, + category: str | None = None, + open_access_only: bool = False, + has_abstract: bool | None = None, + min_citations: int | None = None, + language: str | None = None, + recent_days: int | None = None, + enrich_open_access: bool = False, + config: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Run a bounded, auditable search across selected literature databases.""" + spec = SearchSpec( + query=query, + sources=sources, # type: ignore[arg-type] + query_mode=query_mode, + exact=exact, + limit=limit, + offset=offset, + year_from=year_from, + year_to=year_to, + date_from=date_from, + date_to=date_to, + sort=sort or "relevance", + author=author, + author_id=author_id, + publication_types=publication_types, # type: ignore[arg-type] + fields_of_study=fields_of_study, # type: ignore[arg-type] + venue=venue, + category=category, + open_access_only=open_access_only, + has_abstract=has_abstract, + min_citations=min_citations, + language=language, + recent_days=recent_days, + enrich_open_access=enrich_open_access, + ) + selected_sources = _route_sources(spec) + _validate_retrieval(spec, selected_sources) + runtime_config = config if config is not None else load_config() + + reports_by_source: dict[str, SourceReport] = {} + worker_count = max(1, min(4, len(selected_sources))) + with ThreadPoolExecutor( + max_workers=worker_count, + thread_name_prefix="scansci-search", + ) as pool: + futures = { + pool.submit(_PROVIDERS[source], spec, runtime_config): source + for source in selected_sources + } + for future in as_completed(futures): + source = futures[future] + try: + reports_by_source[source] = future.result() + except Exception as exc: + reports_by_source[source] = SourceReport( + source=source, + error=f"{type(exc).__name__}: {exc}", + ) + + reports = [ + reports_by_source.get(source, SourceReport(source=source)) + for source in selected_sources + ] + raw_results = [ + result + for report in reports + for result in report.results + ] + + needs_enrichment = ( + spec.enrich_open_access or spec.open_access_only + ) and spec.identifier[0] != "doi" + if needs_enrichment: + dois = [result.get("doi", "") for result in raw_results] + oa_report = enrich_unpaywall(dois, runtime_config, max_items=min(10, spec.limit)) + reports.append(oa_report) + raw_results.extend(oa_report.results) + + merged = deduplicate_results(raw_results) + if spec.open_access_only: + merged = [item for item in merged if item.get("is_oa")] + if spec.has_abstract is not None: + merged = [ + item for item in merged + if ( + item.get("has_abstract") + if item.get("has_abstract") is not None + else bool(item.get("abstract")) + ) is spec.has_abstract + ] + if spec.min_citations is not None: + merged = [ + item for item in merged + if (item.get("cited_by_count") or 0) >= spec.min_citations + ] + sort_results(merged, spec.sort) + merged = merged[:spec.limit] + + warnings = [ + f"{report.source}: {warning}" + for report in reports + for warning in report.warnings + ] + errors = { + report.source: report.error + for report in reports + if report.error + } + source_provenance = [report.provenance() for report in reports] + retrieved_by_source = { + report.source: len(report.results) for report in reports + } + total_by_source = { + report.source: report.total for report in reports + if report.total is not None + } + access_date = datetime.now(timezone.utc).date().isoformat() + + return { + "results": merged, + "retrieval": { + "query": spec.query, + "effective_query": spec.effective_query, + "query_mode": "advanced" if spec.advanced else "plain", + "scope": ( + "targeted_lookup" if spec.identifier[0] + else "bounded_search" + ), + "sources_requested": selected_sources, + "sources_queried": [ + report.source for report in reports if report.endpoint + ], + "access_date": access_date, + "limit": spec.limit, + "offset": spec.offset, + "offset_scope": "per_source", + "retrieved_by_source": retrieved_by_source, + "total_by_source": total_by_source, + "deduplicated_count": len(merged), + "warnings": warnings, + "errors": errors, + "partial": bool(errors), + "source_provenance": source_provenance, + }, + } + + +def search_papers_advanced(*args: Any, **kwargs: Any) -> list[dict[str, Any]]: + """Compatibility helper returning only the result list.""" + return search_papers_detailed(*args, **kwargs)["results"] diff --git a/src/scansci_pdf/config.py b/src/scansci_pdf/config.py index c185e3a..5fb4123 100644 --- a/src/scansci_pdf/config.py +++ b/src/scansci_pdf/config.py @@ -47,6 +47,8 @@ "ezproxy_login_url": "", "core_api_key": "", "openalex_api_key": "", + "semantic_scholar_api_key": "", + "ncbi_api_key": "", "elsevier_api_key": "", "elsevier_insttoken": "", "connect_timeout": 15, @@ -186,7 +188,11 @@ def update_config(key: str, value: str) -> dict[str, Any]: def get_config_safe() -> dict[str, Any]: config = load_config() - sensitive_keys = ["core_api_key", "vpnsci_cookie_file", "zotero_api_key", "zotero_library_id", "elsevier_api_key", "elsevier_insttoken"] + sensitive_keys = [ + "core_api_key", "openalex_api_key", "semantic_scholar_api_key", + "ncbi_api_key", "vpnsci_cookie_file", "zotero_api_key", + "zotero_library_id", "elsevier_api_key", "elsevier_insttoken", + ] for key in sensitive_keys: if config.get(key): config[key] = "***" diff --git a/src/scansci_pdf/search.py b/src/scansci_pdf/search.py index df0e244..45f3e43 100644 --- a/src/scansci_pdf/search.py +++ b/src/scansci_pdf/search.py @@ -1,8 +1,8 @@ -"""Paper search via OpenAlex, Semantic Scholar, and Crossref.""" +"""Paper search with advanced multi-database routing and legacy helpers.""" from __future__ import annotations -from concurrent.futures import ThreadPoolExecutor, as_completed +import time from typing import Any import requests @@ -196,22 +196,42 @@ def _search_openalex_by_author( def search_papers( query: str = "", - limit: int = 10, + limit: int = 50, year_from: int | None = None, year_to: int | None = None, sort: str | None = None, *, author: str | None = None, author_id: str | None = None, + sources: list[str] | str | None = None, + query_mode: str = "auto", + exact: bool = False, + offset: int = 0, + date_from: str | None = None, + date_to: str | None = None, + publication_types: list[str] | str | None = None, + fields_of_study: list[str] | str | None = None, + venue: str | None = None, + category: str | None = None, + open_access_only: bool = False, + has_abstract: bool | None = None, + min_citations: int | None = None, + language: str | None = None, + recent_days: int | None = None, + enrich_open_access: bool = False, ) -> list[dict[str, Any]]: - """Search papers from OpenAlex + Semantic Scholar + Crossref in parallel. + """Search up to ten literature databases with a common query contract. - When `author` or `author_id` is provided, searches by author instead of keyword. - - `author`: resolves author name → OpenAlex author ID → works - - `author_id`: directly searches works by OpenAlex author ID + Legacy author-only calls keep OpenAlex author-resolution metadata. + Keyword and advanced calls use the auditable multi-source search internally. """ - # --- Author-based search (fast path) --- - if author_id or author: + advanced_requested = any(( + sources, query_mode != "auto", exact, offset, date_from, date_to, + publication_types, fields_of_study, venue, category, open_access_only, + has_abstract is not None, min_citations is not None, language, + recent_days, enrich_open_access, + )) + if (author_id or author) and not advanced_requested: matched_name = None matched_works = 0 matched_cited = 0 @@ -236,53 +256,20 @@ def search_papers( } return results - # --- Keyword-based search (existing parallel path) --- - if not query: + if not query and not author and not author_id: return [] + from .advanced_search import search_papers_advanced - all_results: list[dict[str, Any]] = [] - per_source = max(5, limit) - - with ThreadPoolExecutor(max_workers=3) as pool: - futures = { - pool.submit(_search_openalex, query, per_source, year_from, year_to, sort): "openalex", - pool.submit(_search_semantic_scholar, query, per_source, year_from, year_to): "semantic_scholar", - pool.submit(_search_crossref, query, per_source, year_from, year_to): "crossref", - } - for future in as_completed(futures, timeout=30): - try: - all_results.extend(future.result()) - except Exception: - pass - - # Deduplicate by DOI (prefer entry with more info) - seen: dict[str, dict[str, Any]] = {} - for r in all_results: - doi = r.get("doi", "").lower() - if not doi: - continue - if doi not in seen: - seen[doi] = r - else: - existing = seen[doi] - # Merge: keep fields from whichever entry has more data - if not existing.get("abstract") and r.get("abstract"): - existing["abstract"] = r["abstract"] - if not existing.get("is_oa") and r.get("is_oa"): - existing["is_oa"] = True - existing["oa_url"] = r.get("oa_url", "") - if r.get("cited_by_count", 0) > existing.get("cited_by_count", 0): - existing["cited_by_count"] = r["cited_by_count"] - existing["source"] = existing.get("source", "") + "+" + r.get("source", "") - - # Sort by relevance or citations - merged = list(seen.values()) - if sort == "cited_by_count": - merged.sort(key=lambda x: x.get("cited_by_count", 0), reverse=True) - elif sort == "publication_date": - merged.sort(key=lambda x: x.get("year", 0), reverse=True) - - return merged[:limit] + return search_papers_advanced( + query, limit=limit, year_from=year_from, year_to=year_to, sort=sort, + author=author, author_id=author_id, sources=sources, + query_mode=query_mode, exact=exact, offset=offset, + date_from=date_from, date_to=date_to, publication_types=publication_types, + fields_of_study=fields_of_study, venue=venue, category=category, + open_access_only=open_access_only, has_abstract=has_abstract, + min_citations=min_citations, language=language, recent_days=recent_days, + enrich_open_access=enrich_open_access, + ) def _search_semantic_scholar( @@ -416,6 +403,105 @@ def _search_crossref( return results +def _search_pubmed( + query: str, limit: int = 10, + year_from: int | None = None, year_to: int | None = None, +) -> list[dict[str, Any]]: + """Search PubMed via NCBI E-utilities API.""" + from .network import _get_session, request_timeout + config = load_config() + + try: + session = _get_session(config) + + # Step 1: Search for PMIDs + params: dict[str, Any] = { + "db": "pubmed", + "term": query, + "retmax": min(limit, 200), + "retmode": "json", + "sort": "relevance", + } + if year_from or year_to: + y_from = year_from or "1900" + y_to = year_to or "2026" + params["mindate"] = str(y_from) + params["maxdate"] = str(y_to) + params["datetype"] = "pdat" + + resp = session.get( + "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi", + params=params, + timeout=request_timeout(config), + ) + if resp.status_code != 200: + return [] + data = resp.json() + pmids = data.get("esearchresult", {}).get("idlist", []) + if not pmids: + return [] + + # Step 2: Fetch summaries for PMIDs + time.sleep(0.3) # respect NCBI rate limit + summary_resp = session.get( + "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi", + params={"db": "pubmed", "id": ",".join(pmids), "retmode": "json"}, + timeout=request_timeout(config), + ) + if summary_resp.status_code != 200: + return [] + summary_data = summary_resp.json() + except Exception: + return [] + + results = [] + for pmid in pmids: + info = summary_data.get("result", {}).get(pmid, {}) + if not info or isinstance(info, str): + continue + + # Extract DOI from articleids + doi = "" + for aid in info.get("articleids", []): + if aid.get("idtype") == "doi": + doi = aid.get("value", "") + break + + if not doi: + continue + + authors = [] + for a in info.get("authors", [])[:5]: + name = a.get("name", "") + if name: + authors.append(name) + + # Year from pubdate + pubdate = info.get("pubdate", "") + year = pubdate[:4] if pubdate and pubdate[:4].isdigit() else "" + + title = info.get("title", "") + # Clean up title (remove trailing period) + if title.endswith("."): + title = title[:-1] + + results.append({ + "title": title, + "doi": doi, + "url": f"https://pubmed.ncbi.nlm.nih.gov/{pmid}/", + "authors": authors, + "year": year, + "cited_by_count": 0, # PubMed doesn't provide citation count + "abstract": "", # Summary doesn't include abstract + "is_oa": False, + "oa_url": "", + "pmid": pmid, + "source": "pubmed", + }) + + return results + + def search_by_title(title: str, config: dict[str, Any] | None = None) -> dict[str, Any] | None: """Search OpenAlex by title and return best match with DOI.""" from difflib import SequenceMatcher diff --git a/src/scansci_pdf/search_provider_arxiv.py b/src/scansci_pdf/search_provider_arxiv.py new file mode 100644 index 0000000..3197971 --- /dev/null +++ b/src/scansci_pdf/search_provider_arxiv.py @@ -0,0 +1,166 @@ +"""arXiv Atom search provider.""" + +from __future__ import annotations + +import threading +import time +import xml.etree.ElementTree as ET +from typing import Any + +import requests + +from .search_query import SearchSpec, SourceReport, adapt_query + +_TIMEOUT = 30 +_USER_AGENT = "scansci-pdf/1.9 (+https://github.com/Rimagination/scansci-pdf)" +_ARXIV_LOCK = threading.Lock() +_ARXIV_LAST_REQUEST = 0.0 + +_ATOM = "http://www.w3.org/2005/Atom" +_ARXIV = "http://arxiv.org/schemas/atom" +_OPENSEARCH = "http://a9.com/-/spec/opensearch/1.1/" + + +def _finish(report: SourceReport, started: float) -> SourceReport: + report.elapsed_ms = round((time.monotonic() - started) * 1000) + return report + + +def _parse_feed(root: ET.Element) -> tuple[list[dict[str, Any]], int | None]: + total_text = root.findtext(f"{{{_OPENSEARCH}}}totalResults") + total = int(total_text) if total_text and total_text.isdigit() else None + results: list[dict[str, Any]] = [] + for entry in root.findall(f"{{{_ATOM}}}entry"): + raw_id = entry.findtext(f"{{{_ATOM}}}id") or "" + arxiv_id = raw_id.rstrip("/").rsplit("/", 1)[-1] + title = " ".join((entry.findtext(f"{{{_ATOM}}}title") or "").split()) + abstract = " ".join( + (entry.findtext(f"{{{_ATOM}}}summary") or "").split() + )[:2000] + published = entry.findtext(f"{{{_ATOM}}}published") or "" + updated = entry.findtext(f"{{{_ATOM}}}updated") or "" + doi = entry.findtext(f"{{{_ARXIV}}}doi") or "" + journal = entry.findtext(f"{{{_ARXIV}}}journal_ref") or "" + authors = [ + author.findtext(f"{{{_ATOM}}}name") or "" + for author in entry.findall(f"{{{_ATOM}}}author") + ] + categories = [ + category.get("term", "") + for category in entry.findall(f"{{{_ATOM}}}category") + if category.get("term") + ] + pdf_url = "" + for link in entry.findall(f"{{{_ATOM}}}link"): + if link.get("title") == "pdf" or link.get("type") == "application/pdf": + pdf_url = link.get("href", "") + break + results.append({ + "title": title, + "doi": doi, + "identifier": doi or arxiv_id, + "arxiv_id": arxiv_id, + "url": raw_id, + "authors": [author for author in authors if author][:10], + "year": published[:4], + "publication_date": published, + "updated_date": updated, + "venue": journal or "arXiv", + "type": "preprint", + "category": ",".join(categories), + "cited_by_count": 0, + "abstract": abstract, + "is_oa": True, + "oa_url": pdf_url, + "source": "arxiv", + }) + return results, total + + +def search_arxiv(spec: SearchSpec, _config: dict[str, Any]) -> SourceReport: + global _ARXIV_LAST_REQUEST + started = time.monotonic() + report = SourceReport( + source="arxiv", + endpoint="https://export.arxiv.org/api/query", + ) + identifier_type, identifier = spec.identifier + params: dict[str, Any] = { + "start": spec.offset, + "max_results": min(spec.limit, 100), + "sortOrder": "descending", + } + if identifier_type == "arxiv": + params["id_list"] = identifier + else: + query, warnings = adapt_query( + spec.effective_query, "arxiv", exact=spec.exact + ) + report.warnings.extend(warnings) + clauses = [query] if query else [] + if spec.category: + clauses.append(f"cat:{spec.category}") + if spec.date_from or spec.date_to: + lower = (spec.date_from or "1900-01-01").replace("-", "") + "0000" + upper = (spec.date_to or "2999-12-31").replace("-", "") + "2359" + clauses.append(f"submittedDate:[{lower} TO {upper}]") + if spec.fields_of_study: + clauses.extend(f'all:"{value}"' for value in spec.fields_of_study) + report.warnings.append( + "arXiv fields_of_study values were searched as text; use category for exact arXiv taxonomy" + ) + if spec.publication_types: + report.warnings.append("arXiv only contains preprints; publication_types was ignored") + if spec.language: + report.warnings.append("arXiv has no language filter") + if spec.min_citations is not None: + report.warnings.append("arXiv has no citation-count filter") + if spec.has_abstract is False: + report.warnings.append("arXiv records include abstracts; has_abstract=false was ignored") + if spec.recent_days: + report.warnings.append( + "arXiv does not support recent_days; use date_from/date_to" + ) + if not clauses: + report.warnings.append("arXiv requires a query or arXiv ID") + report.total = 0 + return _finish(report, started) + params["search_query"] = " AND ".join(clauses) + + params["sortBy"] = { + "relevance": "relevance", + "publication_date": "submittedDate", + "updated_date": "lastUpdatedDate", + "cited_by_count": "relevance", + }[spec.sort] + if spec.sort == "cited_by_count": + report.warnings.append( + "arXiv has no citation sort; relevance was used and merged results are sorted locally" + ) + report.params = params + + try: + with _ARXIV_LOCK: + delay = 3.0 - (time.monotonic() - _ARXIV_LAST_REQUEST) + if delay > 0: + time.sleep(delay) + with requests.Session() as session: + session.headers.update({ + "User-Agent": _USER_AGENT, + "Accept": "application/atom+xml", + }) + response = session.get( + report.endpoint, + params=params, + timeout=_TIMEOUT, + ) + _ARXIV_LAST_REQUEST = time.monotonic() + response.raise_for_status() + report.results, report.total = _parse_feed( + ET.fromstring(response.content) + ) + except requests.RequestException as exc: + report.error = f"{type(exc).__name__}: {exc}" + except (ET.ParseError, TypeError, ValueError) as exc: + report.error = f"invalid arXiv response: {exc}" + return _finish(report, started) diff --git a/src/scansci_pdf/search_providers_access.py b/src/scansci_pdf/search_providers_access.py new file mode 100644 index 0000000..189d2b3 --- /dev/null +++ b/src/scansci_pdf/search_providers_access.py @@ -0,0 +1,269 @@ +"""CORE and Unpaywall search/access providers.""" + +from __future__ import annotations + +import os +import time +from typing import Any +from urllib.parse import quote + +import requests + +from .search_query import ( + SearchSpec, + SourceReport, + adapt_query, + valid_contact_email, +) + +_TIMEOUT = 30 +_USER_AGENT = "scansci-pdf/1.9 (+https://github.com/Rimagination/scansci-pdf)" + + +def _finish(report: SourceReport, started: float) -> SourceReport: + report.elapsed_ms = round((time.monotonic() - started) * 1000) + return report + + +def _parse_core(items: list[dict[str, Any]]) -> list[dict[str, Any]]: + results: list[dict[str, Any]] = [] + for item in items: + doi = item.get("doi") or "" + authors = item.get("authors") or [] + names = [ + author.get("name", "") if isinstance(author, dict) else str(author) + for author in authors[:10] + ] + language = item.get("language") or {} + providers = item.get("dataProviders") or [] + venue = "" + if providers: + first = providers[0] + venue = first.get("name", "") if isinstance(first, dict) else str(first) + download_url = item.get("downloadUrl") or "" + if not download_url: + download_url = next( + ( + link.get("url", "") + for link in (item.get("links") or []) + if isinstance(link, dict) and link.get("type") == "download" + ), + "", + ) + results.append({ + "title": item.get("title") or "", + "doi": doi, + "identifier": doi or str(item.get("id") or ""), + "core_id": str(item.get("id") or ""), + "arxiv_id": item.get("arxivId") or "", + "url": download_url, + "authors": [name for name in names if name], + "year": item.get("yearPublished") or "", + "publication_date": item.get("publishedDate") or "", + "venue": venue, + "type": item.get("documentType") or "", + "language": language.get("code", "") if isinstance(language, dict) else language, + "cited_by_count": item.get("citationCount") or 0, + "abstract": (item.get("abstract") or "")[:2000], + "is_oa": True, + "oa_url": download_url, + "source": "core", + }) + return results + + +def search_core(spec: SearchSpec, config: dict[str, Any]) -> SourceReport: + started = time.monotonic() + report = SourceReport(source="core") + api_key = os.environ.get("CORE_API_KEY") or config.get("core_api_key", "") + headers = { + "User-Agent": _USER_AGENT, + "Accept": "application/json", + } + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + report.authenticated = True + + query, warnings = adapt_query(spec.effective_query, "core", exact=spec.exact) + report.warnings.extend(warnings) + clauses = [query] if query else [] + if spec.date_from: + clauses.append(f'yearPublished>={spec.date_from[:4]}') + if spec.date_to: + clauses.append(f'yearPublished<={spec.date_to[:4]}') + if spec.publication_types: + values = " OR ".join( + f'documentType:"{value}"' for value in spec.publication_types + ) + clauses.append(f"({values})") + if spec.language: + clauses.append(f'language.code:"{spec.language.lower()}"') + if spec.has_abstract is True: + clauses.append("_exists_:abstract") + if spec.has_abstract is False: + clauses.append("NOT _exists_:abstract") + if spec.fields_of_study: + clauses.extend(f'"{value}"' for value in spec.fields_of_study) + report.warnings.append( + "CORE fields_of_study values were added to full-text search" + ) + if spec.category: + report.warnings.append( + "CORE does not expose the arXiv/preprint category filter" + ) + if spec.recent_days: + report.warnings.append( + "CORE does not support recent_days; use date_from/date_to" + ) + params: dict[str, Any] = { + "q": " AND ".join(filter(None, clauses)), + "limit": spec.limit, + "offset": spec.offset, + "sort": "recency" if spec.sort in {"publication_date", "updated_date"} else "relevance", + } + if spec.sort == "cited_by_count": + report.warnings.append( + "CORE cannot sort by citation count; merged results are sorted locally" + ) + report.endpoint = "https://api.core.ac.uk/v3/search/works/" + report.params = params + try: + with requests.Session() as session: + session.headers.update(headers) + response = session.get(report.endpoint, params=params, timeout=_TIMEOUT) + response.raise_for_status() + payload = response.json() + report.results = _parse_core(payload.get("results", [])) + report.total = payload.get("totalHits") + if payload.get("errors"): + report.warnings.append("CORE reported partial shard errors") + except requests.RequestException as exc: + report.error = f"{type(exc).__name__}: {exc}" + except (TypeError, ValueError) as exc: + report.error = f"invalid CORE response: {exc}" + return _finish(report, started) + + +def _parse_unpaywall(item: dict[str, Any]) -> dict[str, Any]: + doi = item.get("doi") or "" + best = item.get("best_oa_location") or {} + return { + "title": item.get("title") or "", + "doi": doi, + "identifier": doi, + "url": item.get("doi_url") or (f"https://doi.org/{doi}" if doi else ""), + "authors": [ + author.get("raw_author_name", "") + for author in (item.get("z_authors") or [])[:10] + if author.get("raw_author_name") + ], + "year": item.get("year") or "", + "publication_date": item.get("published_date") or "", + "venue": item.get("journal_name") or "", + "type": item.get("genre") or "", + "cited_by_count": 0, + "abstract": "", + "is_oa": bool(item.get("is_oa")), + "oa_status": item.get("oa_status") or "", + "oa_url": best.get("url_for_pdf") or best.get("url") + or best.get("url_for_landing_page") or "", + "oa_host_type": best.get("host_type") or "", + "oa_version": best.get("version") or "", + "license": best.get("license") or "", + "source": "unpaywall", + } + + +def search_unpaywall(spec: SearchSpec, config: dict[str, Any]) -> SourceReport: + started = time.monotonic() + report = SourceReport(source="unpaywall") + identifier_type, identifier = spec.identifier + email = config.get("email", "") + report.endpoint = "https://api.unpaywall.org/v2/{doi}" + if identifier_type != "doi": + report.warnings.append( + "Unpaywall is DOI lookup only; discover papers in another source first" + ) + report.total = 0 + return _finish(report, started) + if not valid_contact_email(email): + report.warnings.append( + "Unpaywall requires a real contact email; set config key 'email'" + ) + report.total = 0 + return _finish(report, started) + + endpoint = "https://api.unpaywall.org/v2/" + quote(identifier, safe="/") + report.endpoint = endpoint + report.params = {"email": email} + try: + with requests.Session() as session: + session.headers.update({ + "User-Agent": _USER_AGENT, + "Accept": "application/json", + }) + response = session.get(endpoint, params=report.params, timeout=_TIMEOUT) + if response.status_code == 404: + report.total = 0 + return _finish(report, started) + response.raise_for_status() + report.results = [_parse_unpaywall(response.json())] + report.total = 1 + except requests.RequestException as exc: + report.error = f"{type(exc).__name__}: {exc}" + except (TypeError, ValueError) as exc: + report.error = f"invalid Unpaywall response: {exc}" + return _finish(report, started) + + +def enrich_unpaywall( + dois: list[str], + config: dict[str, Any], + *, + max_items: int = 10, +) -> SourceReport: + """Bounded sequential DOI enrichment; never fan out without an explicit request.""" + started = time.monotonic() + report = SourceReport( + source="unpaywall", + endpoint="https://api.unpaywall.org/v2/{doi}", + ) + email = config.get("email", "") + report.params = { + "email": email, + "doi_count": min(len(dois), max_items), + } + if not valid_contact_email(email): + report.warnings.append( + "Unpaywall enrichment skipped: set a real contact email in config key 'email'" + ) + report.total = 0 + return _finish(report, started) + unique_dois = list(dict.fromkeys(doi for doi in dois if doi))[:max_items] + try: + with requests.Session() as session: + session.headers.update({ + "User-Agent": _USER_AGENT, + "Accept": "application/json", + }) + for doi in unique_dois: + endpoint = "https://api.unpaywall.org/v2/" + quote(doi, safe="/") + response = session.get( + endpoint, + params={"email": email}, + timeout=_TIMEOUT, + ) + if response.status_code == 404: + continue + response.raise_for_status() + report.results.append(_parse_unpaywall(response.json())) + report.total = len(report.results) + if len(dois) > max_items: + report.warnings.append( + f"Unpaywall enrichment is bounded to the first {max_items} unique DOIs" + ) + except requests.RequestException as exc: + report.error = f"{type(exc).__name__}: {exc}" + except (TypeError, ValueError) as exc: + report.error = f"invalid Unpaywall response: {exc}" + return _finish(report, started) diff --git a/src/scansci_pdf/search_providers_biomedical.py b/src/scansci_pdf/search_providers_biomedical.py new file mode 100644 index 0000000..8402935 --- /dev/null +++ b/src/scansci_pdf/search_providers_biomedical.py @@ -0,0 +1,369 @@ +"""PubMed, PMC, bioRxiv, and medRxiv search providers.""" + +from __future__ import annotations + +import os +import threading +import time +from typing import Any +from urllib.parse import quote + +import requests + +from .search_query import ( + SearchSpec, + SourceReport, + adapt_query, + valid_contact_email, +) + +_TIMEOUT = 30 +_USER_AGENT = "scansci-pdf/1.9 (+https://github.com/Rimagination/scansci-pdf)" +_NCBI_LOCK = threading.Lock() +_NCBI_LAST_REQUEST = 0.0 +_BIORXIV_LOCK = threading.Lock() + + +def _finish(report: SourceReport, started: float) -> SourceReport: + report.elapsed_ms = round((time.monotonic() - started) * 1000) + return report + + +def _ncbi_params(config: dict[str, Any]) -> tuple[dict[str, Any], bool]: + params: dict[str, Any] = {"tool": "scansci_pdf"} + email = config.get("email", "") + if valid_contact_email(email): + params["email"] = email + api_key = os.environ.get("NCBI_API_KEY") or config.get("ncbi_api_key", "") + if api_key: + params["api_key"] = api_key + return params, bool(api_key) + + +def _wait_for_ncbi(authenticated: bool) -> None: + global _NCBI_LAST_REQUEST + interval = 0.11 if authenticated else 0.34 + delay = interval - (time.monotonic() - _NCBI_LAST_REQUEST) + if delay > 0: + time.sleep(delay) + + +def _mark_ncbi_request() -> None: + global _NCBI_LAST_REQUEST + _NCBI_LAST_REQUEST = time.monotonic() + + +def _extract_article_ids(info: dict[str, Any]) -> dict[str, str]: + ids: dict[str, str] = {} + for article_id in info.get("articleids") or []: + id_type = str(article_id.get("idtype") or "").lower() + value = str(article_id.get("value") or "") + if id_type and value: + ids[id_type] = value + return ids + + +def _parse_ncbi( + source: str, + ids: list[str], + payload: dict[str, Any], +) -> list[dict[str, Any]]: + results: list[dict[str, Any]] = [] + records = payload.get("result") or {} + for uid in ids: + info = records.get(uid) or {} + if not isinstance(info, dict): + continue + article_ids = _extract_article_ids(info) + doi = article_ids.get("doi", "") + pmid = article_ids.get("pubmed", "") or ( + uid if source == "pubmed" else "" + ) + pmcid = article_ids.get("pmc", "") + if source == "pmc" and not pmcid: + pmcid = "PMC" + uid.removeprefix("PMC") + pubdate = str(info.get("pubdate") or "") + title = str(info.get("title") or "").rstrip(".") + authors = [ + author.get("name", "") + for author in (info.get("authors") or [])[:10] + if author.get("name") + ] + identifier = doi or pmcid or pmid + results.append({ + "title": title, + "doi": doi, + "identifier": identifier, + "pmid": pmid, + "pmcid": pmcid, + "url": ( + f"https://pubmed.ncbi.nlm.nih.gov/{pmid}/" + if source == "pubmed" + else f"https://pmc.ncbi.nlm.nih.gov/articles/{pmcid}/" + ), + "authors": authors, + "year": pubdate[:4] if pubdate[:4].isdigit() else "", + "publication_date": pubdate, + "venue": info.get("fulljournalname") or info.get("source") or "", + "type": ",".join(info.get("pubtype") or []), + "cited_by_count": 0, + "abstract": "", + "is_oa": source == "pmc", + "oa_url": ( + f"https://pmc.ncbi.nlm.nih.gov/articles/{pmcid}/" + if source == "pmc" else "" + ), + "source": source, + }) + return results + + +def _search_ncbi( + source: str, + spec: SearchSpec, + config: dict[str, Any], +) -> SourceReport: + started = time.monotonic() + report = SourceReport(source=source) + db = "pubmed" if source == "pubmed" else "pmc" + common, authenticated = _ncbi_params(config) + report.authenticated = authenticated + query, warnings = adapt_query(spec.effective_query, source, exact=spec.exact) + report.warnings.extend(warnings) + + identifier_type, identifier = spec.identifier + if identifier_type == "doi": + query = f'"{identifier}"[AID]' + for publication_type in spec.publication_types: + query = f"({query}) AND ({publication_type}[PT])" if query else f"{publication_type}[PT]" + if spec.language: + query = f"({query}) AND ({spec.language}[LA])" if query else f"{spec.language}[LA]" + if spec.has_abstract is True: + query = f"({query}) AND hasabstract" if query else "hasabstract" + elif spec.has_abstract is False: + query = f"({query}) NOT hasabstract" if query else "NOT hasabstract" + if spec.fields_of_study: + additions = " AND ".join(f'"{value}"' for value in spec.fields_of_study) + query = f"({query}) AND ({additions})" if query else additions + report.warnings.append( + f"{source} fields_of_study values were searched as text; use mesh: for exact MeSH headings" + ) + if spec.open_access_only and source == "pubmed": + query = f"({query}) AND pmc[filter]" if query else "pmc[filter]" + if spec.min_citations is not None: + report.warnings.append(f"{source} does not expose citation-count filtering") + if spec.category: + report.warnings.append( + f"{source} does not expose the arXiv/preprint category filter" + ) + + params: dict[str, Any] = { + "db": db, + "term": query, + "retmax": spec.limit, + "retstart": spec.offset, + "retmode": "json", + **common, + } + if spec.date_from or spec.date_to: + params["datetype"] = "pdat" + if spec.date_from: + params["mindate"] = spec.date_from.replace("-", "/") + if spec.date_to: + params["maxdate"] = spec.date_to.replace("-", "/") + if spec.recent_days: + params["reldate"] = spec.recent_days + params["datetype"] = "pdat" + if spec.sort in {"publication_date", "updated_date"}: + params["sort"] = "pub_date" + else: + params["sort"] = "relevance" + if spec.sort == "cited_by_count": + report.warnings.append( + f"{source} cannot sort by citations; merged results are sorted locally" + ) + + report.endpoint = ( + "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi" + ) + report.params = params + try: + with _NCBI_LOCK, requests.Session() as session: + session.headers.update({ + "User-Agent": _USER_AGENT, + "Accept": "application/json", + }) + _wait_for_ncbi(authenticated) + response = session.get(report.endpoint, params=params, timeout=_TIMEOUT) + _mark_ncbi_request() + response.raise_for_status() + search_payload = response.json().get("esearchresult") or {} + ids = search_payload.get("idlist") or [] + report.total = int(search_payload.get("count") or 0) + if not ids: + return _finish(report, started) + summary_params = { + "db": db, + "id": ",".join(ids), + "retmode": "json", + **common, + } + _wait_for_ncbi(authenticated) + summary_response = session.get( + "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi", + params=summary_params, + timeout=_TIMEOUT, + ) + _mark_ncbi_request() + summary_response.raise_for_status() + report.results = _parse_ncbi(source, ids, summary_response.json()) + for item in report.results: + if spec.has_abstract is not None: + item["has_abstract"] = spec.has_abstract + if spec.open_access_only and source == "pubmed": + item["is_oa"] = True + if item.get("pmcid"): + item["oa_url"] = ( + "https://pmc.ncbi.nlm.nih.gov/articles/" + + item["pmcid"] + "/" + ) + except requests.RequestException as exc: + report.error = f"{type(exc).__name__}: {exc}" + except (TypeError, ValueError) as exc: + report.error = f"invalid NCBI response: {exc}" + return _finish(report, started) + + +def search_pubmed(spec: SearchSpec, config: dict[str, Any]) -> SourceReport: + return _search_ncbi("pubmed", spec, config) + + +def search_pmc(spec: SearchSpec, config: dict[str, Any]) -> SourceReport: + return _search_ncbi("pmc", spec, config) + + +def _parse_preprints( + source: str, + items: list[dict[str, Any]], +) -> list[dict[str, Any]]: + results: list[dict[str, Any]] = [] + for item in items: + doi = item.get("doi") or "" + published_doi = item.get("published") or "" + if str(published_doi).upper() == "NA": + published_doi = "" + author_text = item.get("authors") or "" + authors = [ + name.strip() for name in str(author_text).split(";") if name.strip() + ] + publication_date = item.get("date") or "" + results.append({ + "title": item.get("title") or "", + "doi": doi, + "identifier": doi, + "published_doi": published_doi, + "url": f"https://doi.org/{doi}" if doi else "", + "authors": authors[:10], + "year": str(publication_date)[:4], + "publication_date": publication_date, + "venue": source, + "type": item.get("type") or "preprint", + "category": item.get("category") or "", + "license": item.get("license") or "", + "version": item.get("version") or "", + "cited_by_count": 0, + "abstract": (item.get("abstract") or "")[:2000], + "is_oa": True, + "oa_url": f"https://doi.org/{doi}" if doi else "", + "source": source, + }) + return results + + +def _search_preprint_server( + source: str, + spec: SearchSpec, + _config: dict[str, Any], +) -> SourceReport: + started = time.monotonic() + report = SourceReport(source=source) + identifier_type, identifier = spec.identifier + if spec.query and identifier_type != "doi": + report.warnings.append( + f"{source} has no keyword API; use OpenAlex or Semantic Scholar for topic search" + ) + report.total = 0 + return _finish(report, started) + + if identifier_type == "doi": + interval = quote(identifier, safe="/") + cursor = "na" + elif spec.recent_days: + interval = f"{spec.recent_days}d" + cursor = str(spec.offset) + elif spec.date_from or spec.date_to: + start_date = spec.date_from or spec.date_to + end_date = spec.date_to or spec.date_from + interval = f"{start_date}/{end_date}" + cursor = str(spec.offset) + else: + report.warnings.append( + f"{source} browse requires date_from/date_to, recent_days, or an exact DOI" + ) + report.total = 0 + return _finish(report, started) + + if spec.offset and identifier_type == "doi": + report.warnings.append(f"{source} DOI lookup ignores offset") + if spec.limit != 100 and identifier_type != "doi": + report.warnings.append( + f"{source} API pages contain 100 records; results are locally limited to {spec.limit}" + ) + if spec.publication_types or spec.fields_of_study or spec.min_citations is not None: + report.warnings.append( + f"{source} does not support publication type, field, or citation filters" + ) + if spec.language or spec.venue: + report.warnings.append( + f"{source} does not support language or venue filters" + ) + + report.endpoint = ( + f"https://api.biorxiv.org/details/{source}/{interval}/{cursor}/json" + ) + params: dict[str, Any] = {} + if spec.category: + params["category"] = spec.category + report.params = params + try: + with _BIORXIV_LOCK, requests.Session() as session: + session.headers.update({ + "User-Agent": _USER_AGENT, + "Accept": "application/json", + }) + response = session.get(report.endpoint, params=params, timeout=_TIMEOUT) + response.raise_for_status() + payload = response.json() + messages = payload.get("messages") or [] + if messages: + report.total = int(messages[0].get("total") or messages[0].get("count") or 0) + results = _parse_preprints(source, payload.get("collection") or []) + if spec.has_abstract is not None: + results = [ + item for item in results + if bool(item.get("abstract")) is spec.has_abstract + ] + report.results = results[:spec.limit] + except requests.RequestException as exc: + report.error = f"{type(exc).__name__}: {exc}" + except (TypeError, ValueError) as exc: + report.error = f"invalid {source} response: {exc}" + return _finish(report, started) + + +def search_biorxiv(spec: SearchSpec, config: dict[str, Any]) -> SourceReport: + return _search_preprint_server("biorxiv", spec, config) + + +def search_medrxiv(spec: SearchSpec, config: dict[str, Any]) -> SourceReport: + return _search_preprint_server("medrxiv", spec, config) diff --git a/src/scansci_pdf/search_providers_index.py b/src/scansci_pdf/search_providers_index.py new file mode 100644 index 0000000..af826be --- /dev/null +++ b/src/scansci_pdf/search_providers_index.py @@ -0,0 +1,524 @@ +"""OpenAlex, Semantic Scholar, and Crossref search providers.""" + +from __future__ import annotations + +import os +import re +import time +from typing import Any +from urllib.parse import quote + +import requests + +from .search_query import ( + SearchSpec, + SourceReport, + adapt_query, + extract_fields, + quote_value, + valid_contact_email, +) + +_TIMEOUT = 30 +_USER_AGENT = "scansci-pdf/1.9 (+https://github.com/Rimagination/scansci-pdf)" + + +def _session(headers: dict[str, str] | None = None) -> requests.Session: + session = requests.Session() + session.headers.update({"User-Agent": _USER_AGENT, "Accept": "application/json"}) + if headers: + session.headers.update(headers) + return session + + +def _finish(report: SourceReport, started: float) -> SourceReport: + report.elapsed_ms = round((time.monotonic() - started) * 1000) + return report + + +def _year_from_date(value: Any) -> Any: + if isinstance(value, dict): + parts = value.get("date-parts") or [[]] + return parts[0][0] if parts and parts[0] else "" + return "" + + +def _parse_openalex(works: list[dict[str, Any]]) -> list[dict[str, Any]]: + results: list[dict[str, Any]] = [] + for work in works: + doi = (work.get("doi") or "").replace("https://doi.org/", "") + ids = work.get("ids") or {} + oa = work.get("open_access") or {} + best = work.get("best_oa_location") or {} + primary = work.get("primary_location") or {} + source = primary.get("source") or {} + authors = [ + entry.get("author", {}).get("display_name", "") + for entry in (work.get("authorships") or [])[:10] + if entry.get("author", {}).get("display_name") + ] + abstract_index = work.get("abstract_inverted_index") + abstract = "" + if isinstance(abstract_index, dict): + positions: list[tuple[int, str]] = [] + for word, indexes in abstract_index.items(): + if isinstance(indexes, list): + positions.extend((index, word) for index in indexes if isinstance(index, int)) + abstract = " ".join(word for _, word in sorted(positions))[:2000] + pmid = str(ids.get("pmid") or "").rsplit("/", 1)[-1] + results.append({ + "title": work.get("title") or work.get("display_name") or "", + "doi": doi, + "identifier": doi or work.get("id", "").rsplit("/", 1)[-1], + "openalex_id": work.get("id", "").rsplit("/", 1)[-1], + "pmid": pmid if pmid.isdigit() else "", + "url": work.get("id") or "", + "authors": authors, + "year": work.get("publication_year") or "", + "publication_date": work.get("publication_date") or "", + "venue": source.get("display_name") or "", + "type": work.get("type") or "", + "cited_by_count": work.get("cited_by_count") or 0, + "abstract": abstract, + "is_oa": bool(oa.get("is_oa")), + "oa_status": oa.get("oa_status") or "", + "oa_url": best.get("pdf_url") or best.get("landing_page_url") + or oa.get("oa_url") or "", + "source": "openalex", + }) + return results + + +def search_openalex(spec: SearchSpec, config: dict[str, Any]) -> SourceReport: + started = time.monotonic() + report = SourceReport(source="openalex") + api_key = os.environ.get("OPENALEX_API_KEY") or config.get("openalex_api_key", "") + contact = config.get("email", "") + params: dict[str, Any] = {} + if api_key: + params["api_key"] = api_key + report.authenticated = True + elif valid_contact_email(contact): + params["mailto"] = contact + + identifier_type, identifier = spec.identifier + try: + with _session() as session: + if identifier_type == "doi": + report.endpoint = ( + "https://api.openalex.org/works/doi:" + quote(identifier, safe="") + ) + report.params = params + response = session.get(report.endpoint, params=params, timeout=_TIMEOUT) + if response.status_code == 404: + report.total = 0 + return _finish(report, started) + response.raise_for_status() + report.results = _parse_openalex([response.json()]) + report.total = len(report.results) + return _finish(report, started) + + filters: list[str] = [] + author_filter_applied = False + if spec.author_id: + filters.append(f"authorships.author.id:{spec.author_id}") + author_filter_applied = True + elif spec.author: + author_response = session.get( + "https://api.openalex.org/authors", + params={"search": spec.author, "per_page": 5, **params}, + timeout=_TIMEOUT, + ) + if author_response.status_code == 200: + candidates = author_response.json().get("results", []) + if candidates: + author = max( + candidates, + key=lambda item: ( + item.get("cited_by_count", 0), + item.get("works_count", 0), + ), + ) + filters.append( + "authorships.author.id:" + author.get("id", "").rsplit("/", 1)[-1] + ) + author_filter_applied = True + else: + report.warnings.append( + "OpenAlex could not resolve the author profile; the author name was searched as text" + ) + if spec.date_from: + filters.append(f"from_publication_date:{spec.date_from}") + if spec.date_to: + filters.append(f"to_publication_date:{spec.date_to}") + if spec.publication_types: + filters.append("type:" + "|".join(spec.publication_types)) + if spec.open_access_only: + filters.append("is_oa:true") + if spec.has_abstract is not None: + filters.append(f"has_abstract:{str(spec.has_abstract).lower()}") + if spec.min_citations is not None: + filters.append(f"cited_by_count:>{max(0, spec.min_citations - 1)}") + if spec.language: + filters.append(f"language:{spec.language.lower()}") + + query_parts = [spec.query] if spec.query else [] + if spec.author and not author_filter_applied: + query_parts.append(f'author:{quote_value(spec.author)}') + if spec.venue: + query_parts.append(f'journal:{quote_value(spec.venue)}') + query, warnings = adapt_query( + " AND ".join(query_parts), "openalex", exact=spec.exact + ) + report.warnings.extend(warnings) + if spec.fields_of_study: + query = " AND ".join( + filter(None, [query, *[f'"{field}"' for field in spec.fields_of_study]]) + ) + report.warnings.append( + "OpenAlex field names were added to full-text search; use topic IDs for exact topic filtering" + ) + if spec.category: + report.warnings.append( + "OpenAlex does not expose the arXiv/preprint category filter" + ) + if spec.recent_days: + report.warnings.append( + "OpenAlex does not support recent_days; use date_from/date_to" + ) + params.update({"per_page": spec.limit}) + if query: + params["search.exact" if spec.exact else "search"] = query + if filters: + params["filter"] = ",".join(filters) + sort_map = { + "relevance": "relevance_score:desc", + "publication_date": "publication_date:desc", + "updated_date": "updated_date:desc", + "cited_by_count": "cited_by_count:desc", + } + params["sort"] = sort_map[spec.sort] + params["page"] = spec.offset // spec.limit + 1 + if spec.offset % spec.limit: + report.warnings.append( + "OpenAlex offset is page-based; use offsets that are multiples of limit" + ) + report.endpoint = "https://api.openalex.org/works" + report.params = params + response = session.get(report.endpoint, params=params, timeout=_TIMEOUT) + response.raise_for_status() + payload = response.json() + report.results = _parse_openalex(payload.get("results", [])) + report.total = payload.get("meta", {}).get("count") + except requests.RequestException as exc: + report.error = f"{type(exc).__name__}: {exc}" + except (TypeError, ValueError) as exc: + report.error = f"invalid OpenAlex response: {exc}" + return _finish(report, started) + + +_S2_FIELDS = ( + "paperId,externalIds,url,title,abstract,venue,year,publicationDate," + "citationCount,isOpenAccess,openAccessPdf,fieldsOfStudy,publicationTypes,authors" +) + + +def _get_semantic( + session: requests.Session, + endpoint: str, + params: dict[str, Any], +) -> requests.Response: + for attempt in range(2): + response = session.get(endpoint, params=params, timeout=_TIMEOUT) + if response.status_code not in {429, 503} or attempt == 1: + return response + time.sleep(1.0) + return response + + +def _parse_semantic(papers: list[dict[str, Any]]) -> list[dict[str, Any]]: + results: list[dict[str, Any]] = [] + for paper in papers: + ids = paper.get("externalIds") or {} + doi = ids.get("DOI") or "" + arxiv_id = ids.get("ArXiv") or "" + pmid = ids.get("PubMed") or "" + oa = paper.get("openAccessPdf") or {} + results.append({ + "title": paper.get("title") or "", + "doi": doi, + "identifier": doi or arxiv_id or pmid or paper.get("paperId") or "", + "semantic_scholar_id": paper.get("paperId") or "", + "arxiv_id": arxiv_id, + "pmid": str(pmid), + "url": paper.get("url") or "", + "authors": [ + author.get("name", "") for author in (paper.get("authors") or [])[:10] + if author.get("name") + ], + "year": paper.get("year") or "", + "publication_date": paper.get("publicationDate") or "", + "venue": paper.get("venue") or "", + "type": ",".join(paper.get("publicationTypes") or []), + "fields_of_study": paper.get("fieldsOfStudy") or [], + "cited_by_count": paper.get("citationCount") or 0, + "abstract": (paper.get("abstract") or "")[:2000], + "is_oa": bool(paper.get("isOpenAccess")), + "oa_url": oa.get("url") or "", + "source": "semantic_scholar", + }) + return results + + +def search_semantic_scholar( + spec: SearchSpec, config: dict[str, Any] +) -> SourceReport: + started = time.monotonic() + report = SourceReport(source="semantic_scholar") + api_key = ( + os.environ.get("S2_API_KEY") + or config.get("semantic_scholar_api_key", "") + ) + headers = {"x-api-key": api_key} if api_key else {} + report.authenticated = bool(api_key) + identifier_type, identifier = spec.identifier + + try: + with _session(headers) as session: + if identifier_type in {"doi", "arxiv"}: + prefix = "DOI" if identifier_type == "doi" else "ARXIV" + paper_id = quote(f"{prefix}:{identifier}", safe=":") + report.endpoint = ( + f"https://api.semanticscholar.org/graph/v1/paper/{paper_id}" + ) + report.params = {"fields": _S2_FIELDS} + response = _get_semantic( + session, report.endpoint, report.params + ) + if response.status_code == 404: + report.total = 0 + return _finish(report, started) + response.raise_for_status() + report.results = _parse_semantic([response.json()]) + report.total = len(report.results) + return _finish(report, started) + + query, warnings = adapt_query( + spec.effective_query, "semantic_scholar", exact=spec.exact + ) + report.warnings.extend(warnings) + bulk = spec.advanced + report.endpoint = ( + "https://api.semanticscholar.org/graph/v1/paper/search/bulk" + if bulk else + "https://api.semanticscholar.org/graph/v1/paper/search" + ) + params: dict[str, Any] = { + "query": query, + "fields": _S2_FIELDS, + "limit": spec.limit, + } + if not bulk: + params["offset"] = spec.offset + elif spec.offset: + report.warnings.append( + "Semantic Scholar bulk search uses tokens; offset was ignored" + ) + if spec.date_from or spec.date_to: + params["publicationDateOrYear"] = ( + f"{spec.date_from or ''}:{spec.date_to or ''}" + ) + if spec.fields_of_study: + params["fieldsOfStudy"] = ",".join(spec.fields_of_study) + if spec.publication_types: + params["publicationTypes"] = ",".join(spec.publication_types) + if spec.open_access_only: + params["openAccessPdf"] = "" + if spec.min_citations is not None: + params["minCitationCount"] = spec.min_citations + if spec.venue: + params["venue"] = spec.venue + if spec.language: + report.warnings.append( + "Semantic Scholar search does not expose a language filter" + ) + if spec.category: + report.warnings.append( + "Semantic Scholar does not expose the arXiv/preprint category filter" + ) + if spec.recent_days: + report.warnings.append( + "Semantic Scholar does not support recent_days; use date_from/date_to" + ) + if bulk and spec.sort != "relevance": + sort_field = { + "publication_date": "publicationDate", + "updated_date": "publicationDate", + "cited_by_count": "citationCount", + }[spec.sort] + params["sort"] = f"{sort_field}:desc" + elif not bulk and spec.sort != "relevance": + report.warnings.append( + "Semantic Scholar relevance endpoint cannot sort; merged results are sorted locally" + ) + report.params = params + response = _get_semantic(session, report.endpoint, params) + response.raise_for_status() + payload = response.json() + report.results = _parse_semantic(payload.get("data", [])) + report.total = payload.get("total") + except requests.RequestException as exc: + report.error = f"{type(exc).__name__}: {exc}" + except (TypeError, ValueError) as exc: + report.error = f"invalid Semantic Scholar response: {exc}" + return _finish(report, started) + + +def _parse_crossref(items: list[dict[str, Any]]) -> list[dict[str, Any]]: + results: list[dict[str, Any]] = [] + for item in items: + doi = item.get("DOI") or "" + title_values = item.get("title") or [] + venue_values = item.get("container-title") or [] + publication_date = ( + item.get("published-online") or item.get("published-print") + or item.get("published") or {} + ) + date_parts = publication_date.get("date-parts") or [[]] + parts = date_parts[0] if date_parts else [] + date_value = "-".join( + [f"{parts[0]:04d}"] + + [f"{part:02d}" for part in parts[1:3]] + ) if parts else "" + links = item.get("link") or [] + pdf_url = next( + ( + link.get("URL", "") for link in links + if link.get("content-type") == "application/pdf" + ), + "", + ) + abstract = re.sub(r"<[^>]+>", "", item.get("abstract") or "")[:2000] + results.append({ + "title": title_values[0] if title_values else "", + "doi": doi, + "identifier": doi, + "url": f"https://doi.org/{doi}" if doi else "", + "authors": [ + " ".join( + filter(None, [author.get("given"), author.get("family")]) + ) + for author in (item.get("author") or [])[:10] + ], + "year": parts[0] if parts else "", + "publication_date": date_value, + "venue": venue_values[0] if venue_values else "", + "type": item.get("type") or "", + "cited_by_count": item.get("is-referenced-by-count") or 0, + "abstract": abstract, + "is_oa": False, + "oa_url": "", + "has_full_text_link": bool(pdf_url), + "full_text_url": pdf_url, + "source": "crossref", + }) + return results + + +def search_crossref(spec: SearchSpec, config: dict[str, Any]) -> SourceReport: + started = time.monotonic() + report = SourceReport(source="crossref") + contact = config.get("email", "") + identifier_type, identifier = spec.identifier + params: dict[str, Any] = {} + if valid_contact_email(contact): + params["mailto"] = contact + try: + with _session() as session: + if identifier_type == "doi": + report.endpoint = ( + "https://api.crossref.org/works/" + quote(identifier, safe="") + ) + report.params = params + response = session.get(report.endpoint, params=params, timeout=_TIMEOUT) + if response.status_code == 404: + report.total = 0 + return _finish(report, started) + response.raise_for_status() + report.results = _parse_crossref([response.json().get("message", {})]) + report.total = len(report.results) + return _finish(report, started) + + query, warnings = adapt_query( + spec.effective_query, "crossref", exact=spec.exact + ) + report.warnings.extend(warnings) + if spec.fields_of_study: + query = " AND ".join( + filter(None, [query, *[f'"{field}"' for field in spec.fields_of_study]]) + ) + report.warnings.append( + "Crossref fields_of_study values were added to bibliographic text search" + ) + fields = extract_fields(spec.effective_query) + params.update({ + "query.bibliographic": query, + "rows": spec.limit, + "offset": spec.offset, + "select": ( + "DOI,title,author,published,published-print,published-online," + "is-referenced-by-count,abstract,link,container-title,type" + ), + }) + if fields.get("author"): + params["query.author"] = " ".join(fields["author"]) + if fields.get("journal") or spec.venue: + params["query.container-title"] = ( + spec.venue or " ".join(fields.get("journal", [])) + ) + filters: list[str] = [] + if spec.date_from: + filters.append(f"from-pub-date:{spec.date_from}") + if spec.date_to: + filters.append(f"until-pub-date:{spec.date_to}") + for publication_type in spec.publication_types: + filters.append(f"type:{publication_type}") + if spec.has_abstract is True: + filters.append("has-abstract:true") + if spec.has_abstract is False: + filters.append("has-abstract:false") + if spec.open_access_only: + report.warnings.append( + "Crossref cannot prove OA status; has-full-text is not treated as equivalent" + ) + if spec.language: + report.warnings.append("Crossref search does not expose a language filter") + if spec.category: + report.warnings.append( + "Crossref does not expose the arXiv/preprint category filter" + ) + if spec.recent_days: + report.warnings.append( + "Crossref does not support recent_days; use date_from/date_to" + ) + if filters: + params["filter"] = ",".join(filters) + if spec.sort != "relevance": + params["sort"] = { + "publication_date": "published", + "updated_date": "updated", + "cited_by_count": "is-referenced-by-count", + }[spec.sort] + params["order"] = "desc" + report.endpoint = "https://api.crossref.org/works" + report.params = params + response = session.get(report.endpoint, params=params, timeout=_TIMEOUT) + response.raise_for_status() + payload = response.json().get("message", {}) + report.results = _parse_crossref(payload.get("items", [])) + report.total = payload.get("total-results") + except requests.RequestException as exc: + report.error = f"{type(exc).__name__}: {exc}" + except (TypeError, ValueError) as exc: + report.error = f"invalid Crossref response: {exc}" + return _finish(report, started) diff --git a/src/scansci_pdf/search_query.py b/src/scansci_pdf/search_query.py new file mode 100644 index 0000000..2e4d3aa --- /dev/null +++ b/src/scansci_pdf/search_query.py @@ -0,0 +1,370 @@ +"""Shared contracts and query translation for advanced literature search.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from datetime import date +from typing import Any + +from .identifiers import normalize_arxiv_id + +SUPPORTED_SOURCES = ( + "pubmed", "pmc", "biorxiv", "medrxiv", "arxiv", + "openalex", "crossref", "semantic_scholar", "core", "unpaywall", +) +DISCOVERY_SOURCES = SUPPORTED_SOURCES[:-1] +FIELD_NAMES = ( + "title", "author", "abstract", "journal", + "doi", "category", "mesh", "fulltext", +) + +_DOI_RE = re.compile(r"(?i)^(?:https?://(?:dx\.)?doi\.org/)?(10\.\d{4,9}/\S+)$") +_FIELD_RE = re.compile( + r"(?i)\b(?Ptitle|author|abstract|journal|doi|category|mesh|fulltext):" + r"(?P\"(?:[^\"]|\\\")*\"|[^\s()]+)" +) +_ADVANCED_RE = re.compile( + r'(?i)(?:\b(?:AND|OR|NOT)\b|[()\"*]|\b(?:' + + "|".join(FIELD_NAMES) + + r")\s*:)" +) +_EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$") + +_SOURCE_ALIASES = { + "semantic-scholar": "semantic_scholar", + "semantic scholar": "semantic_scholar", + "s2": "semantic_scholar", + "bio-rxiv": "biorxiv", + "med-rxiv": "medrxiv", + "open-alex": "openalex", + "pubmed-central": "pmc", +} + + +@dataclass(slots=True) +class SearchSpec: + query: str = "" + sources: list[str] | None = None + query_mode: str = "auto" + exact: bool = False + limit: int = 10 + offset: int = 0 + year_from: int | None = None + year_to: int | None = None + date_from: str | None = None + date_to: str | None = None + sort: str = "relevance" + author: str | None = None + author_id: str | None = None + publication_types: list[str] = field(default_factory=list) + fields_of_study: list[str] = field(default_factory=list) + venue: str | None = None + category: str | None = None + open_access_only: bool = False + has_abstract: bool | None = None + min_citations: int | None = None + language: str | None = None + recent_days: int | None = None + enrich_open_access: bool = False + + def __post_init__(self) -> None: + self.query = (self.query or "").strip() + self.query_mode = (self.query_mode or "auto").strip().lower() + self.sort = (self.sort or "relevance").strip().lower() + if self.query_mode not in {"auto", "plain", "advanced"}: + raise ValueError("query_mode must be auto, plain, or advanced") + if self.sort not in { + "relevance", "publication_date", "updated_date", "cited_by_count", + }: + raise ValueError( + "sort must be relevance, publication_date, updated_date, or cited_by_count" + ) + if not 1 <= int(self.limit) <= 100: + raise ValueError("limit must be between 1 and 100") + if int(self.offset) < 0: + raise ValueError("offset must be >= 0") + self.limit = int(self.limit) + self.offset = int(self.offset) + if self.min_citations is not None and int(self.min_citations) < 0: + raise ValueError("min_citations must be >= 0") + if self.min_citations is not None: + self.min_citations = int(self.min_citations) + if self.recent_days is not None and not 1 <= int(self.recent_days) <= 365: + raise ValueError("recent_days must be between 1 and 365") + if self.recent_days is not None: + self.recent_days = int(self.recent_days) + self.publication_types = _clean_list(self.publication_types) + self.fields_of_study = _clean_list(self.fields_of_study) + self.sources = normalize_sources(self.sources) + self._normalize_dates() + self._validate_years() + + def _normalize_dates(self) -> None: + if self.date_from: + date.fromisoformat(self.date_from) + if self.date_to: + date.fromisoformat(self.date_to) + if self.date_from and self.date_to and self.date_from > self.date_to: + raise ValueError("date_from must be <= date_to") + if not self.date_from and self.year_from: + self.date_from = f"{int(self.year_from):04d}-01-01" + if not self.date_to and self.year_to: + self.date_to = f"{int(self.year_to):04d}-12-31" + + def _validate_years(self) -> None: + current = date.today().year + 1 + for name, value in (("year_from", self.year_from), ("year_to", self.year_to)): + if value is not None and not 1000 <= int(value) <= current: + raise ValueError(f"{name} must be between 1000 and {current}") + if self.year_from and self.year_to and int(self.year_from) > int(self.year_to): + raise ValueError("year_from must be <= year_to") + + @property + def advanced(self) -> bool: + if self.query_mode == "advanced": + return True + if self.query_mode == "plain": + return False + return bool(_ADVANCED_RE.search(self.query)) + + @property + def effective_query(self) -> str: + parts = [self.query] if self.query else [] + if self.author: + parts.append(f'author:{quote_value(self.author)}') + if self.venue: + parts.append(f'journal:{quote_value(self.venue)}') + return " AND ".join(parts) + + @property + def identifier(self) -> tuple[str | None, str | None]: + doi = extract_doi(self.query) + if doi: + return "doi", doi + arxiv = extract_arxiv_id(self.query) + if arxiv: + return "arxiv", arxiv + return None, None + + +@dataclass(slots=True) +class SourceReport: + source: str + results: list[dict[str, Any]] = field(default_factory=list) + total: int | None = None + endpoint: str = "" + params: dict[str, Any] = field(default_factory=dict) + warnings: list[str] = field(default_factory=list) + error: str | None = None + elapsed_ms: int = 0 + authenticated: bool = False + + def provenance(self) -> dict[str, Any]: + return { + "source": self.source, + "endpoint": self.endpoint, + "parameters": redact_params(self.params), + "total": self.total, + "retrieved": len(self.results), + "warnings": self.warnings, + "error": self.error, + "elapsed_ms": self.elapsed_ms, + "authenticated": self.authenticated, + } + + +def _clean_list(values: Any) -> list[str]: + if values is None: + return [] + if isinstance(values, str): + values = values.split(",") + return [str(value).strip() for value in values if str(value).strip()] + + +def normalize_sources(sources: Any) -> list[str] | None: + values = _clean_list(sources) + if not values: + return None + normalized: list[str] = [] + for raw in values: + name = _SOURCE_ALIASES.get(raw.lower(), raw.lower().replace("-", "_")) + if name == "auto": + continue + if name == "all": + for source in SUPPORTED_SOURCES: + if source not in normalized: + normalized.append(source) + continue + if name not in SUPPORTED_SOURCES: + raise ValueError( + f"unsupported source {raw!r}; choose from {', '.join(SUPPORTED_SOURCES)}" + ) + if name not in normalized: + normalized.append(name) + return normalized or None + + +def quote_value(value: str) -> str: + value = value.strip() + if value.startswith('"') and value.endswith('"'): + return value + if re.search(r"\s", value): + return '"' + value.replace('"', r'\"') + '"' + return value + + +def extract_doi(value: str) -> str | None: + match = _DOI_RE.match((value or "").strip().rstrip(".,;")) + return match.group(1) if match else None + + +def extract_arxiv_id(value: str) -> str | None: + return normalize_arxiv_id(value or "") + + +def valid_contact_email(value: str | None) -> bool: + if not value or not _EMAIL_RE.match(value): + return False + lowered = value.lower() + return not lowered.endswith(("@example.com", "@example.invalid", "@invalid")) + + +def extract_fields(query: str) -> dict[str, list[str]]: + fields: dict[str, list[str]] = {} + for match in _FIELD_RE.finditer(query or ""): + value = match.group("value").strip() + if value.startswith('"') and value.endswith('"'): + value = value[1:-1].replace(r'\"', '"') + fields.setdefault(match.group("field").lower(), []).append(value) + return fields + + +_FIELD_MAPS: dict[str, dict[str, str | None]] = { + "pubmed": { + "title": "TI", "author": "AU", "abstract": "TIAB", "journal": "TA", + "doi": "AID", "category": "SB", "mesh": "MH", "fulltext": "TW", + }, + "pmc": { + "title": "TI", "author": "AU", "abstract": "TIAB", "journal": "TA", + "doi": "AID", "category": "SB", "mesh": "MH", "fulltext": "TW", + }, + "arxiv": { + "title": "ti", "author": "au", "abstract": "abs", "journal": "jr", + "doi": "all", "category": "cat", "mesh": None, "fulltext": "all", + }, + "core": { + "title": "title", "author": "authors", "abstract": "abstract", + "journal": "dataProviders", "doi": "doi", "category": "documentType", + "mesh": None, "fulltext": "fullText", + }, +} + + +def adapt_query(query: str, source: str, *, exact: bool = False) -> tuple[str, list[str]]: + """Translate the common field syntax while preserving boolean structure.""" + warnings: list[str] = [] + query = (query or "").strip() + mapping = _FIELD_MAPS.get(source) + + def replace_field(match: re.Match[str]) -> str: + field_name = match.group("field").lower() + value = match.group("value") + if mapping is None: + warnings.append( + f"{source} has no exact {field_name}: mapping; searched the value as text" + ) + return value + target = mapping.get(field_name) + if not target: + warnings.append( + f"{source} does not support {field_name}: exactly; searched the value as text" + ) + return value + if source in {"pubmed", "pmc"}: + return f"({value}[{target}])" + return f"{target}:{value}" + + translated = _FIELD_RE.sub(replace_field, query) + if exact and translated and not _ADVANCED_RE.search(query) and not ( + translated.startswith('"') and translated.endswith('"') + ): + translated = '"' + translated.replace('"', r'\"') + '"' + if source == "arxiv": + translated = re.sub(r"(?i)\bNOT\b", "ANDNOT", translated) + elif source == "semantic_scholar": + translated = re.sub(r"(?i)\s+AND\s+", " + ", translated) + translated = re.sub(r"(?i)\s+OR\s+", " | ", translated) + translated = re.sub(r"(?i)\bNOT\s+", "-", translated) + return translated.strip(), list(dict.fromkeys(warnings)) + + +def redact_params(params: dict[str, Any]) -> dict[str, Any]: + redacted: dict[str, Any] = {} + for key, value in params.items(): + lowered = key.lower() + if any(token in lowered for token in ("api_key", "apikey", "token", "secret")): + redacted[key] = "***" if value else "" + elif lowered in {"email", "mailto"}: + redacted[key] = "configured" if value else "" + else: + redacted[key] = value + return redacted + + +def deduplicate_results(results: list[dict[str, Any]]) -> list[dict[str, Any]]: + merged: dict[str, dict[str, Any]] = {} + for raw in results: + item = dict(raw) + doi = (item.get("doi") or "").lower().strip() + arxiv_id = re.sub(r"(?i)v\d+$", "", item.get("arxiv_id") or "") + pmid = str(item.get("pmid") or "") + pmcid = str(item.get("pmcid") or "") + title_key = re.sub(r"\W+", "", (item.get("title") or "").lower())[:160] + year = str(item.get("year") or "") + key = ( + f"doi:{doi}" if doi else + f"arxiv:{arxiv_id}" if arxiv_id else + f"pmid:{pmid}" if pmid else + f"pmcid:{pmcid}" if pmcid else + f"title:{title_key}:{year}" + ) + if not (doi or arxiv_id or pmid or pmcid or title_key): + continue + source = item.get("source") or "unknown" + item.setdefault("sources", [source]) + item.setdefault( + "identifier", + item.get("doi") or item.get("arxiv_id") or item.get("pmcid") + or item.get("pmid") or item.get("core_id") or "", + ) + if key not in merged: + merged[key] = item + continue + current = merged[key] + for field_name, value in item.items(): + if field_name in {"source", "sources"}: + continue + if value not in (None, "", [], {}) and current.get(field_name) in ( + None, "", [], {}, + ): + current[field_name] = value + current["is_oa"] = bool(current.get("is_oa") or item.get("is_oa")) + if item.get("cited_by_count", 0) > current.get("cited_by_count", 0): + current["cited_by_count"] = item["cited_by_count"] + for name in item.get("sources", [source]): + if name not in current["sources"]: + current["sources"].append(name) + current["source"] = "+".join(current["sources"]) + return list(merged.values()) + + +def sort_results(results: list[dict[str, Any]], sort: str) -> None: + if sort == "cited_by_count": + results.sort(key=lambda item: item.get("cited_by_count", 0) or 0, reverse=True) + elif sort in {"publication_date", "updated_date"}: + field_name = "updated_date" if sort == "updated_date" else "publication_date" + results.sort( + key=lambda item: item.get(field_name) or str(item.get("year") or ""), + reverse=True, + ) diff --git a/src/scansci_pdf/server.py b/src/scansci_pdf/server.py index 2b39a6a..91c2c69 100644 --- a/src/scansci_pdf/server.py +++ b/src/scansci_pdf/server.py @@ -5,22 +5,23 @@ import json import time from pathlib import Path -from typing import Any +from typing import Annotated, Any from mcp.server.fastmcp import FastMCP +from pydantic import Field +from .advanced_search import search_papers_detailed from .cache import cache_clear, cache_get from .config import get_config_safe, load_config, update_config from .network import fetch_json from .paperlist import PaperEntry, parse_paper_list from .resolver import batch_resolve -from .search import search_papers from .sources import batch_download, download from .tor import check_tor_circuit mcp_app = FastMCP( name="scansci-pdf", - instructions="Academic paper downloader with 13+ sources, multi-university WebVPN, Tor, and Sci-Hub support. Supports DOI, arXiv ID, keyword search, and resumable batch downloads.", + instructions="Academic paper downloader and auditable literature search service. Search supports 10 scholarly APIs, advanced Boolean and field expressions, structured filters, provenance, DOI/arXiv lookup, and bounded pagination; downloads use 13+ sources, institutional access, Tor, and resumable batches.", ) @@ -182,35 +183,66 @@ def _progress_report(current: int, total: int, identifier: str, result: dict[str @mcp_app.tool() def scansci_pdf_search( - query: str = "", - limit: int = 10, - year_from: int | None = None, - year_to: int | None = None, - sort: str | None = None, - author: str | None = None, - author_id: str | None = None, + query: Annotated[str, Field(description="Plain keywords, DOI/arXiv ID, or advanced Boolean/field expression.")] = "", + limit: Annotated[int, Field(description="Final result limit from 1 to 100; each source is also bounded.")] = 10, + year_from: Annotated[int | None, Field(description="Compatibility lower publication year.")] = None, + year_to: Annotated[int | None, Field(description="Compatibility upper publication year.")] = None, + sort: Annotated[str | None, Field(description="relevance, publication_date, updated_date, or cited_by_count.")] = None, + author: Annotated[str | None, Field(description="Author name; OpenAlex resolves a profile and other sources search author text.")] = None, + author_id: Annotated[str | None, Field(description="Exact OpenAlex author ID, for example A5102961214.")] = None, + sources: Annotated[list[str] | None, Field(description="Selected sources: pubmed, pmc, biorxiv, medrxiv, arxiv, openalex, crossref, semantic_scholar, core, unpaywall.")] = None, + query_mode: Annotated[str, Field(description="auto, plain, or advanced.")] = "auto", + exact: Annotated[bool, Field(description="Request exact matching for plain queries where supported.")] = False, + offset: Annotated[int, Field(description="Per-source zero-based offset where supported; page/token APIs report warnings.")] = 0, + date_from: Annotated[str | None, Field(description="Lower ISO publication date YYYY-MM-DD.")] = None, + date_to: Annotated[str | None, Field(description="Upper ISO publication date YYYY-MM-DD.")] = None, + publication_types: Annotated[list[str] | None, Field(description="Source-native publication or work types.")] = None, + fields_of_study: Annotated[list[str] | None, Field(description="Semantic Scholar fields or broad text topics elsewhere.")] = None, + venue: Annotated[str | None, Field(description="Journal or conference name.")] = None, + category: Annotated[str | None, Field(description="arXiv, bioRxiv, or medRxiv category.")] = None, + open_access_only: Annotated[bool, Field(description="Keep only results confirmed as open access.")] = False, + has_abstract: Annotated[bool | None, Field(description="True requires an abstract, False excludes records with one, null does not filter.")] = None, + min_citations: Annotated[int | None, Field(description="Minimum known citation count.")] = None, + language: Annotated[str | None, Field(description="Language code or name where supported.")] = None, + recent_days: Annotated[int | None, Field(description="1-365 days for recent PubMed or bioRxiv/medRxiv retrieval.")] = None, + enrich_open_access: Annotated[bool, Field(description="Check up to 10 result DOIs in Unpaywall; requires a real config email.")] = False, ) -> str: - """Search for academic papers by keyword or author using OpenAlex API. + """Search academic literature with auditable multi-database retrieval. Args: - query: Search query (e.g. "machine learning drug discovery"). Leave empty when using --author/--author_id. - limit: Maximum number of results (default 10, max 50) - year_from: Filter papers published from this year (e.g. 2020) - year_to: Filter papers published up to this year (e.g. 2025) - sort: Sort order - "cited_by_count" (most cited first), "publication_date" (newest first), or omit for relevance - author: Search by author name — resolves to OpenAlex author ID automatically (e.g. "Fang Jingyun") - author_id: Search by OpenAlex author ID directly (e.g. "A5102961214") + query: Keywords, DOI/arXiv ID, or Boolean/field syntax such as + title:"climate change" AND (author:Smith OR mesh:"global warming"). + sources: Omit for intent routing; use ["all"] only for explicit fan-out. + offset: Per-source offset where supported, not a global merged offset. + + The response preserves the results field and adds retrieval provenance, + non-secret parameters, source totals, warnings, and errors. + + bioRxiv and medRxiv have no keyword API; use DOI/date browsing directly, + or discover topic matches through OpenAlex and Semantic Scholar. """ - results = search_papers( - query, - limit=min(limit, 50), - year_from=year_from, - year_to=year_to, - sort=sort, - author=author, - author_id=author_id, - ) - return json.dumps({"results": results}, ensure_ascii=False) + try: + result = search_papers_detailed( + query=query, limit=limit, year_from=year_from, year_to=year_to, + sort=sort, author=author, author_id=author_id, sources=sources, + query_mode=query_mode, exact=exact, offset=offset, + date_from=date_from, date_to=date_to, + publication_types=publication_types, + fields_of_study=fields_of_study, venue=venue, category=category, + open_access_only=open_access_only, has_abstract=has_abstract, + min_citations=min_citations, language=language, + recent_days=recent_days, enrich_open_access=enrich_open_access, + ) + except ValueError as exc: + result = { + "results": [], + "error": str(exc), + "supported_sources": [ + "pubmed", "pmc", "biorxiv", "medrxiv", "arxiv", + "openalex", "crossref", "semantic_scholar", "core", "unpaywall", + ], + } + return json.dumps(result, ensure_ascii=False) @mcp_app.tool() diff --git a/src/scansci_pdf/templates/index.html b/src/scansci_pdf/templates/index.html index 64400df..a4e3600 100644 --- a/src/scansci_pdf/templates/index.html +++ b/src/scansci_pdf/templates/index.html @@ -54,23 +54,24 @@

ScanSci PDF

-
+
- -
+
+
+ + +
+ @click="showAdvanced = !showAdvanced" + class="px-3 py-2 text-sm text-gray-700 border border-gray-300 rounded-lg hover:bg-gray-50 transition" + x-text="showAdvanced ? 'Hide advanced search' : 'Advanced search'"> + @click="doAction()" + :disabled="loading || downloading || !canAct()" + class="px-4 py-2 text-sm rounded-lg text-white transition disabled:bg-gray-300 + disabled:cursor-not-allowed" + :class="mode === 'search' ? 'bg-blue-600 hover:bg-blue-700' : 'bg-emerald-600 hover:bg-emerald-700'" + x-text="mode === 'search' ? 'Run search' : 'Download PDF'"> +
+ +
+
+

Expression builder

+
+ + + +
+
+ +
+

+ Common syntax: quoted phrases, parentheses, wildcard *, + AND/OR/NOT, and the field prefixes above. +

+
+ +
+
+

Databases

+ +
+
+ +
+

+ No selection routes DOI lookups to Crossref + Semantic Scholar + Unpaywall, + arXiv IDs to arXiv + Semantic Scholar + OpenAlex, and topics to + PubMed + OpenAlex + Semantic Scholar + Crossref. +

+
+ +
+ + + + + + + + + + + + + + +
+ +
+ + + +
+ +
+ Supported parameters by source +
+ + + + + + + + + + + + + + + +
SourceAdvanced queryFiltersPagination / note
PubMed / PMCFields, Boolean, MeSHdate, type, language, abstract, OAretstart; PMC is full text
arXivFields, Boolean, categorydate, categorystart; 1 request / 3 seconds
OpenAlexBoolean, phrase, wildcard, fuzzydate, type, OA, abstract, citations, languagepage; author ID supported
Semantic ScholarBulk Boolean, phrase, wildcarddate, field, type, venue, OA, citationsoffset or bulk token
CrossrefBibliographic + author/journal fieldsdate, type, abstractoffset; OA cannot be proven
COREFields, Boolean, range, existsdate, type, language, abstractoffset; key improves quota
bioRxiv / medRxivNo keyword APIdate, recent days, category, DOI100-record cursor pages
UnpaywallDOI onlyOA status/locationReal config email required
+
+
-
+
@@ -123,6 +300,48 @@

ScanSci PDF

+ +
+
+

Retrieval provenance

+ +
+
+ +
+
+ +
+
+ +
+
+ + + + + + + + +
SourceRetrieved / totalTimeEndpoint
+
+
+
@@ -160,11 +379,14 @@

Open Access +

- + + Open + @@ -227,6 +455,36 @@

Recent Downloads

errorGuidance: [], status: 'checking', sources: {}, + showAdvanced: false, + builderField: 'title', + builderValue: '', + availableSources: [ + {id: 'pubmed', label: 'PubMed'}, {id: 'pmc', label: 'PMC'}, + {id: 'biorxiv', label: 'bioRxiv'}, {id: 'medrxiv', label: 'medRxiv'}, + {id: 'arxiv', label: 'arXiv'}, {id: 'openalex', label: 'OpenAlex'}, + {id: 'crossref', label: 'Crossref'}, {id: 'semantic_scholar', label: 'Semantic Scholar'}, + {id: 'core', label: 'CORE'}, {id: 'unpaywall', label: 'Unpaywall'}, + ], + selectedSources: [], + queryMode: 'auto', + exact: false, + dateFrom: '', + dateTo: '', + searchSort: 'relevance', + publicationTypes: '', + fieldsOfStudy: '', + venue: '', + category: '', + language: '', + minCitations: null, + searchLimit: 10, + searchOffset: 0, + recentDays: null, + hasAbstract: '', + openAccessOnly: false, + enrichOpenAccess: false, + retrieval: null, + searchWarnings: [], async init() { try { @@ -237,6 +495,46 @@

Recent Downloads

} catch { this.status = 'offline'; } }, + canAct() { + if (this.mode === 'download') return Boolean(this.query.trim()); + if (this.query.trim()) return true; + const browseOnly = this.selectedSources.length > 0 + && this.selectedSources.every(source => ['biorxiv', 'medrxiv'].includes(source)); + return browseOnly && Boolean(this.dateFrom || this.dateTo || this.recentDays); + }, + + splitList(value) { + return value.split(',').map(item => item.trim()).filter(Boolean); + }, + + downloadableIdentifier(paper) { + return paper.published_doi || paper.doi || paper.arxiv_id || ''; + }, + + openUrl(paper) { + return paper.oa_url || paper.url || ''; + }, + + insertQueryToken(token) { + const punctuation = token === ')' || token === '*'; + const prefix = this.query && !this.query.endsWith(' ') && !punctuation ? ' ' : ''; + const suffix = ['AND', 'OR', 'NOT'].includes(token) ? ' ' : ''; + this.query += prefix + token + suffix; + this.mode = 'search'; + }, + + appendFieldTerm() { + const raw = this.builderValue.trim(); + if (!raw) return; + const value = /\s/.test(raw) && !(raw.startsWith('"') && raw.endsWith('"')) + ? '"' + raw.replaceAll('"', '\\"') + '"' + : raw; + const term = this.builderField + ':' + value; + this.query += (this.query.trim() ? ' AND ' : '') + term; + this.builderValue = ''; + this.mode = 'search'; + }, + onInput() { this.error = ''; this.errorHint = ''; @@ -249,7 +547,7 @@

Recent Downloads

}, doAction() { - if (!this.query.trim()) return; + if (!this.canAct() || this.downloading) return; if (this.mode === 'download') { this.downloadPaper(this.query.trim()); } else { @@ -263,21 +561,50 @@

Recent Downloads

this.errorHint = ''; this.errorGuidance = []; this.results = []; + this.retrieval = null; + this.searchWarnings = []; + + const body = { + query: this.query.trim(), + limit: Number(this.searchLimit) || 10, + detailed: true, + offset: Number(this.searchOffset) || 0, + sources: this.selectedSources.length ? this.selectedSources : null, + query_mode: this.queryMode, + exact: this.exact, + date_from: this.dateFrom || null, + date_to: this.dateTo || null, + sort: this.searchSort, + publication_types: this.splitList(this.publicationTypes), + fields_of_study: this.splitList(this.fieldsOfStudy), + venue: this.venue.trim() || null, + category: this.category.trim() || null, + open_access_only: this.openAccessOnly, + has_abstract: this.hasAbstract === '' ? null : this.hasAbstract === 'true', + min_citations: this.minCitations === '' || this.minCitations === null + ? null : Number(this.minCitations), + language: this.language.trim() || null, + recent_days: this.recentDays === '' || this.recentDays === null + ? null : Number(this.recentDays), + enrich_open_access: this.enrichOpenAccess, + }; try { const resp = await fetch('/api/search', { method: 'POST', headers: {'Content-Type': 'application/json'}, - body: JSON.stringify({query: this.query.trim(), limit: 10}), + body: JSON.stringify(body), }); + const data = await resp.json(); if (!resp.ok) { - const d = await resp.json(); - this.error = d.error || 'Search failed'; + this.error = data.error || 'Search failed'; return; } - this.results = await resp.json(); + this.results = Array.isArray(data) ? data : (data.results || []); + this.retrieval = Array.isArray(data) ? null : (data.retrieval || null); + this.searchWarnings = this.retrieval?.warnings || []; if (this.results.length === 0) { - this.error = 'No results found. Try different keywords.'; + this.error = 'No results found for the selected contract.'; } } catch (e) { this.error = 'Network error: ' + e.message; @@ -287,7 +614,7 @@

Recent Downloads

}, async downloadPaper(identifier) { - if (!identifier) return; + if (!identifier || this.downloading) return; this.downloading = true; this.downloadMsg = identifier; this.downloadElapsed = 0; @@ -306,28 +633,30 @@

Recent Downloads

}); if (resp.ok && resp.headers.get('content-type')?.includes('application/pdf')) { - // Trigger browser download const blob = await resp.blob(); const cd = resp.headers.get('content-disposition') || ''; let filename = 'paper.pdf'; - const m = cd.match(/filename[*]?=(?:UTF-8''|"?)([^";]+)/i); - if (m) filename = decodeURIComponent(m[1].replace(/"/g, '')); + const match = cd.match(/filename[*]?=(?:UTF-8''|"?)([^";]+)/i); + if (match) filename = decodeURIComponent(match[1].replace(/"/g, '')); const url = URL.createObjectURL(blob); - const a = document.createElement('a'); - a.href = url; a.download = filename; - document.body.appendChild(a); a.click(); - document.body.removeChild(a); + const link = document.createElement('a'); + link.href = url; + link.download = filename; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); URL.revokeObjectURL(url); - const realSource = resp.headers.get('x-scansci-source') || 'unknown'; - this.history.unshift({title: filename.replace('.pdf',''), identifier, source: realSource}); + const source = resp.headers.get('x-scansci-source') || 'unknown'; + this.history.unshift({ + title: filename.replace('.pdf', ''), identifier, source, + }); if (this.history.length > 20) this.history.pop(); } else { const d = await resp.json(); this.error = d.error || d.reason || 'Download failed'; this.errorHint = d.hint?.message || ''; this.errorGuidance = d.guidance || []; - // Update source status if returned if (d.sources) this.sources = d.sources; } } catch (e) { diff --git a/src/scansci_pdf/web.py b/src/scansci_pdf/web.py index 689cca2..c9b69c8 100644 --- a/src/scansci_pdf/web.py +++ b/src/scansci_pdf/web.py @@ -2,6 +2,8 @@ from __future__ import annotations +import asyncio +import inspect import re from pathlib import Path from typing import Any @@ -9,12 +11,12 @@ from fastapi import FastAPI, Request from fastapi.responses import FileResponse, HTMLResponse, JSONResponse from fastapi.templating import Jinja2Templates -from pydantic import BaseModel +from pydantic import BaseModel, Field +from .advanced_search import search_papers_detailed from .config import load_config from .identifiers import is_arxiv_identifier, normalize_doi from .log import get_logger -from .search import search_papers from .sources import download log = get_logger() @@ -22,6 +24,10 @@ _TEMPLATE_DIR = Path(__file__).parent / "templates" templates = Jinja2Templates(directory=str(_TEMPLATE_DIR)) templates.env.cache_size = 0 +_TEMPLATE_REQUEST_FIRST = ( + next(iter(inspect.signature(templates.TemplateResponse).parameters), "") + == "request" +) app = FastAPI(title="ScanSci PDF", description="Academic paper downloader web UI") @@ -33,8 +39,30 @@ class DownloadRequest(BaseModel): class SearchRequest(BaseModel): - query: str - limit: int = 10 + query: str = "" + limit: int = Field(default=50, ge=1, le=100) + detailed: bool = False + sources: list[str] | None = None + query_mode: str = "auto" + exact: bool = False + offset: int = Field(default=0, ge=0) + year_from: int | None = None + year_to: int | None = None + date_from: str | None = None + date_to: str | None = None + sort: str = "relevance" + author: str | None = None + author_id: str | None = None + publication_types: list[str] | None = None + fields_of_study: list[str] | None = None + venue: str | None = None + category: str | None = None + open_access_only: bool = False + has_abstract: bool | None = None + min_citations: int | None = Field(default=None, ge=0) + language: str | None = None + recent_days: int | None = Field(default=None, ge=1, le=365) + enrich_open_access: bool = False # --- Helper --- @@ -91,6 +119,8 @@ def _check_sources(config: dict[str, Any]) -> dict[str, Any]: @app.get("/", response_class=HTMLResponse) async def index(request: Request): + if _TEMPLATE_REQUEST_FIRST: + return templates.TemplateResponse(request, "index.html") return templates.TemplateResponse("index.html", {"request": request}) @@ -160,21 +190,41 @@ async def api_download(req: DownloadRequest): @app.post("/api/search") async def api_search(req: SearchRequest): - """Search papers by keyword. Returns list of results.""" + """Search papers; detailed mode adds retrieval provenance.""" query = req.query.strip() - if not query: + if not query and not ( + req.author or req.author_id or req.date_from or req.date_to or req.recent_days + ): return JSONResponse([], status_code=400) # Normalize DOI URL if _DOI_URL_PATTERN.match(query): query = _DOI_URL_PATTERN.sub("", query) - # If input is a DOI/arXiv, skip search and return a single-item result - if _is_doi_or_arxiv(query): + # Preserve the legacy direct-identifier response unless detailed mode is requested. + if not req.detailed and _is_doi_or_arxiv(query): return JSONResponse([{"doi": normalize_doi(query) if not is_arxiv_identifier(query) else query, "title": "", "is_direct": True}]) - results = search_papers(query, limit=req.limit) - return JSONResponse(results) + try: + result = await asyncio.to_thread( + search_papers_detailed, + query=query, limit=req.limit, sources=req.sources, + query_mode=req.query_mode, exact=req.exact, offset=req.offset, + year_from=req.year_from, year_to=req.year_to, + date_from=req.date_from, date_to=req.date_to, sort=req.sort, + author=req.author, author_id=req.author_id, + publication_types=req.publication_types, + fields_of_study=req.fields_of_study, venue=req.venue, + category=req.category, open_access_only=req.open_access_only, + has_abstract=req.has_abstract, min_citations=req.min_citations, + language=req.language, recent_days=req.recent_days, + enrich_open_access=req.enrich_open_access, + ) + except ValueError as exc: + if req.detailed: + return JSONResponse({"results": [], "error": str(exc)}, status_code=400) + return JSONResponse([], status_code=400) + return JSONResponse(result if req.detailed else result["results"]) @app.get("/api/status")