diff --git a/Dockerfile b/Dockerfile index 05e3a77..f855654 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,10 +24,10 @@ RUN apt-get update && apt-get upgrade -y && \ WORKDIR ${APP_HOME} COPY . ./ -# Install the package with the optional OCR backend, then pre-fetch NLTK data and -# warm the bundled OCR models so the service can start fully offline. Only the -# model warm-up may fail softly; a failed dependency install fails the build. -RUN uv sync --frozen --extra ocr --no-dev && \ +# Install the full service runtime with OCR, then pre-fetch NLTK data and warm +# the bundled OCR models so the service can start fully offline. Only the model +# warm-up may fail softly; a failed dependency install fails the build. +RUN uv sync --frozen --extra all --no-dev && \ python -m nltk.downloader -d /usr/share/nltk_data punkt punkt_tab stopwords && \ { python -c "from rapidocr_onnxruntime import RapidOCR; RapidOCR()" || true; } diff --git a/Dockerfile.test b/Dockerfile.test index 8ad970c..633b1f9 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -20,10 +20,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ WORKDIR ${APP_HOME} COPY . . -# Dev group provides pytest. The OCR model warm-up alone is allowed to fail -# (it only pre-fetches; the service degrades gracefully without it) — a failed -# dependency install must fail the build, so no blanket `|| true` here. -RUN uv sync --frozen --extra ocr --group dev && \ +# Dev group provides pytest and service test dependencies. The OCR model warm-up +# alone is allowed to fail (it only pre-fetches; the service degrades gracefully +# without it) — a failed dependency install must fail the build, so no blanket +# `|| true` here. +RUN uv sync --frozen --extra all --group dev && \ python -m nltk.downloader -d /usr/share/nltk_data punkt punkt_tab stopwords && \ { python -c "from rapidocr_onnxruntime import RapidOCR; RapidOCR()" || true; } diff --git a/Makefile b/Makefile index 6ebe412..6043ed4 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ run_fmt: format check: lint generate_requirements: - uv export --format requirements.txt --extra ocr --group dev --output-file requirements.txt + uv export --format requirements.txt --extra all --group dev --output-file requirements.txt download_nltk_data: uv run python -m nltk.downloader punkt punkt_tab stopwords diff --git a/README.md b/README.md index b2ce059..213cf28 100644 --- a/README.md +++ b/README.md @@ -69,12 +69,25 @@ reproduction commands are in [benchmarks/parsebench/RESULTS.md](benchmarks/parse ## Installation ```bash -# install the project and dev tools with uv +# parser-only install +pip install "warp-ingest[parser]" + +# hosted service runtime (FastAPI/uvicorn) +pip install "warp-ingest[service]" + +# full service runtime with OCR support +pip install "warp-ingest[all]" + +# install the project and dev/test tools with uv +# (dev includes service dependencies because the test suite covers the API) uv sync --group dev # include the optional OCR backend for scanned PDFs uv sync --group dev --extra ocr +# full service runtime with OCR +uv sync --group dev --extra all + # one-time NLTK data download uv run python -m nltk.downloader punkt punkt_tab stopwords ``` @@ -82,6 +95,7 @@ uv run python -m nltk.downloader punkt punkt_tab stopwords ## Running the service ```bash +# install with the `service` or `all` extra first python -m warp_ingest.ingestion_daemon # or: ./run.sh (FastAPI/uvicorn, port 5001) ``` diff --git a/pyproject.toml b/pyproject.toml index 0ba2b47..4ef272c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,16 +26,21 @@ classifiers = [ ] dependencies = [ "beautifulsoup4>=4.12.0,<5.0.0", - "fastapi>=0.115,<1.0", "nltk>=3.9.1", "numpy>=1.24,<3", "pdfplumber>=0.11.4,<0.12.0", "pypdfium2>=5", - "python-multipart>=0.0.9", - "uvicorn>=0.30,<1.0", ] [project.optional-dependencies] +# Parser dependencies live in the base install. This no-op extra exists so +# `pip install warp-ingest[parser]` is an explicit parser-only spelling. +parser = [] +service = [ + "fastapi>=0.115,<1.0", + "python-multipart>=0.0.9", + "uvicorn>=0.30,<1.0", +] # opencv is rapidocr's image-ops backend (resize/normalize/perspective-crop # around the ONNX models); the headless build has the identical cv2 API without # linking GUI libs (libGL), which a server image doesn't have. rapidocr's own @@ -51,6 +56,22 @@ ocr = [ "shapely>=2.1.2", "six>=1.17.0", ] +# Full runtime install for the hosted ingestion service. The benchmark-only +# `tables` extra remains separate because it pulls an ablation provider, not a +# normal production dependency. +all = [ + "fastapi>=0.115,<1.0", + "onnxruntime>=1.17.0,<1.23; python_version == '3.10'", + "onnxruntime>=1.27.0; python_version >= '3.11' and python_version < '3.15'", + "opencv-python-headless>=4.13.0.92", + "pyclipper>=1.4.0", + "python-multipart>=0.0.9", + "pyyaml>=6.0.3", + "rapidocr-onnxruntime>=1.3.0,<2.0.0", + "shapely>=2.1.2", + "six>=1.17.0", + "uvicorn>=0.30,<1.0", +] # Benchmark-ablation baseline for the pluggable table provider. Warp's own # native table engine (warp_ingest/ingestor/table_engine.py) is the default and # needs nothing beyond the core install; this extra exists only to reproduce @@ -67,10 +88,13 @@ Issues = "https://github.com/Open-Source-Legal/Warp-Ingest/issues" [dependency-groups] dev = [ "black>=26.3.1", + "fastapi>=0.115,<1.0", "httpx>=0.27", "isort>=5.13.2", "pytest>=9.0.3", + "python-multipart>=0.0.9", "reportlab>=4.0", + "uvicorn>=0.30,<1.0", ] [build-system] diff --git a/requirements.txt b/requirements.txt index e802f91..3bf5a2e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv export --format requirements.txt --extra ocr --group dev --output-file requirements.txt +# uv export --format requirements.txt --extra all --group dev --output-file requirements.txt -e . annotated-doc==0.0.4 \ --hash=sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320 \ diff --git a/tests/test_package_metadata.py b/tests/test_package_metadata.py new file mode 100644 index 0000000..9d244f1 --- /dev/null +++ b/tests/test_package_metadata.py @@ -0,0 +1,99 @@ +"""Package metadata guards for optional install profiles.""" + +import re +from pathlib import Path + +import pytest + +try: + import tomllib +except ModuleNotFoundError: # Python 3.10 + import tomli as tomllib + +from warp_ingest.ingestion_daemon import __main__ as service_main +from warp_ingest.ingestion_daemon.service_dependencies import ( + MissingServiceExtraError, + require_service_dependency, +) + +PROJECT_ROOT = Path(__file__).resolve().parents[1] + + +def _metadata(): + return tomllib.loads((PROJECT_ROOT / "pyproject.toml").read_text()) + + +def _requirement_names(requirements): + names = set() + for requirement in requirements: + name = re.split(r"\s*(?:\[|[<>=!~; ])", requirement, maxsplit=1)[0] + names.add(name.lower().replace("_", "-")) + return names + + +def test_base_install_is_parser_only(): + project = _metadata()["project"] + base = _requirement_names(project["dependencies"]) + + assert { + "beautifulsoup4", + "nltk", + "numpy", + "pdfplumber", + "pypdfium2", + } <= base + assert {"fastapi", "python-multipart", "uvicorn"}.isdisjoint(base) + assert project["optional-dependencies"]["parser"] == [] + + +def test_service_and_all_extras_are_explicit_profiles(): + extras = _metadata()["project"]["optional-dependencies"] + service = _requirement_names(extras["service"]) + ocr = _requirement_names(extras["ocr"]) + all_extra = _requirement_names(extras["all"]) + + assert {"fastapi", "python-multipart", "uvicorn"} <= service + assert service <= all_extra + assert ocr <= all_extra + assert "pymupdf4llm" not in all_extra + + +def test_missing_service_dependency_has_extra_install_hint(): + def missing_dependency(_module_name): + raise ModuleNotFoundError(name="uvicorn") + + with pytest.raises(MissingServiceExtraError) as exc: + require_service_dependency("uvicorn", importer=missing_dependency) + + message = str(exc.value) + assert 'pip install "warp-ingest[service]"' in message + assert 'pip install "warp-ingest[all]"' in message + assert "Missing dependency: uvicorn." in message + + +def test_transitive_import_errors_are_not_masked(): + def missing_transitive(_module_name): + raise ModuleNotFoundError(name="h11") + + with pytest.raises(ModuleNotFoundError) as exc: + require_service_dependency("uvicorn", importer=missing_transitive) + + assert exc.value.name == "h11" + + +def test_service_launcher_exits_with_install_hint(monkeypatch): + def missing_service_dependency(_module_name, package_name=None): + raise MissingServiceExtraError( + f"install service extra for {package_name or _module_name}" + ) + + monkeypatch.setattr( + service_main, + "require_service_dependency", + missing_service_dependency, + ) + + with pytest.raises(SystemExit) as exc: + service_main.main() + + assert str(exc.value) == "install service extra for uvicorn" diff --git a/uv.lock b/uv.lock index f192de7..0ed03f5 100644 --- a/uv.lock +++ b/uv.lock @@ -1885,18 +1885,28 @@ version = "1.0.2" source = { editable = "." } dependencies = [ { name = "beautifulsoup4" }, - { name = "fastapi" }, { name = "nltk" }, { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, { name = "numpy", version = "2.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "pdfplumber" }, { name = "pypdfium2" }, - { name = "python-multipart" }, - { name = "uvicorn" }, ] [package.optional-dependencies] +all = [ + { name = "fastapi" }, + { name = "onnxruntime", version = "1.22.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "onnxruntime", version = "1.27.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "opencv-python-headless" }, + { name = "pyclipper" }, + { name = "python-multipart" }, + { name = "pyyaml" }, + { name = "rapidocr-onnxruntime" }, + { name = "shapely" }, + { name = "six" }, + { name = "uvicorn" }, +] ocr = [ { name = "onnxruntime", version = "1.22.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "onnxruntime", version = "1.27.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, @@ -1907,6 +1917,11 @@ ocr = [ { name = "shapely" }, { name = "six" }, ] +service = [ + { name = "fastapi" }, + { name = "python-multipart" }, + { name = "uvicorn" }, +] tables = [ { name = "markdown2" }, { name = "pymupdf4llm" }, @@ -1915,40 +1930,57 @@ tables = [ [package.dev-dependencies] dev = [ { name = "black" }, + { name = "fastapi" }, { name = "httpx" }, { name = "isort" }, { name = "pytest" }, + { name = "python-multipart" }, { name = "reportlab" }, + { name = "uvicorn" }, ] [package.metadata] requires-dist = [ { name = "beautifulsoup4", specifier = ">=4.12.0,<5.0.0" }, - { name = "fastapi", specifier = ">=0.115,<1.0" }, + { name = "fastapi", marker = "extra == 'all'", specifier = ">=0.115,<1.0" }, + { name = "fastapi", marker = "extra == 'service'", specifier = ">=0.115,<1.0" }, { name = "markdown2", marker = "extra == 'tables'", specifier = ">=2.4.0" }, { name = "nltk", specifier = ">=3.9.1" }, { name = "numpy", specifier = ">=1.24,<3" }, + { name = "onnxruntime", marker = "python_full_version == '3.10.*' and extra == 'all'", specifier = ">=1.17.0,<1.23" }, { name = "onnxruntime", marker = "python_full_version == '3.10.*' and extra == 'ocr'", specifier = ">=1.17.0,<1.23" }, + { name = "onnxruntime", marker = "python_full_version >= '3.11' and python_full_version < '3.15' and extra == 'all'", specifier = ">=1.27.0" }, { name = "onnxruntime", marker = "python_full_version >= '3.11' and python_full_version < '3.15' and extra == 'ocr'", specifier = ">=1.27.0" }, + { name = "opencv-python-headless", marker = "extra == 'all'", specifier = ">=4.13.0.92" }, { name = "opencv-python-headless", marker = "extra == 'ocr'", specifier = ">=4.13.0.92" }, { name = "pdfplumber", specifier = ">=0.11.4,<0.12.0" }, + { name = "pyclipper", marker = "extra == 'all'", specifier = ">=1.4.0" }, { name = "pyclipper", marker = "extra == 'ocr'", specifier = ">=1.4.0" }, { name = "pymupdf4llm", marker = "extra == 'tables'", specifier = ">=0.0.17" }, { name = "pypdfium2", specifier = ">=5" }, - { name = "python-multipart", specifier = ">=0.0.9" }, + { name = "python-multipart", marker = "extra == 'all'", specifier = ">=0.0.9" }, + { name = "python-multipart", marker = "extra == 'service'", specifier = ">=0.0.9" }, + { name = "pyyaml", marker = "extra == 'all'", specifier = ">=6.0.3" }, { name = "pyyaml", marker = "extra == 'ocr'", specifier = ">=6.0.3" }, + { name = "rapidocr-onnxruntime", marker = "extra == 'all'", specifier = ">=1.3.0,<2.0.0" }, { name = "rapidocr-onnxruntime", marker = "extra == 'ocr'", specifier = ">=1.3.0,<2.0.0" }, + { name = "shapely", marker = "extra == 'all'", specifier = ">=2.1.2" }, { name = "shapely", marker = "extra == 'ocr'", specifier = ">=2.1.2" }, + { name = "six", marker = "extra == 'all'", specifier = ">=1.17.0" }, { name = "six", marker = "extra == 'ocr'", specifier = ">=1.17.0" }, - { name = "uvicorn", specifier = ">=0.30,<1.0" }, + { name = "uvicorn", marker = "extra == 'all'", specifier = ">=0.30,<1.0" }, + { name = "uvicorn", marker = "extra == 'service'", specifier = ">=0.30,<1.0" }, ] -provides-extras = ["ocr", "tables"] +provides-extras = ["parser", "service", "ocr", "all", "tables"] [package.metadata.requires-dev] dev = [ { name = "black", specifier = ">=26.3.1" }, + { name = "fastapi", specifier = ">=0.115,<1.0" }, { name = "httpx", specifier = ">=0.27" }, { name = "isort", specifier = ">=5.13.2" }, { name = "pytest", specifier = ">=9.0.3" }, + { name = "python-multipart", specifier = ">=0.0.9" }, { name = "reportlab", specifier = ">=4.0" }, + { name = "uvicorn", specifier = ">=0.30,<1.0" }, ] diff --git a/warp_ingest/ingestion_daemon/__main__.py b/warp_ingest/ingestion_daemon/__main__.py index eed6477..04ed241 100644 --- a/warp_ingest/ingestion_daemon/__main__.py +++ b/warp_ingest/ingestion_daemon/__main__.py @@ -11,14 +11,23 @@ import os -import uvicorn - import warp_ingest.ingestion_daemon.config as cfg -from warp_ingest.ingestion_daemon.auth import DEFAULT_API_KEY, expected_api_key from warp_ingest.ingestion_daemon.autotune import compute_settings +from warp_ingest.ingestion_daemon.service_dependencies import ( + MissingServiceExtraError, + require_service_dependency, +) def main(): + try: + uvicorn = require_service_dependency("uvicorn") + require_service_dependency("fastapi") + except MissingServiceExtraError as exc: + raise SystemExit(str(exc)) from None + + from warp_ingest.ingestion_daemon.auth import DEFAULT_API_KEY, expected_api_key + settings = compute_settings() # Export the computed budget for the worker processes (a user-set variable # is already reflected in `settings` and setdefault leaves it alone). diff --git a/warp_ingest/ingestion_daemon/app.py b/warp_ingest/ingestion_daemon/app.py index 1140a13..6cefc52 100644 --- a/warp_ingest/ingestion_daemon/app.py +++ b/warp_ingest/ingestion_daemon/app.py @@ -19,9 +19,21 @@ from importlib import metadata from typing import Literal -from fastapi import Depends, FastAPI, File, HTTPException, Query, UploadFile - import warp_ingest.ingestion_daemon.config as cfg +from warp_ingest.ingestion_daemon.service_dependencies import ( + require_any_service_dependency, + require_service_dependency, +) + +_fastapi = require_service_dependency("fastapi") +require_any_service_dependency(("python_multipart", "multipart"), "python-multipart") +Depends = _fastapi.Depends +FastAPI = _fastapi.FastAPI +File = _fastapi.File +HTTPException = _fastapi.HTTPException +Query = _fastapi.Query +UploadFile = _fastapi.UploadFile + from warp_ingest.ingestion_daemon.auth import require_api_key from warp_ingest.ingestion_daemon.autotune import compute_settings from warp_ingest.ingestor import ingestor_api diff --git a/warp_ingest/ingestion_daemon/auth.py b/warp_ingest/ingestion_daemon/auth.py index bd7a5a9..262e61c 100644 --- a/warp_ingest/ingestion_daemon/auth.py +++ b/warp_ingest/ingestion_daemon/auth.py @@ -10,8 +10,13 @@ import os import secrets -from fastapi import HTTPException, Security -from fastapi.security import APIKeyHeader +from warp_ingest.ingestion_daemon.service_dependencies import require_service_dependency + +_fastapi = require_service_dependency("fastapi") +_fastapi_security = require_service_dependency("fastapi.security", "fastapi") +HTTPException = _fastapi.HTTPException +Security = _fastapi.Security +APIKeyHeader = _fastapi_security.APIKeyHeader DEFAULT_API_KEY = "abc123" diff --git a/warp_ingest/ingestion_daemon/service_dependencies.py b/warp_ingest/ingestion_daemon/service_dependencies.py new file mode 100644 index 0000000..a91138b --- /dev/null +++ b/warp_ingest/ingestion_daemon/service_dependencies.py @@ -0,0 +1,58 @@ +"""Helpers for optional service-runtime dependencies.""" + +import importlib +from collections.abc import Callable, Sequence +from types import ModuleType + +SERVICE_EXTRA_INSTALL_HINT = ( + "Warp-Ingest service dependencies are not installed. " + 'Install them with `pip install "warp-ingest[service]"` ' + 'or `pip install "warp-ingest[all]"`.' +) + + +class MissingServiceExtraError(RuntimeError): + """Raised when the service extra is needed but not installed.""" + + +def _missing_service_dependency(package_name: str) -> MissingServiceExtraError: + return MissingServiceExtraError( + f"{SERVICE_EXTRA_INSTALL_HINT} Missing dependency: {package_name}." + ) + + +def require_service_dependency( + module_name: str, + package_name: str | None = None, + importer: Callable[[str], ModuleType] = importlib.import_module, +) -> ModuleType: + """Import a service dependency or raise an install-extra error. + + Only the top-level requested module is converted to the install-extra error; + missing transitive imports are left untouched because they indicate a broken + or incompatible installation rather than a missing optional extra. + """ + try: + return importer(module_name) + except ModuleNotFoundError as exc: + root_module = module_name.partition(".")[0] + if exc.name == root_module: + raise _missing_service_dependency(package_name or root_module) from None + raise + + +def require_any_service_dependency( + module_names: Sequence[str], + package_name: str, + importer: Callable[[str], ModuleType] = importlib.import_module, +) -> ModuleType: + """Import the first available module for a service dependency package.""" + last_missing = None + for module_name in module_names: + try: + return require_service_dependency(module_name, package_name, importer) + except MissingServiceExtraError as exc: + last_missing = exc + if last_missing is not None: + raise last_missing + raise ValueError("module_names must not be empty")