diff --git a/CHANGES.md b/CHANGES.md index cdedc6e2..dae2e76e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,9 @@ Note: Minor version `0.X.0` update might break the API, It's recommended to pin ## [unreleased] +* add: optional Prometheus metrics endpoint at `/metrics` with low-cardinality `operation` labels +* fix: exclude `/healthz` and `/metrics` from Cache-Control headers + ## [1.4.0] - 2026-06-11 * change: switch from `pygeofilter` to `cql2` for CQL filter parsing and evaluation diff --git a/docker-compose.yml b/docker-compose.yml index cd0b242e..ef59c3a8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,7 @@ services: - POSTGRES_HOST=database - POSTGRES_PORT=5432 - DEBUG=TRUE + - TIPG_ENABLE_METRICS=TRUE ports: - "${MY_DOCKER_IP:-127.0.0.1}:8081:8081" depends_on: diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index c073a182..2ed2e10f 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -21,7 +21,7 @@ COPY LICENSE LICENSE COPY tipg/ tipg/ COPY pyproject.toml pyproject.toml -RUN python -m pip install . --no-cache-dir +RUN python -m pip install ".[metrics]" --no-cache-dir RUN rm -rf tipg/ README.md pyproject.toml LICENSE RUN groupadd -g 1000 user && \ diff --git a/docs/src/user_guide/configuration.md b/docs/src/user_guide/configuration.md index a0f92c28..11be7f39 100644 --- a/docs/src/user_guide/configuration.md +++ b/docs/src/user_guide/configuration.md @@ -156,8 +156,57 @@ prefix: **`TIPG_`** - **NAME** (str): Set custom name for `TiPG` app. Default is `TiPg: OGC Features and Tiles API` - **DEBUG** (bool): Default is `False` - **CORS_ORIGIN** (str): Default is `*` -- **CACHECONTROL** (str): Default is `public, max-age=3600` +- **CACHECONTROL** (str): Default is `public, max-age=3600`. `/healthz` and `/metrics` are excluded. - **TEMPLATE_DIRECTORY** (str): Path to custom template directory to overwrite the HTML files. - **ROOT_PATH** (str): A path prefix handled by a proxy that is not seen by the application but is seen by external clients. - **ADD_TILES_VIEWER** (bool): Defaults is `True` +- **ENABLE_METRICS** (bool): Enable the Prometheus `/metrics` endpoint. Default is `False`. Requires the optional `metrics` extra. - **CATALOG_TTL** (int, in seconds): Tables/Functions catalog **Time To Live** cache (default to 300 seconds). + +## Prometheus metrics + +Install the optional `metrics` extra and set `TIPG_ENABLE_METRICS=TRUE` to expose HTTP request metrics at `/metrics`: + +```bash +pip install 'tipg[metrics]' +export TIPG_ENABLE_METRICS=TRUE +``` + +The project Docker image installs the `metrics` extra. Enable the endpoint with `TIPG_ENABLE_METRICS=TRUE`. + +Once enabled, `/metrics` records: + +- `tipg_http_requests_total{operation,method,status}` +- `tipg_http_request_duration_seconds{operation,method}` + +`operation` values are low-cardinality labels: `landing`, `conformance`, `list_collections`, `get_collection`, `queryables`, `list_items`, `get_item`, `tiles`, `tile_matrix_sets`, `other`, and `unknown`. `status` is grouped (`2xx`, `3xx`, `4xx`, `5xx`). `/healthz` and the scrape endpoint are excluded from request counters. Untemplated paths (for example bare 404s with no matching route) are also ignored. `/healthz` and `/metrics` are also excluded from Cache-Control so probes and scrapes are not cached by CDNs or proxies. + +### Multi-worker deployments + +For multi-worker deployments (for example `uvicorn --workers N` or Gunicorn), set `PROMETHEUS_MULTIPROC_DIR` to an existing writable directory **before** the application is imported, and clear that directory before each server start. An invalid path fails startup with `ValueError`. + +```bash +export PROMETHEUS_MULTIPROC_DIR=/tmp/tipg-prometheus +mkdir -p "$PROMETHEUS_MULTIPROC_DIR" +rm -rf "$PROMETHEUS_MULTIPROC_DIR"/* +``` + +With Gunicorn, also mark workers dead on exit so stale metric files are cleaned up: + +```python +from prometheus_client import multiprocess + + +def child_exit(server, worker): + multiprocess.mark_process_dead(worker.pid) +``` + +### Custom apps + +Custom apps can enable the same instrumentation: + +```python +from tipg.metrics import instrument_app + +instrument_app(app) +``` diff --git a/pyproject.toml b/pyproject.toml index 08aa907e..7d121abf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ dev = [ "numpy", "pre-commit", "bump-my-version", + "tipg[metrics]", ] server = [ "uvicorn[standard]>=0.12.0,<0.19.0", @@ -59,6 +60,11 @@ docs = [ "mkdocstrings[python]>=0.25.1", ] +[project.optional-dependencies] +metrics = [ + "prometheus-fastapi-instrumentator>=8.0.2", +] + [project.urls] Homepage = "https://developmentseed.org/tipg" Source = "https://github.com/developmentseed/tipg" diff --git a/tests/conftest.py b/tests/conftest.py index 53c2f565..89106195 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -146,6 +146,7 @@ def app(database_url, monkeypatch): # API config monkeypatch.setenv("TIPG_TEMPLATE_DIRECTORY", TEMPLATE_DIRECTORY) + monkeypatch.setenv("TIPG_ENABLE_METRICS", "TRUE") # Custom Functions monkeypatch.setenv("TIPG_CUSTOM_SQL_DIRECTORY", SQL_FUNCTIONS_DIRECTORY) diff --git a/tests/test_main.py b/tests/test_main.py index 2192028a..25f22c37 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -6,6 +6,7 @@ def test_health(app): response = app.get("/healthz") assert response.status_code == 200 assert response.json() == {"ping": "pong!"} + assert "Cache-Control" not in response.headers response = app.get("/rawcatalog") assert response.status_code == 200 diff --git a/tests/test_metrics.py b/tests/test_metrics.py new file mode 100644 index 00000000..94da0d90 --- /dev/null +++ b/tests/test_metrics.py @@ -0,0 +1,262 @@ +"""Test tipg Prometheus metrics.""" + +import os +import subprocess +import sys +import textwrap +from pathlib import Path + +import pytest + +from tipg.metrics import REQUESTS, instrument_app, resolve_operation + +from fastapi import FastAPI + +from starlette.testclient import TestClient + + +def _counter_total() -> float: + """Sum tipg request counter samples across all label sets.""" + total = 0.0 + for metric in REQUESTS.collect(): + for sample in metric.samples: + if sample.name == "tipg_http_requests_total": + total += sample.value + return total + + +@pytest.mark.parametrize( + "route,expected", + [ + ("/", "landing"), + ("/conformance", "conformance"), + ("/collections", "list_collections"), + ("/collections/{collectionId}", "get_collection"), + ("/collections/{collectionId}/queryables", "queryables"), + ("/collections/{collectionId}/items", "list_items"), + ("/collections/{collectionId}/items/{itemId}", "get_item"), + ("/collections/{collectionId}/tiles", "tiles"), + ( + "/collections/{collectionId}/tiles/{tileMatrixSetId}/{z}/{x}/{y}", + "tiles", + ), + ("/tileMatrixSets", "tile_matrix_sets"), + ("/tileMatrixSets/{tileMatrixSetId}", "tile_matrix_sets"), + (None, "unknown"), + ("/api", "other"), + ], +) +def test_resolve_operation(route, expected): + """Map tipg route templates to low-cardinality operation labels.""" + assert resolve_operation(route) == expected + + +def test_metrics_endpoint(app): + """Metrics returns Prometheus exposition with tipg operation labels.""" + assert app.get("/collections").status_code == 200 + + response = app.get("/metrics") + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/plain") + assert "Cache-Control" not in response.headers + assert "tipg_http_requests_total{" in response.text + assert 'operation="list_collections"' in response.text + assert "tipg_http_request_duration_seconds_bucket{" in response.text + assert 'status="2xx"' in response.text + + +def test_metrics_exclude_healthz_and_metrics(app): + """Health and scrape endpoints are not counted under any operation label.""" + before = _counter_total() + assert app.get("/healthz").status_code == 200 + assert app.get("/metrics").status_code == 200 + assert _counter_total() == before + + +def test_healthz_exclusion_is_exact(): + """Only the exact /healthz path is excluded from request counters.""" + app = FastAPI() + + @app.get("/foo/healthz-report") + def healthz_report(): + return {"ok": True} + + instrument_app(app) + + with TestClient(app) as client: + before = _counter_total() + assert client.get("/foo/healthz-report").status_code == 200 + assert _counter_total() == before + 1 + + +def test_custom_metrics_endpoint_excluded(): + """A custom scrape endpoint is excluded from request counters.""" + app = FastAPI() + + @app.get("/hello") + def hello(): + return {"ok": True} + + instrument_app(app, endpoint="/prom") + + with TestClient(app) as client: + before = _counter_total() + assert client.get("/prom").status_code == 200 + assert _counter_total() == before + + assert client.get("/hello").status_code == 200 + assert _counter_total() == before + 1 + body = client.get("/prom").text + assert 'operation="other"' in body + + +def test_instrument_app_is_idempotent(): + """Calling instrument_app twice on the same app is a no-op.""" + app = FastAPI() + instrument_app(app) + instrument_app(app) + + metric_routes = [ + getattr(route, "path", None) + for route in app.routes + if getattr(route, "path", None) == "/metrics" + ] + assert len(metric_routes) == 1 + + +def test_optional_metrics_import_error(tmp_path: Path): + """Missing prometheus packages raise ImportError from tipg.metrics.""" + script = textwrap.dedent( + """ + import builtins + import sys + + real_import = builtins.__import__ + + def fake_import(name, globals=None, locals=None, fromlist=(), level=0): + if name.startswith("prometheus_"): + raise ImportError(name) + return real_import(name, globals, locals, fromlist, level) + + builtins.__import__ = fake_import + sys.modules.pop("tipg.metrics", None) + + try: + import tipg.metrics # noqa: F401 + except ImportError: + raise SystemExit(0) + raise SystemExit(1) + """ + ) + script_path = tmp_path / "check_import_error.py" + script_path.write_text(script) + result = subprocess.run( + [sys.executable, str(script_path)], + check=False, + capture_output=True, + text=True, + cwd=str(Path(__file__).resolve().parents[1]), + env={**os.environ, "PYTHONPATH": str(Path(__file__).resolve().parents[1])}, + ) + assert result.returncode == 0, result.stderr + + +def test_instrument_app_errors_are_not_swallowed(monkeypatch): + """Non-ImportError failures from instrumentation propagate to the caller.""" + import tipg.metrics as metrics + + app = FastAPI() + metrics._INSTRUMENTED_APPS.discard(app) + + def boom(*args, **kwargs): + raise ValueError("PROMETHEUS_MULTIPROC_DIR is invalid") + + monkeypatch.setattr(metrics, "Instrumentator", boom) + with pytest.raises(ValueError, match="PROMETHEUS_MULTIPROC_DIR"): + instrument_app(app) + + +def test_main_propagates_instrumentation_errors(tmp_path: Path): + """tipg.main only swallows ImportError from importing tipg.metrics.""" + script = textwrap.dedent( + """ + import sys + from types import ModuleType + + fake = ModuleType("tipg.metrics") + + def instrument_app(app): + raise ImportError("broken installed metrics dependency") + + fake.instrument_app = instrument_app + sys.modules["tipg.metrics"] = fake + sys.modules.pop("tipg.main", None) + + try: + import tipg.main # noqa: F401 + except ImportError as exc: + if "broken installed metrics dependency" in str(exc): + raise SystemExit(0) + raise + raise SystemExit(1) + """ + ) + script_path = tmp_path / "check_main_import.py" + script_path.write_text(script) + result = subprocess.run( + [sys.executable, str(script_path)], + check=False, + capture_output=True, + text=True, + cwd=str(Path(__file__).resolve().parents[1]), + env={ + **os.environ, + "PYTHONPATH": str(Path(__file__).resolve().parents[1]), + "DATABASE_URL": "postgresql://user:pass@localhost:5432/db", + }, + ) + assert result.returncode == 0, result.stderr + + +def test_multiprocess_metrics_exposition(tmp_path: Path): + """Multiprocess scrape works when PROMETHEUS_MULTIPROC_DIR is set before import.""" + multiproc_dir = tmp_path / "multiproc" + multiproc_dir.mkdir() + script = textwrap.dedent( + f""" + import os + + os.environ["PROMETHEUS_MULTIPROC_DIR"] = {str(multiproc_dir)!r} + + from fastapi import FastAPI + from starlette.testclient import TestClient + + from tipg.metrics import instrument_app + + app = FastAPI() + + @app.get("/collections") + def collections(): + return [] + + instrument_app(app) + + with TestClient(app) as client: + assert client.get("/collections").status_code == 200 + response = client.get("/metrics") + assert response.status_code == 200 + assert "tipg_http_requests_total" in response.text + assert 'operation="list_collections"' in response.text + """ + ) + script_path = tmp_path / "check_multiproc.py" + script_path.write_text(script) + result = subprocess.run( + [sys.executable, str(script_path)], + check=False, + capture_output=True, + text=True, + cwd=str(Path(__file__).resolve().parents[1]), + env={**os.environ, "PYTHONPATH": str(Path(__file__).resolve().parents[1])}, + ) + assert result.returncode == 0, result.stderr diff --git a/tipg/main.py b/tipg/main.py index 50b3ccf3..0562efc4 100644 --- a/tipg/main.py +++ b/tipg/main.py @@ -78,6 +78,12 @@ async def lifespan(app: FastAPI): ) app.include_router(ogc_api.router) + +if settings.enable_metrics: + from tipg.metrics import instrument_app + + instrument_app(app) + # Set all CORS enabled origins if settings.cors_origins: app.add_middleware( @@ -88,7 +94,11 @@ async def lifespan(app: FastAPI): allow_headers=["*"], ) -app.add_middleware(CacheControlMiddleware, cachecontrol=settings.cachecontrol) +app.add_middleware( + CacheControlMiddleware, + cachecontrol=settings.cachecontrol, + exclude_path={r"^/healthz$", r"^/metrics$"}, +) app.add_middleware(CompressionMiddleware, compression_level=6) if settings.catalog_ttl: diff --git a/tipg/metrics.py b/tipg/metrics.py new file mode 100644 index 00000000..7680bb12 --- /dev/null +++ b/tipg/metrics.py @@ -0,0 +1,83 @@ +"""Optional Prometheus metrics for tipg.""" + +from __future__ import annotations + +import re +from typing import TYPE_CHECKING +from weakref import WeakSet + +from prometheus_client import Counter, Histogram +from prometheus_fastapi_instrumentator import Instrumentator + +if TYPE_CHECKING: + from prometheus_fastapi_instrumentator.metrics import Info + + from fastapi import FastAPI + +_INSTRUMENTED_APPS: WeakSet[FastAPI] = WeakSet() + +_ROUTE_RULES: list[tuple[re.Pattern[str], str]] = [ + (re.compile(r"^/$"), "landing"), + (re.compile(r"^/conformance$"), "conformance"), + (re.compile(r"^/collections$"), "list_collections"), + (re.compile(r"^/collections/[^/]+$"), "get_collection"), + (re.compile(r"^/collections/[^/]+/queryables$"), "queryables"), + (re.compile(r"^/collections/[^/]+/items$"), "list_items"), + (re.compile(r"^/collections/[^/]+/items/[^/]+$"), "get_item"), + (re.compile(r"/tiles(/|$)"), "tiles"), + (re.compile(r"^/tileMatrixSets"), "tile_matrix_sets"), +] + + +def resolve_operation(route: str | None) -> str: + """Map a request to a low-cardinality tipg operation label.""" + if not route: + return "unknown" + + for pattern, operation in _ROUTE_RULES: + if pattern.search(route): + return operation + + return "other" + + +REQUESTS = Counter( + "tipg_http_requests_total", + "Total HTTP requests by tipg operation.", + labelnames=("operation", "method", "status"), +) +LATENCY = Histogram( + "tipg_http_request_duration_seconds", + "HTTP request latency by tipg operation.", + labelnames=("operation", "method"), + buckets=(0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, float("inf")), +) + + +def record_service_metrics(info: Info) -> None: + """Record request count and latency using low-cardinality operation labels.""" + route = info.request.scope.get("route") + route_path = getattr(route, "path", None) + operation = resolve_operation(route_path) + + REQUESTS.labels(operation, info.method, info.modified_status).inc() + LATENCY.labels(operation, info.method).observe(info.modified_duration) + + +def instrument_app(app: FastAPI, endpoint: str = "/metrics") -> None: + """Instrument a FastAPI app and expose Prometheus metrics.""" + if app in _INSTRUMENTED_APPS: + return + + ( + Instrumentator( + should_group_status_codes=True, + should_ignore_untemplated=True, + excluded_handlers=[r"^/healthz$", rf"^{re.escape(endpoint)}$"], + ) + .add(record_service_metrics) + .instrument(app) + .expose(app, endpoint=endpoint, include_in_schema=False) + ) + + _INSTRUMENTED_APPS.add(app) diff --git a/tipg/settings.py b/tipg/settings.py index 14781497..bd381e56 100644 --- a/tipg/settings.py +++ b/tipg/settings.py @@ -26,6 +26,8 @@ class APISettings(BaseSettings): template_directory: Optional[str] = None root_path: str = "" + enable_metrics: bool = False + add_tiles_viewer: bool = True catalog_ttl: int = 300 diff --git a/uv.lock b/uv.lock index 1797657c..59c73c4f 100644 --- a/uv.lock +++ b/uv.lock @@ -1509,6 +1509,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/80/6e/4b28b62ecb6aae56769c34a8ff1d661473ec1e9519e2d5f8b2c150086b26/pre_commit-4.6.0-py2.py3-none-any.whl", hash = "sha256:e2cf246f7299edcabcf15f9b0571fdce06058527f0a06535068a86d38089f29b", size = 226472, upload-time = "2026-04-21T20:31:40.092Z" }, ] +[[package]] +name = "prometheus-client" +version = "0.25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/fb/d9aa83ffe43ce1f19e557c0971d04b90561b0cfd50762aafb01968285553/prometheus_client-0.25.0.tar.gz", hash = "sha256:5e373b75c31afb3c86f1a52fa1ad470c9aace18082d39ec0d2f918d11cc9ba28", size = 86035, upload-time = "2026-04-09T19:53:42.359Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/9b/d4b1e644385499c8346fa9b622a3f030dce14cd6ef8a1871c221a17a67e7/prometheus_client-0.25.0-py3-none-any.whl", hash = "sha256:d5aec89e349a6ec230805d0df882f3807f74fd6c1a2fa86864e3c2279059fed1", size = 64154, upload-time = "2026-04-09T19:53:41.324Z" }, +] + +[[package]] +name = "prometheus-fastapi-instrumentator" +version = "8.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "prometheus-client" }, + { name = "starlette" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1b/e9/2065686d1dfa62296fdc158b6e8fd25b0cb3dca09b0632cabeb5ae81fe4d/prometheus_fastapi_instrumentator-8.0.2.tar.gz", hash = "sha256:3c252e748151768a7aefd66824a04a870144f71de48a67aed211749a9ca2a548", size = 21342, upload-time = "2026-06-23T09:39:31.611Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c6/c7/fa2b3f469a2e6001b829e0d6bc8680755349aa1329a87bf48731e9d5d30a/prometheus_fastapi_instrumentator-8.0.2-py3-none-any.whl", hash = "sha256:746002ec1e2c58b93f61444e1d104de959a9463a6a3f1c8909ac3757e16c3866", size = 20549, upload-time = "2026-06-23T09:39:32.616Z" }, +] + [[package]] name = "prompt-toolkit" version = "3.0.52" @@ -2235,6 +2257,11 @@ dependencies = [ { name = "starlette-cramjam" }, ] +[package.optional-dependencies] +metrics = [ + { name = "prometheus-fastapi-instrumentator" }, +] + [package.dev-dependencies] dev = [ { name = "bump-my-version" }, @@ -2247,6 +2274,7 @@ dev = [ { name = "pytest-benchmark" }, { name = "pytest-cov" }, { name = "pytest-postgresql" }, + { name = "tipg", extra = ["metrics"] }, ] docs = [ { name = "black" }, @@ -2269,11 +2297,13 @@ requires-dist = [ { name = "jinja2", specifier = ">=2.11.2,<4.0.0" }, { name = "morecantile", specifier = ">=5.0,<7.0" }, { name = "orjson" }, + { name = "prometheus-fastapi-instrumentator", marker = "extra == 'metrics'", specifier = ">=8.0.2" }, { name = "pydantic", specifier = ">=2.4,<3.0" }, { name = "pydantic-settings", specifier = "~=2.0" }, { name = "starlette", specifier = ">=1.0" }, { name = "starlette-cramjam", specifier = ">=0.4,<1.0" }, ] +provides-extras = ["metrics"] [package.metadata.requires-dev] dev = [ @@ -2287,6 +2317,7 @@ dev = [ { name = "pytest-benchmark" }, { name = "pytest-cov" }, { name = "pytest-postgresql" }, + { name = "tipg", extras = ["metrics"] }, ] docs = [ { name = "black", specifier = ">=23.10.1" },