Adding OpenTelemetry with connection to Grafana cloud

.github/requirements/test.txt

@@ -5,4 +5,8 @@ PyJWT
 httpx
 pytest>=7.4
 pytest-asyncio>=0.23
-modal
+modal
+opentelemetry-sdk
+opentelemetry-exporter-otlp-proto-http>=1.24
+opentelemetry-semantic-conventions>=0.45b0
+opentelemetry-instrumentation-logging>=0.45b0

modal_app.py

@@ -14,16 +14,46 @@
 APP_NAME = f"{env_config.app_name}-{env_config.env_name}"
 app = modal.App(APP_NAME)
 
+_otlp_endpoint = os.environ.get("GRAFANA_OTLP_ENDPOINT") or env_config.otel_endpoint
+if _otlp_endpoint:
+    os.environ.setdefault("OTEL_EXPORTER_OTLP_ENDPOINT", _otlp_endpoint)
+os.environ.setdefault("OTEL_SERVICE_NAME", env_config.service_name or env_config.app_name)
+os.environ.setdefault("MODAL_ENV", env_config.env_name)
+
+
 # SETTING MODAL PROJECT
-@app.function(**build_fastapi_config(env_config))
-@modal.asgi_app()
+@app.cls(**build_fastapi_config(env_config))
 @modal.concurrent(max_inputs=env_config.max_concurrent_requests)
-def fastapi_app():
-    from src.main import app as fastapi_app
-    return fastapi_app
+class FastAPIService:
+    @modal.enter(snap=True)
+    def preload(self) -> None:
+        # Runs once before the CPU snapshot is taken (modal deploy only).
+        # Pre-importing the FastAPI app and all its dependencies bakes them into
+        # the snapshot so subsequent cold starts restore from memory (~50-150ms)
+        # instead of re-importing every module from disk (~300-800ms).
+        import src.main  # noqa: F401
+
+    @modal.enter(snap=False)
+    def startup(self) -> None:
+        # Runs once per container after snapshot restore — never on the request hot path.
+        # Network-bound setup (OTLP connections) must live here; they cannot survive
+        # a snapshot because file descriptors and sockets are not portable across restores.
+        from src.observability import setup_telemetry
+        setup_telemetry()
+
+    @modal.asgi_app()
+    def fastapi_app(self):
+        # src.main is already in sys.modules from preload() — this is a cache hit.
+        from src.main import app as fastapi_app
+        return fastapi_app
+
 
 @app.local_entrypoint()
 def main():
+    # Mirror what @enter does in the Modal container so telemetry works locally too.
+    # The env vars above are already set; setup_telemetry() reads them at call time.
+    from src.observability import setup_telemetry
+    setup_telemetry()
     from src.main import app as fastapi_app
     from uvicorn import run
-    run(fastapi_app, host=env_config.server_host, port=env_config.server_port)
+    run(fastapi_app, host=env_config.server_host, port=env_config.server_port)

modal_common.py

@@ -18,10 +18,16 @@
     "websockets",
     "pydantic",
     "PyJWT",
+    # OpenTelemetry — OTLP/HTTP push to Grafana Cloud (or any OTLP-compatible backend)
+    "opentelemetry-sdk>=1.24",
+    "opentelemetry-exporter-otlp-proto-http>=1.24",
+    "opentelemetry-semantic-conventions>=0.45b0",
+    # Injects otelTraceID/otelSpanID into every LogRecord for log-trace correlation
+    "opentelemetry-instrumentation-logging>=0.45b0",
 ]
 
 cpu_image = (
-    modal.Image.debian_slim(python_version="3.10")
+    modal.Image.debian_slim(python_version="3.11")
     .apt_install(*APT_PACKAGES)
     .uv_pip_install(*PIP_PACKAGES)
     .add_local_dir(".", remote_path="/root")
@@ -72,29 +78,39 @@ class EnvConfig:
     secrets: list = field(default_factory=list)
     volumes: Dict[str, modal.Volume] = field(default_factory=lambda: FASTAPI_VOLUME)
 
+    # OBSERVABILITY — set in prod preset only; None = telemetry disabled (feat/dev)
+    otel_endpoint: Optional[str] = None  # Grafana Cloud OTLP base URL
+    service_name:  Optional[str] = None  # defaults to app_name when None
+
 
 FEAT = EnvConfig(
     env_name="feat",
     server_domain="feat-app.modal.run",
+    otel_endpoint=None,  # endpoint comes from GRAFANA_OTLP_ENDPOINT inside the grafana-otlp secret
     secrets=[
         modal.Secret.from_name("fastapi-auth-secrets"),
+        modal.Secret.from_name("grafana-otlp"),
     ],
 )
 
 DEV = EnvConfig(
     env_name="dev",
     server_domain="dev-app.modal.run",
+    otel_endpoint=None,  # no telemetry in dev — keeps cost at zero
     secrets=[
         modal.Secret.from_name("fastapi-auth-secrets"),
+        modal.Secret.from_name("grafana-otlp"),
     ],
 )
 
 PROD = EnvConfig(
     env_name="prod",
     server_domain="prod-app.modal.run",
-    # min_containers=1, # Uncomment this to run 1 container in production, when building Apps
+    # min_containers=1, # Uncomment to keep 1 warm container in production
+    otel_endpoint=None,  # endpoint comes from GRAFANA_OTLP_ENDPOINT inside the grafana-otlp secret
     secrets=[
         modal.Secret.from_name("fastapi-auth-secrets"),
+        modal.Secret.from_name("grafana-otlp"),
     ],
 )
 
@@ -124,6 +140,7 @@ def build_fastapi_config(env: EnvConfig) -> dict:
         "secrets": env.secrets + [modal.Secret.from_dict({"MODAL_ENV": env.env_name})],
         "volumes": env.volumes,
         "min_containers": env.min_containers,
+        "enable_memory_snapshot": True,
     }
 
     if env.gpu_type:

src/main.py

@@ -10,6 +10,7 @@
 
 from src.api.models import ErrorDetail
 from src.api.routes import router
+from src.observability.middleware import TelemetryMiddleware
 
 from modal_common import get_env_config
 
@@ -41,6 +42,8 @@ async def lifespan(app: FastAPI):
     allow_methods=["*"],
     allow_headers=["*"],
 )
+# Registered last → executes outermost, so it wraps CORS, auth, validation errors, and 404s.
+app.add_middleware(TelemetryMiddleware)
 
 
 @app.exception_handler(RequestValidationError)

src/observability/__init__.py

@@ -0,0 +1,29 @@
+"""
+src/observability
+=================
+OpenTelemetry instrumentation layer for modal-template-fastapi.
+
+Template-owned files (copy verbatim, never edit per-service):
+    setup.py       — SDK init, OTLP/Grafana Cloud exporter
+    middleware.py  — FastAPI auto-instrumentation
+    __init__.py    — this file
+
+Service-owned file (replace entirely per service):
+    metrics.py     — domain-specific metric instruments
+
+Public surface:
+    setup_telemetry()      call once in Modal @enter hook
+    get_tracer(name?)      get a named tracer anywhere
+    get_meter(name?)       get a named meter (used in metrics.py)
+    TelemetryMiddleware    add to FastAPI app in main.py
+"""
+
+from src.observability.setup import get_meter, get_tracer, setup_telemetry
+from src.observability.middleware import TelemetryMiddleware
+
+__all__ = [
+    "setup_telemetry",
+    "get_tracer",
+    "get_meter",
+    "TelemetryMiddleware",
+]

src/observability/metrics.py

@@ -0,0 +1,46 @@
+"""
+src/observability/metrics.py
+=============================
+THIS FILE IS A STUB. Replace it entirely in each new service.
+The template ships this empty on purpose.
+
+─────────────────────────────────────────────────────────────
+CONTRACT
+─────────────────────────────────────────────────────────────
+• Import get_meter from .setup — never instantiate MeterProvider directly.
+• Name your instruments with a service prefix:  "myservice.thing.unit"
+• Use snake_case for Python variables, dot.notation for metric names.
+• All instruments are module-level singletons — create once, use everywhere.
+• Labels (attributes) are added at record time, not at instrument creation.
+
+─────────────────────────────────────────────────────────────
+EXAMPLE  (delete this and replace with your own)
+─────────────────────────────────────────────────────────────
+
+from src.observability.setup import get_meter
+
+_meter = get_meter("modal-fastapi")
+
+# HTTP layer — these are already recorded by TelemetryMiddleware.
+# Only add business-level instruments here.
+
+items_created = _meter.create_counter(
+    name="app.items.created",
+    description="Items successfully created via POST /items",
+    unit="1",
+)
+
+items_fetch_duration = _meter.create_histogram(
+    name="app.items.fetch.duration",
+    description="Time to fetch a single item from the store, in ms",
+    unit="ms",
+)
+
+# Usage in your route handler:
+#   from src.observability.metrics import items_created, items_fetch_duration
+#   items_created.add(1, {"item_type": item.type})
+#   items_fetch_duration.record(elapsed_ms, {"cache_hit": "false"})
+"""
+
+# Leave this file empty until you have real business metrics to add.
+# The middleware handles all HTTP-level metrics automatically.