From e7dd4950727a5ee8ac1606ad771a9353fa9761c7 Mon Sep 17 00:00:00 2001 From: Tobias Macey Date: Tue, 18 Aug 2026 13:29:30 -0400 Subject: [PATCH] feat(ocw_studio,ovs): export OTel traces from two more Django apps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both apps have been running in applications-{qa,production} with no tracing at all. Once mitodl/ocw-studio#TBD and mitodl/odl-video-service#TBD ship, the only thing still missing is the endpoint, which is what this sets. ★ ORDERING: LAND THIS AFTER THOSE IMAGES ARE DEPLOYED. ★ The env vars are inert without the app-side change (ocw-studio has no observability app installed at all; OVS has one but no instrumentors), so landing this first buys config that reads as working and does nothing. ── WHAT EACH VARIABLE IS FOR, AND WHAT IS DELIBERATELY ABSENT ── The six existing OTel stacks carry nine variables each. Three of those nine are read by nothing, so the new adopters get five rather than inheriting them: OPENTELEMETRY_ENDPOINT a Django setting the app reads; the library hands it to OTLPSpanExporter verbatim, hence the full /v1/traces path rather than a base URL OTEL_SERVICE_NAME read straight from os.environ by the library OTEL_RESOURCE_ATTRIBUTES read by the SDK's OTELResourceDetector OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT read by the SDK when it builds SpanLimits OTEL_TRACES_SAMPLER read by TracerProvider's default sampler NOT SET: OTEL_PROPAGATORS -- the library calls set_global_textmap() with an explicit CompositePropagator, which overrides it. NOT SET: OTEL_EXPORTER_OTLP_PROTOCOL -- consumed by the opentelemetry-instrument agent, not by the http exporter the library constructs directly. NOT SET: OTEL_METRIC_EXPORT_INTERVAL -- there is no MeterProvider until mitodl/ol-django#553 ships, and no metrics endpoint until the task that owns OTEL_EXPORTER_OTLP_ENDPOINT runs. Both belong to that change, together. Also NOT SET: deployment.environment. The library passes it to Resource.create() from settings.ENVIRONMENT, and Resource.create merges explicit attributes over env-detected ones, so an OTEL_RESOURCE_ATTRIBUTES value for that key never reaches a span. Verified in QA Tempo: learn-webapp, learn-ai-webapp and mitxonline-webapp all report deployment.environment="rc" (their app-level ENVIRONMENT) despite every one of those stacks setting "qa". Filed separately rather than copied forward. ── parentbased_always_on, NOT the 0.25 ratio the older stacks use ── Alloy already tail-samples (keep errors, keep >5000ms, 15% of the rest -- substructure/aws/eks/grafana.py). Head sampling in front of that multiplies with it instead of composing: a trace is dropped by the coin flip before any policy can look at whether it errored. This is the same reasoning as #5487 (witan) and #5471 (mit-learn-nextjs); the house pattern is stated in components/services/apisix.py, where the gateway runs always_on. Narrower in practice than it looks: APISIX propagates a sampled traceparent and `parentbased_` honours it, so anything arriving through the gateway was already at 100%. The ratio would only ever have applied to traces these apps root themselves -- celery tasks and cron -- which is exactly where full capture is worth most. ── CI IS DELIBERATELY DARK ── setup_grafana returns early for `ci`, so no Alloy runs there and grafana-k8s-monitoring-alloy-receiver does not resolve. Setting the endpoint uniformly is not free: it buys a connection failure per span batch, forever, in the environment nobody watches. ocw_studio gets the block only in Pulumi.{QA,Production}.yaml; OVS, which has no `vars:` config block and builds every env var in its program, gets an explicit env_suffix guard instead. ── PREVIEW ── odl_video_service QA: 3 to update (webapp, celery worker, celery beat), 133 unchanged. CI: no OTel env in the diff, confirming the guard. ocw_studio QA: 6 to update, 1 replace -- the replace is the pre-deploy Job, which is immutable, so any env change replaces it. micromasters and xpro are the other two apps in this epic. Their stack configs are deliberately not included: both repos currently have an unsatisfiable uv.lock on master, so neither can add the instrumentors this depends on. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01E5y3R8zEkr2A26X3Y43aBi --- .../ocw_studio/Pulumi.Production.yaml | 5 +++ .../applications/ocw_studio/Pulumi.QA.yaml | 5 +++ .../odl_video_service/__main__.py | 36 +++++++++++++++++++ 3 files changed, 46 insertions(+) diff --git a/src/ol_infrastructure/applications/ocw_studio/Pulumi.Production.yaml b/src/ol_infrastructure/applications/ocw_studio/Pulumi.Production.yaml index 212dde312b..4d7bf0ecb7 100644 --- a/src/ol_infrastructure/applications/ocw_studio/Pulumi.Production.yaml +++ b/src/ol_infrastructure/applications/ocw_studio/Pulumi.Production.yaml @@ -40,6 +40,11 @@ config: OCW_STUDIO_LIVE_URL: https://ocw.mit.edu/ OCW_STUDIO_LOG_LEVEL: INFO OCW_STUDIO_SUPPORT_EMAIL: 'ocw-studio-support@mit.edu' + OPENTELEMETRY_ENDPOINT: "http://grafana-k8s-monitoring-alloy-receiver.grafana.svc.cluster.local:4318/v1/traces" + OTEL_RESOURCE_ATTRIBUTES: "service.namespace=ocw-studio,service.instance.id=$(KUBERNETES_POD_NAME)" + OTEL_SERVICE_NAME: "ocw-studio-webapp" + OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT: "128" + OTEL_TRACES_SAMPLER: "parentbased_always_on" OPEN_CATALOG_URLS: 'https://open.mit.edu/api/v0/ocw_next_webhook/,https://api.learn.mit.edu/api/v1/ocw_next_webhook/' POST_TRANSCODE_ACTIONS: 'videos.api.update_video_job' PUBLISH_POSTHOG_API_HOST: https://ph.ol.mit.edu diff --git a/src/ol_infrastructure/applications/ocw_studio/Pulumi.QA.yaml b/src/ol_infrastructure/applications/ocw_studio/Pulumi.QA.yaml index b1adf5356d..79552ea4dc 100644 --- a/src/ol_infrastructure/applications/ocw_studio/Pulumi.QA.yaml +++ b/src/ol_infrastructure/applications/ocw_studio/Pulumi.QA.yaml @@ -44,6 +44,11 @@ config: OCW_STUDIO_LIVE_URL: https://live-qa.ocw.mit.edu/ OCW_STUDIO_LOG_LEVEL: INFO OCW_STUDIO_SUPPORT_EMAIL: 'ocw-studio-rc-support@mit.edu' + OPENTELEMETRY_ENDPOINT: "http://grafana-k8s-monitoring-alloy-receiver.grafana.svc.cluster.local:4318/v1/traces" + OTEL_RESOURCE_ATTRIBUTES: "service.namespace=ocw-studio,service.instance.id=$(KUBERNETES_POD_NAME)" + OTEL_SERVICE_NAME: "ocw-studio-webapp" + OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT: "128" + OTEL_TRACES_SAMPLER: "parentbased_always_on" OPEN_CATALOG_URLS: 'https://discussions-rc.odl.mit.edu/api/v0/ocw_next_webhook/,https://api.rc.learn.mit.edu/api/v1/ocw_next_webhook/' PUBLISH_POSTHOG_ENABLED: 'true' PUBLISH_POSTHOG_API_HOST: https://ph.ol.mit.edu diff --git a/src/ol_infrastructure/applications/odl_video_service/__main__.py b/src/ol_infrastructure/applications/odl_video_service/__main__.py index 73d8fbae30..7b96d12e1a 100644 --- a/src/ol_infrastructure/applications/odl_video_service/__main__.py +++ b/src/ol_infrastructure/applications/odl_video_service/__main__.py @@ -816,6 +816,42 @@ } app_env_vars.update(k8s_extra_vars) +# OVS has no `vars:` block in its stack configs -- every non-secret env var is +# literal here -- so unlike micromasters/xpro/ocw_studio the OTel block lives in +# the program rather than in Pulumi.{QA,Production}.yaml. +# +# Gated on CI because `setup_grafana` (substructure/aws/eks/grafana.py) returns +# early for `ci`, so no Grafana Alloy runs there and +# grafana-k8s-monitoring-alloy-receiver does not resolve. Setting the endpoint +# uniformly is not free: it buys a connection failure per span batch, forever, +# in the one environment nobody watches. +# +# OTEL_SERVICE_NAME and OTEL_RESOURCE_ATTRIBUTES are read straight from the +# environment; OPENTELEMETRY_ENDPOINT is a Django setting odl_video/settings.py +# reads, which is why it carries the full /v1/traces path rather than a base URL +# the SDK would append to. +if stack_info.env_suffix != "ci": + app_env_vars.update( + { + "OPENTELEMETRY_ENDPOINT": ( + "http://grafana-k8s-monitoring-alloy-receiver.grafana.svc" + ".cluster.local:4318/v1/traces" + ), + "OTEL_RESOURCE_ATTRIBUTES": ( + f"service.namespace={ovs_namespace}," + "service.instance.id=$(KUBERNETES_POD_NAME)" + ), + "OTEL_SERVICE_NAME": "ovs-webapp", + "OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT": "128", + # Alloy tail-samples (keep errors, keep >5000ms, 15% of the rest -- + # substructure/aws/eks/grafana.py), so head sampling here would + # multiply with it rather than compose. `parentbased_` still honours + # an inbound "not sampled" so a trace crossing from APISIX stays + # coherent. + "OTEL_TRACES_SAMPLER": "parentbased_always_on", + } + ) + # Unconditionally append k8s labels to OTEL_RESOURCE_ATTRIBUTES so all telemetry # signals carry organizational metadata regardless of stack environment. merge_otel_resource_attributes(app_env_vars, k8s_app_labels)