diff --git a/src/ol_infrastructure/applications/ocw_studio/Pulumi.Production.yaml b/src/ol_infrastructure/applications/ocw_studio/Pulumi.Production.yaml index 212dde312b..4d7bf0ecb7 100644 --- a/src/ol_infrastructure/applications/ocw_studio/Pulumi.Production.yaml +++ b/src/ol_infrastructure/applications/ocw_studio/Pulumi.Production.yaml @@ -40,6 +40,11 @@ config: OCW_STUDIO_LIVE_URL: https://ocw.mit.edu/ OCW_STUDIO_LOG_LEVEL: INFO OCW_STUDIO_SUPPORT_EMAIL: 'ocw-studio-support@mit.edu' + OPENTELEMETRY_ENDPOINT: "http://grafana-k8s-monitoring-alloy-receiver.grafana.svc.cluster.local:4318/v1/traces" + OTEL_RESOURCE_ATTRIBUTES: "service.namespace=ocw-studio,service.instance.id=$(KUBERNETES_POD_NAME)" + OTEL_SERVICE_NAME: "ocw-studio-webapp" + OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT: "128" + OTEL_TRACES_SAMPLER: "parentbased_always_on" OPEN_CATALOG_URLS: 'https://open.mit.edu/api/v0/ocw_next_webhook/,https://api.learn.mit.edu/api/v1/ocw_next_webhook/' POST_TRANSCODE_ACTIONS: 'videos.api.update_video_job' PUBLISH_POSTHOG_API_HOST: https://ph.ol.mit.edu diff --git a/src/ol_infrastructure/applications/ocw_studio/Pulumi.QA.yaml b/src/ol_infrastructure/applications/ocw_studio/Pulumi.QA.yaml index b1adf5356d..79552ea4dc 100644 --- a/src/ol_infrastructure/applications/ocw_studio/Pulumi.QA.yaml +++ b/src/ol_infrastructure/applications/ocw_studio/Pulumi.QA.yaml @@ -44,6 +44,11 @@ config: OCW_STUDIO_LIVE_URL: https://live-qa.ocw.mit.edu/ OCW_STUDIO_LOG_LEVEL: INFO OCW_STUDIO_SUPPORT_EMAIL: 'ocw-studio-rc-support@mit.edu' + OPENTELEMETRY_ENDPOINT: "http://grafana-k8s-monitoring-alloy-receiver.grafana.svc.cluster.local:4318/v1/traces" + OTEL_RESOURCE_ATTRIBUTES: "service.namespace=ocw-studio,service.instance.id=$(KUBERNETES_POD_NAME)" + OTEL_SERVICE_NAME: "ocw-studio-webapp" + OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT: "128" + OTEL_TRACES_SAMPLER: "parentbased_always_on" OPEN_CATALOG_URLS: 'https://discussions-rc.odl.mit.edu/api/v0/ocw_next_webhook/,https://api.rc.learn.mit.edu/api/v1/ocw_next_webhook/' PUBLISH_POSTHOG_ENABLED: 'true' PUBLISH_POSTHOG_API_HOST: https://ph.ol.mit.edu diff --git a/src/ol_infrastructure/applications/odl_video_service/__main__.py b/src/ol_infrastructure/applications/odl_video_service/__main__.py index 73d8fbae30..7b96d12e1a 100644 --- a/src/ol_infrastructure/applications/odl_video_service/__main__.py +++ b/src/ol_infrastructure/applications/odl_video_service/__main__.py @@ -816,6 +816,42 @@ } app_env_vars.update(k8s_extra_vars) +# OVS has no `vars:` block in its stack configs -- every non-secret env var is +# literal here -- so unlike micromasters/xpro/ocw_studio the OTel block lives in +# the program rather than in Pulumi.{QA,Production}.yaml. +# +# Gated on CI because `setup_grafana` (substructure/aws/eks/grafana.py) returns +# early for `ci`, so no Grafana Alloy runs there and +# grafana-k8s-monitoring-alloy-receiver does not resolve. Setting the endpoint +# uniformly is not free: it buys a connection failure per span batch, forever, +# in the one environment nobody watches. +# +# OTEL_SERVICE_NAME and OTEL_RESOURCE_ATTRIBUTES are read straight from the +# environment; OPENTELEMETRY_ENDPOINT is a Django setting odl_video/settings.py +# reads, which is why it carries the full /v1/traces path rather than a base URL +# the SDK would append to. +if stack_info.env_suffix != "ci": + app_env_vars.update( + { + "OPENTELEMETRY_ENDPOINT": ( + "http://grafana-k8s-monitoring-alloy-receiver.grafana.svc" + ".cluster.local:4318/v1/traces" + ), + "OTEL_RESOURCE_ATTRIBUTES": ( + f"service.namespace={ovs_namespace}," + "service.instance.id=$(KUBERNETES_POD_NAME)" + ), + "OTEL_SERVICE_NAME": "ovs-webapp", + "OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT": "128", + # Alloy tail-samples (keep errors, keep >5000ms, 15% of the rest -- + # substructure/aws/eks/grafana.py), so head sampling here would + # multiply with it rather than compose. `parentbased_` still honours + # an inbound "not sampled" so a trace crossing from APISIX stays + # coherent. + "OTEL_TRACES_SAMPLER": "parentbased_always_on", + } + ) + # Unconditionally append k8s labels to OTEL_RESOURCE_ATTRIBUTES so all telemetry # signals carry organizational metadata regardless of stack environment. merge_otel_resource_attributes(app_env_vars, k8s_app_labels)