diff --git a/hugo/config/_default/menus/main.en.yaml b/hugo/config/_default/menus/main.en.yaml
index 3dc8f396f44..10ec4da44dd 100644
--- a/hugo/config/_default/menus/main.en.yaml
+++ b/hugo/config/_default/menus/main.en.yaml
@@ -6876,11 +6876,6 @@ menu:
parent: observability_pipelines_sources
identifier: observability_pipelines_sources_opentelemetry
weight: 214
- - name: Prometheus
- url: observability_pipelines/sources/prometheus/
- parent: observability_pipelines_sources
- identifier: observability_pipelines_sources_prometheus
- weight: 215
- name: Kafka
url: observability_pipelines/sources/kafka/
parent: observability_pipelines_sources
@@ -7046,11 +7041,6 @@ menu:
parent: observability_pipelines_processors
identifier: observability_pipelines_processors_tag_cardinality_control
weight: 322
- - name: Tail-Based Sampling
- url: observability_pipelines/processors/tail_based_sampling
- parent: observability_pipelines_processors
- identifier: observability_pipelines_processors_tail_based_sampling
- weight: 323
- name: Throttle
url: observability_pipelines/processors/throttle
parent: observability_pipelines_processors
@@ -7096,11 +7086,6 @@ menu:
url: observability_pipelines/destinations/databricks/
parent: observability_pipelines_destinations
weight: 407
- - name: Datadog APM
- identifier: observability_pipelines_datadog_apm
- url: observability_pipelines/destinations/datadog_apm/
- parent: observability_pipelines_destinations
- weight: 408
- name: Datadog Archives
identifier: observability_pipelines_destinations_datadog_archives
url: observability_pipelines/destinations/datadog_archives/
@@ -7166,16 +7151,6 @@ menu:
parent: observability_pipelines_destinations
identifier: observability_pipelines_opensearch
weight: 421
- - name: OpenTelemetry
- url: observability_pipelines/destinations/opentelemetry/metrics/
- parent: observability_pipelines_destinations
- identifier: observability_pipelines_destinations_opentelemetry
- weight: 422
- - name: Prometheus
- url: /observability_pipelines/destinations/prometheus/
- parent: observability_pipelines_destinations
- identifier: observability_pipelines_prometheus
- weight: 423
- name: SentinelOne
url: observability_pipelines/destinations/sentinelone
parent: observability_pipelines_destinations
diff --git a/hugo/content/en/data_observability/jobs_monitoring/airflow_troubleshooting_dag.md b/hugo/content/en/data_observability/jobs_monitoring/airflow_troubleshooting_dag.md
index cfb859c8b44..f418d727058 100644
--- a/hugo/content/en/data_observability/jobs_monitoring/airflow_troubleshooting_dag.md
+++ b/hugo/content/en/data_observability/jobs_monitoring/airflow_troubleshooting_dag.md
@@ -26,7 +26,7 @@ The DAG checks the following:
| Package installation | OpenLineage package is installed and importable |
| Package version | Installed version compared to the latest available on PyPI |
| Listener accessibility | OpenLineage plugin listener can be loaded by Airflow |
-| Enabled state | OpenLineage is not disabled by environment variable or Airflow config |
+| Provider active | OpenLineage provider is active and not explicitly disabled |
| Transport configuration | A valid transport is configured (HTTP or Datadog) |
| Datadog endpoint | Transport URL points to a Datadog intake endpoint |
| Network connectivity | TCP connection to the configured backend URL succeeds |
@@ -76,14 +76,15 @@ validation_results = {
"is_astronomer": None,
"is_datadog": None,
"is_listener_accessible": None,
- "is_disabled": None,
+ "provider_active": None,
+ "inactive_reason": None, # "explicit" | "no_config" | None
"config_path": None,
"transport": None,
"transport_type": None,
"transport_config": None,
"transport_url": None,
"conflicts": [],
- "connectivity": None
+ "connectivity": None,
}
@@ -91,54 +92,30 @@ def generate_validation_summary():
"""Generate a summary of all validation checks performed."""
log.info("===== OpenLineage Validation Summary =====")
+ # --- Installation ---
if validation_results["installed_package"]:
log.info("✓ OpenLineage Package: %s version %s",
validation_results["installed_package"],
validation_results["package_version"])
else:
log.error("✗ OpenLineage not installed properly")
+ log.info("========================================")
+ log.error("Critical issues found. OpenLineage events will not be sent properly.")
+ return False
- if validation_results["is_disabled"]:
- log.error("✗ OpenLineage is disabled")
- else:
- log.info("✓ OpenLineage is enabled")
-
- if validation_results["is_listener_accessible"]:
- log.info("✓ OpenLineage listener is accessible")
- else:
- log.error("✗ OpenLineage listener is not accessible")
-
- if validation_results["transport"] and validation_results["transport_config"]:
- config = validation_results["transport_config"]
- transport_type = validation_results["transport_type"]
-
- if transport_type == "http":
- log.info("✓ Transport Type: HTTP")
- elif transport_type == "datadog":
- log.info("✓ Transport Type: Datadog")
- elif transport_type == "console":
- log.error("✗ Transport Type: Console (won't send events to Datadog)")
- elif transport_type == "composite":
- has_http_transport = False
- for name, nested_transports in config.get("transports", {}).items():
- if nested_transports.get("type", "") == "http":
- has_http_transport = True
- log.info("✓ Composite Transport with HTTP transport: `%s`", name)
- if not has_http_transport:
- log.error("✗ Composite Transport is set up without HTTP transport")
+ # --- Provider Status ---
+ if validation_results["provider_active"] is True:
+ log.info("✓ OpenLineage provider is active")
+ elif validation_results["provider_active"] is False:
+ if validation_results["inactive_reason"] == "explicit":
+ log.error("✗ OpenLineage provider is turned off")
+ log.error(" Check and remove whichever of these is set: "
+ "AIRFLOW__OPENLINEAGE__DISABLED, OPENLINEAGE_DISABLED, "
+ "or openlineage.disabled in airflow.cfg")
else:
- log.error("✗ Unknown transport type: %s", transport_type)
-
- if validation_results.get("is_datadog"):
- log.info("✓ Integration: Datadog")
- else:
- log.error("✗ No transport configured")
-
- if validation_results["connectivity"]:
- log.info("✓ Network connectivity to backend is successful")
- else:
- log.error("✗ Network connectivity check failed")
+ log.error("✗ OpenLineage provider is not active (no transport configuration found)")
+ # --- Configuration (always runs) ---
if validation_results["conflicts"]:
log.warning("! Configuration conflicts detected:")
for conflict in validation_results["conflicts"]:
@@ -146,18 +123,65 @@ def generate_validation_summary():
else:
log.info("✓ No configuration conflicts detected")
+ # --- Live Transport, Connectivity, Listener ---
+ # These only ran when the provider was active — group them and propagate N/A together.
+ if validation_results["provider_active"] is False:
+ log.warning("- Live transport: N/A (provider not active — fix provider status first)")
+ log.warning("- Network: N/A (provider not active — fix provider status first)")
+ log.warning("- Listener: N/A (provider not active — fix provider status first)")
+ else:
+ if validation_results["transport"] and validation_results["transport_config"]:
+ config = validation_results["transport_config"]
+ transport_type = validation_results["transport_type"]
+
+ if transport_type == "http":
+ log.info("✓ Transport Type: HTTP")
+ elif transport_type == "datadog":
+ log.info("✓ Transport Type: Datadog")
+ elif transport_type == "console":
+ log.error("✗ Transport Type: Console (won't send events to Datadog)")
+ elif transport_type == "composite":
+ has_http_transport = False
+ for name, nested_transports in config.get("transports", {}).items():
+ if nested_transports.get("type", "") == "http":
+ has_http_transport = True
+ log.info("✓ Composite Transport with HTTP transport: `%s`", name)
+ if not has_http_transport:
+ log.error("✗ Composite Transport is set up without HTTP transport")
+ else:
+ log.error("✗ Unknown transport type: %s", transport_type)
+
+ if validation_results.get("is_datadog"):
+ log.info("✓ Integration: Datadog")
+ else:
+ log.warning("! Transport does not appear to point to a Datadog endpoint")
+ else:
+ log.error("✗ Failed to resolve active transport")
+
+ if validation_results["connectivity"] is True:
+ log.info("✓ Network connectivity to backend is successful")
+ elif validation_results["connectivity"] is False:
+ log.error("✗ Network connectivity check failed")
+
+ if validation_results["is_listener_accessible"] is True:
+ log.info("✓ OpenLineage listener is accessible")
+ elif validation_results["is_listener_accessible"] is False:
+ log.error("✗ OpenLineage listener is not accessible")
+
+ # --- Platform Info ---
if validation_results["is_mwaa"]:
log.info("ℹ Running on Amazon MWAA")
-
if validation_results.get("is_astronomer"):
log.info("ℹ Running on Astronomer")
log.info("========================================")
critical_error = (
- validation_results["is_disabled"] or
- not validation_results["is_listener_accessible"] or
- not validation_results["transport"]
+ validation_results["provider_active"] is False
+ or validation_results["is_listener_accessible"] is False
+ or (validation_results["provider_active"] and not validation_results["transport"])
+ or validation_results["connectivity"] is False
+ or validation_results["transport_type"] == "console"
)
if critical_error:
@@ -175,9 +199,9 @@ def print_environment_info():
ol_python_ver = _get_installed_package_version("openlineage-python")
if ol_python_ver:
- log.info(f"OpenLineage Python Version: {ol_python_ver}")
+ log.info(f"OpenLineage Python Version: {ol_python_ver}")
else:
- log.info("OpenLineage Python Version: Not Found")
+ log.info("OpenLineage Python Version: Not Found")
if _provider_can_be_used():
provider = "apache-airflow-providers-openlineage"
@@ -190,7 +214,6 @@ def print_environment_info():
else:
log.info(f"OpenLineage Provider Version: Not Found ({provider})")
- # Run platform checks
check_mwaa_status()
check_astronomer_status()
@@ -201,32 +224,35 @@ def validate_setup() -> None:
"""Run all validation checks for OpenLineage configuration."""
log.info("Starting OpenLineage validation...")
- # Print environment info
+ # 1. Environment info
print_environment_info()
- # Check package installation
- if _provider_can_be_used():
- validate_installation("apache-airflow-providers-openlineage")
- else:
- validate_installation("openlineage-airflow")
+ # 2. Installation check — prerequisite for everything else
+ package_name = "apache-airflow-providers-openlineage" if _provider_can_be_used() else "openlineage-airflow"
+ if not validate_installation(package_name):
+ generate_validation_summary()
+ return
- # Check listener and validation
- is_listener_accessible()
- is_ol_disabled()
+ # 3. Disabled check
+ check_provider_enabled()
- # Check for configuration conflicts
+ # 4. Static transport checks — env vars, config files, transport source logging.
+ # No OL dependency, always run regardless of disabled state.
check_configuration_conflicts()
+ validate_transport_config()
+
+ # 5+6. Live transport resolution, network connectivity, and listener check all require
+ # the provider to be active. When inactive, the plugin registers no listeners, so all
+ # three would fail for the same upstream reason — skip them as a group.
+ if validation_results["provider_active"]:
+ resolve_transport()
+ check_network_connectivity()
+ is_listener_accessible()
+ else:
+ log.warning("Skipping live transport resolution, network connectivity, and listener check: "
+ "provider is not active. Address the transport configuration first.")
- # Check connection configuration
- validate_connection()
-
- # Check for Datadog
- check_is_datadog()
-
- # Check network connectivity (using transport URL from previous steps)
- check_network_connectivity()
-
- # Generate validation summary
+ # 7. Summary
generate_validation_summary()
@@ -237,7 +263,7 @@ try:
start_date=datetime.datetime(2025, 1, 1),
schedule_interval="@once",
)
-except Exception as e:
+except Exception:
dag = DAG(
dag_id="openlineage_preflight_check_dag",
description="A DAG to check OpenLineage setup and configurations",
@@ -245,7 +271,8 @@ except Exception as e:
schedule="@once",
)
-validate_setup = PythonOperator(
+# Named differently from validate_setup() to avoid overwriting the function reference
+validate_setup_task = PythonOperator(
task_id="validate_setup",
python_callable=validate_setup,
dag=dag,
@@ -259,11 +286,10 @@ def validate_installation(package_name: str) -> bool:
log.error(f"Failed to get installed version for `{package_name}`. Skipping version check.")
return False
- # Store in global results
validation_results["installed_package"] = package_name
validation_results["package_version"] = str(package_version)
- except Exception as e:
+ except Exception:
log.exception(f"Failed to get installed version for `{package_name}`.")
return False
@@ -298,7 +324,7 @@ def check_mwaa_status():
'MWAA_COMMAND': os.getenv('MWAA_COMMAND'),
'AIRFLOW_ENV_NAME': os.getenv('AIRFLOW_ENV_NAME'),
'AWS_REGION': os.getenv('AWS_REGION'),
- 'AIRFLOW_VERSION': os.getenv('AIRFLOW_VERSION')
+ 'AIRFLOW_VERSION': os.getenv('AIRFLOW_VERSION'),
}
log.info(f"MWAA Environment Details: {mwaa_env}")
else:
@@ -329,7 +355,7 @@ def is_listener_accessible():
if _provider_can_be_used():
try:
from airflow.providers.openlineage.plugins.openlineage import OpenLineageProviderPlugin as plugin
- except ImportError as e:
+ except ImportError:
log.error("OpenLineage provider is not accessible: can't import airflow.providers.openlineage.plugins.openlineage.OpenLineageProviderPlugin")
log.error("Please check if the provider is properly configured.")
log.error("The installation docs can be found at https://docs.datadoghq.com/data_jobs/airflow/")
@@ -338,70 +364,84 @@ def is_listener_accessible():
else:
try:
from openlineage.airflow.plugin import OpenLineagePlugin as plugin
- except ImportError as e:
+ except ImportError:
log.error("OpenLineage is not accessible: can't import openlineage.airflow.plugin.OpenLineagePlugin")
log.error("Please check if the provider is properly configured.")
log.error("The installation docs can be found at https://docs.datadoghq.com/data_jobs/airflow/")
validation_results["is_listener_accessible"] = False
return False
- if len(plugin.listeners) == 1:
- validation_results["is_listener_accessible"] = True
- return True
+ num_listeners = len(plugin.listeners)
+ if num_listeners == 0:
+ log.error("OpenLineage listener is not registered. The plugin loaded but no listeners are active.")
+ validation_results["is_listener_accessible"] = False
+ return False
+ elif num_listeners > 1:
+ log.error("OpenLineage has unexpected multiple listeners registered: %s", plugin.listeners)
+ validation_results["is_listener_accessible"] = False
+ return False
- log.error("OpenLineage is not accessible: multiple listeners found. %s", plugin.listeners)
- validation_results["is_listener_accessible"] = False
- return False
+ validation_results["is_listener_accessible"] = True
+ return True
-def is_ol_disabled():
+def check_provider_enabled():
if _provider_can_be_used():
try:
- # apache-airflow-providers-openlineage >= 1.7.0
from airflow.providers.openlineage.conf import is_disabled
except ImportError:
- # apache-airflow-providers-openlineage < 1.7.0
from airflow.providers.openlineage.plugins.openlineage import _is_disabled as is_disabled
else:
from openlineage.airflow.plugin import _is_disabled as is_disabled
- is_disabled_result = is_disabled()
- validation_results["is_disabled"] = is_disabled_result
+ is_inactive = is_disabled()
+ validation_results["provider_active"] = not is_inactive
+
+ if not is_inactive:
+ return
+
+ # Determine whether it was explicitly turned off or just has no transport config
+ if _provider_can_be_used() and os.getenv("AIRFLOW__OPENLINEAGE__DISABLED", "false").lower() == "true":
+ log.error("OpenLineage provider is turned off via AIRFLOW__OPENLINEAGE__DISABLED")
+ validation_results["inactive_reason"] = "explicit"
+ return
+ if conf.getboolean("openlineage", "disabled", fallback=False):
+ log.error("OpenLineage provider is turned off via openlineage.disabled in airflow.cfg")
+ validation_results["inactive_reason"] = "explicit"
+ return
+ if os.getenv("OPENLINEAGE_DISABLED", "false").lower() == "true":
+ log.error("OpenLineage provider is turned off via OPENLINEAGE_DISABLED")
+ validation_results["inactive_reason"] = "explicit"
+ return
+
+ # No explicit flag — provider is inactive because no transport config was found
+ log.error(
+ "OpenLineage provider is not active: no transport configuration was found. "
+ "See https://airflow.apache.org/docs/apache-airflow-providers-openlineage/stable/configurations-ref.html"
+ )
+ validation_results["inactive_reason"] = "no_config"
- if is_disabled_result:
- if _provider_can_be_used() and os.getenv("AIRFLOW__OPENLINEAGE__DISABLED", "false").lower() == "true":
- log.error("OpenLineage is disabled in Airflow Config by environment variable AIRFLOW__OPENLINEAGE__DISABLED")
- return True
- elif conf.getboolean("openlineage", "disabled", fallback=False):
- log.error("OpenLineage is disabled in Airflow Config: openlineage.disabled")
- return True
- elif os.getenv("OPENLINEAGE_DISABLED", "false").lower() == "true":
- log.error(
- "OpenLineage is disabled due to the environment variable OPENLINEAGE_DISABLED"
- )
- return True
- log.error(
- "OpenLineage is disabled because required config/env variables are not set. "
- "Please refer to "
- "https://airflow.apache.org/docs/apache-airflow-providers-openlineage/stable/guides/user.html"
- )
- return True
- return False
+def validate_transport_config() -> None:
+ """Check static transport configuration — env vars, config files, transport source logging.
-def validate_connection() -> bool:
- """Validate the OpenLineage connection configuration."""
+ No OL package dependency; safe to run regardless of disabled state.
+ """
_validate_config_set()
- config_files = [
- "openlineage.yml",
- "~/.openlineage/openlineage.yml"
- ]
-
- for file_path in config_files:
+ for file_path in ["openlineage.yml", "~/.openlineage/openlineage.yml"]:
if _check_openlineage_yml(file_path):
break
+ _verify_transport_source()
+
+
+def resolve_transport() -> bool:
+ """Instantiate the OL plugin to get the active transport object and validate it.
+
+ Requires OL to be enabled — listeners list is empty when disabled, so this will
+ always fail if called in that state. Gate this call in validate_setup().
+ """
try:
transport = _get_configured_transport()
if transport is None:
@@ -416,45 +456,50 @@ def validate_connection() -> bool:
validation_results["transport_config"] = config
if transport.kind == "http":
- validation_results["transport_url"] = config.get("url")
+ url = config.get("url")
+ validation_results["transport_url"] = url
+ validation_results["is_datadog"] = _is_datadog_url(url) if url else False
+ if url and not validation_results["is_datadog"]:
+ log.warning("HTTP transport URL does not point to a known Datadog endpoint: %s", url)
+ elif transport.kind == "datadog":
+ validation_results["is_datadog"] = True
elif transport.kind == "composite":
transport_valid = False
for key, value in config.get("transports", {}).items():
log.info("Checking nested transport `%s`", key)
transport_valid = _verify_transport(value)
if value.get("type") == "http":
- validation_results["transport_url"] = value.get("url")
+ url = value.get("url")
+ validation_results["transport_url"] = url
+ validation_results["is_datadog"] = _is_datadog_url(url) if url else False
break
return transport_valid
- _verify_transport_source()
return True
-
except Exception as e:
- log.error("There was an error when trying to validate connection: %s", e)
+ log.error("There was an error when trying to resolve transport: %s", e)
log.exception("Full traceback:")
return False
def _validate_config_set():
- if config_path := os.getenv("OPENLINEAGE_CONFIG"):
- log.info("Found OpenLineage config path: env variable OPENLINEAGE_CONFIG is set to: %s", config_path)
- validation_results["config_path"] = config_path
- elif config_path := os.getenv("AIRFLOW__OPENLINEAGE__CONFIG", "") and _provider_can_be_used():
- log.info("Found OpenLineage config path: env variable AIRFLOW__OPENLINEAGE__CONFIG is set to: %s", config_path)
- validation_results["config_path"] = config_path
- elif config_path := conf.get("openlineage", "config_path", fallback="") and _provider_can_be_used():
- log.info("Found OpenLineage config path: Airflow config openlineage.config_path is set to: %s", config_path)
- validation_results["config_path"] = config_path
-
+ config_path = None
+
+ if env_path := os.getenv("OPENLINEAGE_CONFIG"):
+ log.info("Found OpenLineage config path: env variable OPENLINEAGE_CONFIG is set to: %s", env_path)
+ config_path = env_path
+ elif _provider_can_be_used() and (env_path := os.getenv("AIRFLOW__OPENLINEAGE__CONFIG")):
+ log.info("Found OpenLineage config path: env variable AIRFLOW__OPENLINEAGE__CONFIG is set to: %s", env_path)
+ config_path = env_path
+ elif _provider_can_be_used() and (cfg_path := conf.get("openlineage", "config_path", fallback="")):
+ log.info("Found OpenLineage config path: Airflow config openlineage.config_path is set to: %s", cfg_path)
+ config_path = cfg_path
if config_path:
+ validation_results["config_path"] = config_path
if not _check_openlineage_yml(config_path):
- log.error(
- "Config file is empty or does not exist: `%s`",
- config_path,
- )
+ log.error("Config file is empty or does not exist: `%s`", config_path)
return False
log.info("OpenLineage config file `%s` is valid.", config_path)
return True
@@ -463,41 +508,22 @@ def _validate_config_set():
return True
-def check_is_datadog():
- """Check if the transport is configured for Datadog."""
- # We rely on validate_connection being run first
- transport_type = validation_results.get("transport_type")
- transport_url = validation_results.get("transport_url", "")
-
- is_datadog = False
-
- if transport_type == "datadog":
- is_datadog = True
- elif transport_type == "http" and transport_url:
- # Check for known Datadog domains
- datadog_domains = [
- "datadoghq.com",
- "datadoghq.eu",
- "datad0g.com",
- "datad0g.eu",
- "us3.datadoghq.com",
- "us5.datadoghq.com",
- "ap1.datadoghq.com"
- ]
- if any(domain in transport_url for domain in datadog_domains):
- is_datadog = True
- log.info("Transport type is HTTP but URL points to Datadog. Considered as Datadog transport.")
-
- validation_results["is_datadog"] = is_datadog
- return is_datadog
+def _is_datadog_url(url: str) -> bool:
+ """Check if a URL points to a known Datadog domain."""
+ datadog_domains = [
+ "datadoghq.com",
+ "datadoghq.eu",
+ "us3.datadoghq.com",
+ "us5.datadoghq.com",
+ "ap1.datadoghq.com",
+ "ap2.datadoghq.com",
+ "uk1.datadoghq.com",
+ ]
+ return any(domain in url for domain in datadog_domains)
def _redact_api_keys(obj) -> None:
- """Recursively search and redact API keys in a dictionary.
-
- This function modifies the dictionary in place, redacting any values where
- the key contains 'api_key' (case insensitive).
- """
+ """Recursively redact API keys and auth values in a dictionary (in-place)."""
if isinstance(obj, dict):
for key, value in obj.items():
if isinstance(key, str) and ("api_key" in key.lower() or "auth" in key.lower()):
@@ -515,7 +541,7 @@ def _verify_transport_source() -> None:
if endpoint := os.getenv("OPENLINEAGE_ENDPOINT"):
url = urljoin(url, endpoint)
log.info("OPENLINEAGE_ENDPOINT is set to: `%s`", url)
- log.info("Final URL that is configured by env variables is set to: `%s`", url)
+ log.info("Final URL configured by env variables: `%s`", url)
if os.getenv("OPENLINEAGE_API_KEY"):
log.info("OPENLINEAGE_API_KEY is set [value redacted]")
@@ -549,7 +575,7 @@ def _verify_transport_source() -> None:
except json.JSONDecodeError:
log.error("AIRFLOW__OPENLINEAGE__TRANSPORT is set but contains invalid JSON: `%s`", transport_var)
else:
- log.info("AIRFLOW__OPENLINEAGE__TRANSPORT variable is not set.")
+ log.info("AIRFLOW__OPENLINEAGE__TRANSPORT is not set.")
for key, value in os.environ.items():
if key.startswith("AIRFLOW__OPENLINEAGE__TRANSPORT_"):
@@ -566,6 +592,13 @@ def _verify_transport_source() -> None:
else:
log.info("Airflow config openlineage.transport is not set.")
+ conn_id = os.getenv("AIRFLOW__OPENLINEAGE__CONN_ID") or conf.get("openlineage", "conn_id", fallback="")
+ if conn_id:
+ log.info("OpenLineage Airflow connection configured: conn_id=`%s`", conn_id)
+ _check_connection_transport(conn_id)
+ else:
+ log.info("No OpenLineage Airflow connection configured (AIRFLOW__OPENLINEAGE__CONN_ID / openlineage.conn_id).")
+
def _check_openlineage_yml(file_path) -> bool:
log.info("Checking OpenLineage config file: `%s`", file_path)
@@ -577,9 +610,9 @@ def _check_openlineage_yml(file_path) -> bool:
log.error(f"Empty openlineage.yml file: `{file_path}`")
return False
log.info(
- f"File found at `{file_path}` with the following content: `{content}`. "
- "Make sure the configuration is correct."
- )
+ f"File found at `{file_path}` with the following content: `{content}`. "
+ "Make sure the configuration is correct."
+ )
return True
return False
@@ -591,15 +624,47 @@ def _get_configured_transport():
transport = OpenLineageProviderPlugin().listeners[0].adapter.get_or_create_openlineage_client().transport
else:
from openlineage.airflow.plugin import OpenLineagePlugin
- transport = (
- OpenLineagePlugin.listeners[0].adapter.get_or_create_openlineage_client().transport
- )
+ transport = OpenLineagePlugin.listeners[0].adapter.get_or_create_openlineage_client().transport
except Exception as e:
log.error("There was an error when trying to get OpenLineage Transport: %s", e)
return None
return transport
+def _check_connection_transport(conn_id: str) -> str | None:
+ """Validate an OpenLineage Airflow connection and return its URL if resolvable."""
+ try:
+ from airflow.hooks.base import BaseHook
+ conn = BaseHook.get_connection(conn_id)
+
+ schema = conn.schema or "https"
+ host = conn.host
+ if not host:
+ log.error("Connection `%s` has no host configured", conn_id)
+ return None
+
+ url = f"{schema}://{host}"
+ if conn.port:
+ url += f":{conn.port}"
+
+ log.info("Connection `%s` URL: %s", conn_id, url)
+
+ if conn.password:
+ log.info("Connection `%s` has a password configured [value redacted]", conn_id)
+ else:
+ log.warning("Connection `%s` has no password configured", conn_id)
+
+ if _is_datadog_url(url):
+ log.info("Connection `%s` points to a Datadog endpoint", conn_id)
+ else:
+ log.warning("Connection `%s` does not appear to point to a Datadog endpoint", conn_id)
+
+ return url
+ except Exception as e:
+ log.error("Failed to retrieve Airflow connection `%s`: %s", conn_id, e)
+ return None
+
+
def _verify_transport(config: dict, name: str = ""):
if not config:
log.error("Empty transport configuration")
@@ -625,7 +690,7 @@ def _verify_transport(config: dict, name: str = ""):
valid_transports = 0
for i, transport_config in enumerate(transports):
- log.info("Checking nested transport #%d...", i+1)
+ log.info("Checking nested transport #%d...", i + 1)
if _verify_transport(transport_config):
valid_transports += 1
@@ -655,7 +720,6 @@ def _verify_http_backend(config: dict, name: str = ""):
return False
log.info("HTTP transport URL is configured: %s", config.get("url"))
- log.info("HTTP transport %s auth: %s", name, config.get("auth"))
if config.get("auth") is not None:
log.info("HTTP transport %s has API key authentication configured", name)
else:
@@ -703,9 +767,8 @@ def _is_mwaa_environment() -> bool:
'AIRFLOW_ENV_ID',
'AWS_EXECUTION_ENV',
'MWAA_AIRFLOW_COMPONENT',
- 'AIRFLOW_ENV_NAME'
+ 'AIRFLOW_ENV_NAME',
]
-
return any(var in os.environ for var in mwaa_indicators)
@@ -716,7 +779,6 @@ def _is_astronomer_environment() -> bool:
'ASTRONOMER_DEPLOYMENT_ID',
'ASTRONOMER_WORKSPACE_ID',
]
-
return any(var in os.environ for var in astro_indicators)
@@ -788,6 +850,11 @@ def check_configuration_conflicts():
if os.getenv("OPENLINEAGE_URL"):
transport_sources.append("OPENLINEAGE_URL environment variable")
+ if _provider_can_be_used():
+ conn_id = os.getenv("AIRFLOW__OPENLINEAGE__CONN_ID") or conf.get("openlineage", "conn_id", fallback="")
+ if conn_id:
+ transport_sources.append(f"Airflow connection (conn_id={conn_id})")
+
if len(transport_sources) > 1:
conflict_msg = "Multiple transport configurations found: " + ", ".join(transport_sources)
log.warning(conflict_msg)
@@ -855,23 +922,24 @@ This Airflow installation is running on Amazon MWAA
### Validation summary
-The validation summary appears at the end of the task log. Each line starts with one of three symbols:
+The validation summary appears at the end of the task log. Each line starts with one of the following symbols:
- `✓`: Check passed.
- `✗`: Check failed (events are not sent to Datadog).
- `!`: Warning (may indicate a configuration issue).
+- `-`: Check skipped (a prerequisite check failed).
The following output indicates a healthy setup:
```
===== OpenLineage Validation Summary =====
✓ OpenLineage Package: apache-airflow-providers-openlineage version 2.7.3
-✓ OpenLineage is enabled
-✓ OpenLineage listener is accessible
+✓ OpenLineage provider is active
+✓ No configuration conflicts detected
✓ Transport Type: Datadog
✓ Integration: Datadog
✓ Network connectivity to backend is successful
-✓ No configuration conflicts detected
+✓ OpenLineage listener is accessible
========================================
OpenLineage appears to be configured properly, but check the logs for warnings
```
@@ -882,12 +950,14 @@ Use this table to resolve common failures:
| Log message | Cause | Resolution |
|---|---|---|
-| `✗ OpenLineage not installed properly` | The OpenLineage package is missing or corrupted. | Confirm `apache-airflow-providers-openlineage` is included in your Airflow installation. For Amazon MWAA, see [Upgrade OpenLineage provider on Amazon MWAA][3]. |
-| `✗ OpenLineage is disabled` | `AIRFLOW__OPENLINEAGE__DISABLED=true` or `OPENLINEAGE_DISABLED=true` is set, or required transport configuration is missing. | Remove or set the disable variable to `false`, and verify that the transport is configured properly. |
-| `✗ OpenLineage listener is not accessible` | The provider plugin cannot be imported. | Confirm the package is installed on **both scheduler and worker** pods/processes. |
-| `✗ No transport configured` | No transport environment variables are set. | Follow the [Airflow setup guide][2] to configure a transport. |
+| `✗ OpenLineage not installed properly` | The OpenLineage package is missing or corrupted. All remaining checks are skipped. | Confirm `apache-airflow-providers-openlineage` is included in your Airflow installation. For Amazon MWAA, see [Upgrade OpenLineage provider on Amazon MWAA][3]. |
+| `✗ OpenLineage provider is turned off` | The provider is explicitly disabled by `AIRFLOW__OPENLINEAGE__DISABLED`, `OPENLINEAGE_DISABLED`, or `openlineage.disabled` in `airflow.cfg`. | Remove the setting, or set it to `false`. |
+| `✗ OpenLineage provider is not active (no transport configuration found)` | The provider deactivates itself when no transport configuration is present, even though nothing explicitly disabled it. | Follow the [Airflow setup guide][2] to configure a transport. |
+| `✗ Failed to resolve active transport` | The transport could not be instantiated from the resolved configuration. | Verify the transport configuration is valid and well-formed. Check the task log for the underlying error. |
| `✗ Transport Type: Console (won't send events to Datadog)` | Transport is set to `console`. | Change the transport to `datadog` or `http` pointing to the Datadog intake URL. |
+| `! Transport does not appear to point to a Datadog endpoint` | The transport URL does not match a known Datadog domain. | Verify the URL points to a Datadog intake endpoint. |
| `✗ Network connectivity check failed` | Airflow workers cannot reach the Datadog intake endpoint. | Check firewall or network policies; confirm the URL and port 443 are accessible. |
+| `✗ OpenLineage listener is not accessible` | The provider plugin cannot be imported, no listeners are registered, or multiple listeners are found. The specific cause is logged earlier in the task log. | Confirm the package is installed on **both scheduler and worker** pods or processes. If the plugin loaded but no listeners are active, check provider version compatibility. |
| `! Configuration conflicts detected` | Multiple transport or config file sources are active. | Remove duplicate configurations and keep one authoritative source. |
**Note**: If `✗ OpenLineage listener is not accessible` appears together with package installation failures, the package is likely installed only on the scheduler and not on the workers. OpenLineage requires the provider on both.
diff --git a/hugo/content/en/database_monitoring/setup_mongodb/selfhosted.md b/hugo/content/en/database_monitoring/setup_mongodb/selfhosted.md
index a9ecf4ababd..a2ddf94f9e0 100644
--- a/hugo/content/en/database_monitoring/setup_mongodb/selfhosted.md
+++ b/hugo/content/en/database_monitoring/setup_mongodb/selfhosted.md
@@ -186,6 +186,30 @@ Datadog recommends installing the Agent directly on the MongoDB host, as that en
{{% /tab %}}
{{< /tabs >}}
+## Query Metrics
+
+Query metrics for self-hosted MongoDB require MongoDB 8.0 or later and rely on the `$queryStats` aggregation pipeline. Query stats collection is disabled by default on the MongoDB server. To enable it, set the `internalQueryStatsSampleRate` server parameter to `1.0` (100% sampling) on each `mongod` or `mongos` process.
+
+Add the parameter to the MongoDB configuration file:
+
+{{< code-block lang="yaml" >}}
+setParameter:
+ internalQueryStatsSampleRate: 1.0
+{{< /code-block >}}
+
+Or pass it on the command line:
+
+{{< code-block lang="shell" >}}
+mongod --setParameter internalQueryStatsSampleRate=1.0
+{{< /code-block >}}
+
+To enable it at runtime without restarting (not persistent — resets on restart), run:
+
+{{< code-block lang="javascript" >}}
+db.adminCommand({setParameter: 1, internalQueryStatsSampleRate: 1.0})
+{{< /code-block >}}
+
+For more information about the server parameters that control query stats collection, see the [MongoDB Query Stats documentation][4].
## Data Collected
@@ -235,3 +259,4 @@ instances:
[1]: /account_management/api-app-keys/
[2]: /integrations/mongo/?tab=standalone#metrics
[3]: /database_monitoring/query_metrics/
+[4]: https://github.com/mongodb/mongo/blob/master/src/mongo/db/query/query_stats/README.md#server-parameters
diff --git a/hugo/content/en/observability_pipelines/_index.md b/hugo/content/en/observability_pipelines/_index.md
index 4d0d8e0e959..c326e2fd86d 100644
--- a/hugo/content/en/observability_pipelines/_index.md
+++ b/hugo/content/en/observability_pipelines/_index.md
@@ -1,6 +1,6 @@
---
title: Observability Pipelines
-description: Learn how Observability Pipelines lets you collect, process, and route logs, metrics, and traces within your own infrastructure to destinations such as Datadog, Amazon S3, Splunk, and Microsoft Sentinel.
+description: Learn how Observability Pipelines lets you collect, process, and route logs and metrics within your own infrastructure to destinations such as Datadog, Amazon S3, Splunk, and Microsoft Sentinel.
disable_toc: false
further_reading:
- link: "/observability_pipelines/configuration/explore_templates/"
@@ -78,7 +78,7 @@ further_reading:
{{< img src="observability_pipelines/op_marketecture_06042025.png" alt="A graphic showing data being aggregated from a variety of sources, processed and enriched by the observability pipelines worker in your own environment, and then being routed to the security, analytics, and storage destinations of your choice" style="width:100%;" >}}
-Datadog Observability Pipelines allows you to collect and process logs, metrics, and traces ({{< tooltip text="in Preview" tooltip="Traces Pipeline is in Preview. Contact your account manager to request access." >}}) within your own infrastructure, and then route the data to different destinations. It gives you control over your observability data before it leaves your environment.
+Datadog Observability Pipelines allows you to collect and process logs and metrics within your own infrastructure, and then route the data to different destinations. It gives you control over your observability data before it leaves your environment.
With out-of-the-box templates, you can build pipelines that redact sensitive data, enrich data, filter out noisy events, and route data to destinations like Datadog, SIEM tools, or cloud storage.
@@ -141,15 +141,6 @@ Observability Pipelines includes prebuilt templates for common data routing and
|----------|-------------|
| Metric Tag Governance | Manage the quality and volume of your metrics by keeping only the metrics you need, standardizing metrics tagging, and removing unwanted tags to prevent high cardinality. |
-{{% /tab %}}
-{{% tab "Traces" %}}
-
-Traces Pipeline is in Preview. Contact your account manager to request access.
-
-| Template | Description |
-|----------|-------------|
-| Trace Sampling | Ingest, process, and route traces to control costs while retaining the traces you need for troubleshooting and analysis. |
-
{{% /tab %}}
{{< /tabs >}}
diff --git a/hugo/content/en/observability_pipelines/configuration/_index.md b/hugo/content/en/observability_pipelines/configuration/_index.md
index f7c9f965624..0343c74ec3e 100644
--- a/hugo/content/en/observability_pipelines/configuration/_index.md
+++ b/hugo/content/en/observability_pipelines/configuration/_index.md
@@ -21,7 +21,7 @@ further_reading:
{{< img src="observability_pipelines/setup/pipeline_ui.png" alt="The pipelines page with a source going to two processors groups and two destinations" style="width:100%;" >}}
-Observability Pipelines lets you collect and process logs, metrics, and traces ({{< tooltip text="in Preview" tooltip="Traces Pipeline is in Preview. Contact your account manager to request access." >}}) within your own infrastructure, and then route them to different destinations. A pipeline consists of three core components:
+Observability Pipelines lets you collect and process logs and metrics within your own infrastructure, and then route them to different destinations. A pipeline consists of three core components:
- [Source][1]: Receives data from a tool like the Datadog Agent.
- [Processors][2]: Transform, enrich, or filter data.
@@ -121,19 +121,6 @@ See [Metric Types][3] for more information.
[3]: /metrics/types/?tab=gauge#metric-types
[4]: https://opentelemetry.io/docs/specs/otel/metrics/data-model/#temporality
-{{% /tab %}}
-
-{{% tab "Traces" %}}
-
-Traces Pipeline is in Preview. Contact your account manager to request access.
-
-You can ingest, process, and send traces to different destinations using the [Trace Sampling][1] template.
-
-See [Set Up Pipelines][2] for more information on setting up a source, processors, and destinations.
-
-[1]: /observability_pipelines/configuration/explore_templates/?tab=traces#trace-sampling
-[2]: /observability_pipelines/configuration/set_up_pipelines/
-
{{% /tab %}}
{{< /tabs >}}
diff --git a/hugo/content/en/observability_pipelines/configuration/explore_templates.md b/hugo/content/en/observability_pipelines/configuration/explore_templates.md
index 82a21856317..c608167e292 100644
--- a/hugo/content/en/observability_pipelines/configuration/explore_templates.md
+++ b/hugo/content/en/observability_pipelines/configuration/explore_templates.md
@@ -76,15 +76,6 @@ Metrics capture signals about your environment and offer insight into your syste
To help you manage the quality and volume of your metrics, use the Metric Tag Governance template to process them in Observability Pipelines before sending them to your destinations. You can use processors to keep only the metrics you need, standardize metrics tagging, and remove unwanted tags to prevent high cardinality.
-{{% /tab %}}
-{{% tab "Traces" %}}
-
-Traces Pipeline is in Preview. Contact your account manager to request access.
-
-### Trace Sampling
-
-Use the Trace Sampling template to ingest, process, and route your traces telemetry data to control costs while retaining the traces you need for troubleshooting and analysis.
-
{{% /tab %}}
{{< /tabs >}}
diff --git a/hugo/content/en/observability_pipelines/configuration/install_the_worker/_index.mdoc.md b/hugo/content/en/observability_pipelines/configuration/install_the_worker/_index.mdoc.md
index 8cd50d13c16..35731d80363 100644
--- a/hugo/content/en/observability_pipelines/configuration/install_the_worker/_index.mdoc.md
+++ b/hugo/content/en/observability_pipelines/configuration/install_the_worker/_index.mdoc.md
@@ -47,7 +47,7 @@ For RHEL and CentOS, the Observability Pipelines Worker supports versions 8.0 or
{% /if %}
-The Observability Pipelines Worker is software that runs in your environment to centrally aggregate and process your logs, metrics, and traces ({% tooltip contents="Traces Pipeline is in Preview. Contact your account manager to request access." %}in Preview{% /tooltip %}), and then route them to different destinations.
+The Observability Pipelines Worker is software that runs in your environment to centrally aggregate and process your logs and metrics, and then route them to different destinations.
{% if equals($platform, "kubernetes") %}
diff --git a/hugo/content/en/observability_pipelines/configuration/live_capture.md b/hugo/content/en/observability_pipelines/configuration/live_capture.md
index 34a11a95825..8c7eb3da1ef 100644
--- a/hugo/content/en/observability_pipelines/configuration/live_capture.md
+++ b/hugo/content/en/observability_pipelines/configuration/live_capture.md
@@ -21,9 +21,6 @@ products:
- name: Metrics
icon: metrics
url: /observability_pipelines/configuration/?tab=metrics#pipeline-types
-- name: Traces
- icon: apm
- url: /observability_pipelines/configuration/?tab=traces#pipeline-types
---
{{< product-availability >}}
diff --git a/hugo/content/en/observability_pipelines/configuration/set_up_pipelines.md b/hugo/content/en/observability_pipelines/configuration/set_up_pipelines.md
index e62dfa0065a..ce1dfd6b649 100644
--- a/hugo/content/en/observability_pipelines/configuration/set_up_pipelines.md
+++ b/hugo/content/en/observability_pipelines/configuration/set_up_pipelines.md
@@ -111,33 +111,6 @@ See [Export a Pipeline Configuration to JSON or Terraform][14] if you want to pr
[6]: /observability_pipelines/configuration/pipeline_simulation/
[11]: /observability_pipelines/search_syntax/metrics/
-{{% /tab %}}
-{{% tab "Traces" %}}
-
-Traces Pipeline is in Preview. Contact your account manager to request access.
-
-1. Navigate to [Observability Pipelines][1].
-1. Select the [Trace Sampling][2] template, or click {{< ui >}}New Pipeline{{< /ui >}} and select {{< ui >}}Traces Pipeline{{< /ui >}}.
-1. Select and set up a [trace source][3].
-1. Select and set up [destinations][5] for your processed traces.
-1. Click {{< ui >}}Edit{{< /ui >}} on a processor group to add, update, and validate your [processors][4] using [Pipeline Simulation][6].
- - **Notes**:
- - For a pipeline canvas, there is a limit of 25 processor groups and a total of 150 processors.
- - To copy a processor, click the copy icon for that processor and then paste it (`Cmd+V` on Mac, `Ctrl+V` on Windows or Linux).
-
-#### Add another processor group
-
-{{< img src="observability_pipelines/setup/another_processor_group.png" alt="The Pipelines page showing two processor groups sending logs to the same destination" style="width:100%;" >}}
-
-{{% observability_pipelines/set_up_pipelines/add_another_processor_group %}}
-
-[1]: https://app.datadoghq.com/observability-pipelines
-[2]: /observability_pipelines/configuration/explore_templates/?tab=traces#trace-sampling
-[3]: /observability_pipelines/sources/?tab=traces#sources
-[4]: /observability_pipelines/processors/?tab=traces#processors
-[5]: /observability_pipelines/destinations/?tab=traces#destinations
-[6]: /observability_pipelines/configuration/pipeline_simulation/
-
{{% /tab %}}
{{< /tabs >}}
diff --git a/hugo/content/en/observability_pipelines/configuration/update_existing_pipelines.md b/hugo/content/en/observability_pipelines/configuration/update_existing_pipelines.md
index 8584781733f..c88c9b89f4b 100644
--- a/hugo/content/en/observability_pipelines/configuration/update_existing_pipelines.md
+++ b/hugo/content/en/observability_pipelines/configuration/update_existing_pipelines.md
@@ -84,11 +84,6 @@ On the Worker installation page:
{{% observability_pipelines/configure_existing_pipelines/source_env_vars/opentelemetry %}}
-{{% /tab %}}
-{{% tab "Prometheus" %}}
-
-{{% observability_pipelines/configure_existing_pipelines/source_env_vars/prometheus %}}
-
{{% /tab %}}
{{% tab "Socket" %}}
@@ -206,22 +201,6 @@ On the Worker installation page:
{{% observability_pipelines/configure_existing_pipelines/destination_env_vars/opensearch %}}
-{{% /tab %}}
-{{% tab "OpenTelemetry" %}}
-
-**Metrics**
-
-{{% observability_pipelines/configure_existing_pipelines/destination_env_vars/opentelemetry_metrics %}}
-
-**Traces**
-
-{{% observability_pipelines/configure_existing_pipelines/destination_env_vars/opentelemetry_traces %}}
-
-{{% /tab %}}
-{{% tab "Prometheus" %}}
-
-{{% observability_pipelines/configure_existing_pipelines/destination_env_vars/prometheus %}}
-
{{% /tab %}}
{{% tab "SentinelOne" %}}
diff --git a/hugo/content/en/observability_pipelines/destinations/_index.md b/hugo/content/en/observability_pipelines/destinations/_index.md
index b570caae2f0..c8551f20a28 100644
--- a/hugo/content/en/observability_pipelines/destinations/_index.md
+++ b/hugo/content/en/observability_pipelines/destinations/_index.md
@@ -2,6 +2,11 @@
title: Destinations
description: Learn about the destinations available for the Observability Pipelines Worker.
disable_toc: false
+aliases:
+ - /observability_pipelines/destinations/datadog_apm/
+ - /observability_pipelines/destinations/opentelemetry/traces/
+ - /observability_pipelines/destinations/opentelemetry/metrics/
+ - /observability_pipelines/destinations/prometheus/
further_reading:
- link: "logs/processing/pipelines"
tag: "Documentation"
@@ -10,7 +15,7 @@ further_reading:
## Overview
-Use the Observability Pipelines Worker to send your processed logs, metrics, and traces ({{< tooltip text="in Preview" tooltip="Traces Pipeline is in Preview. Contact your account manager to request access." >}}) to different destinations. Most Observability Pipelines destinations send events in batches to the downstream integration. See [Event batching](#event-batching) for more information. Some Observability Pipelines destinations also have fields that support template syntax, so you can set these fields based on specific fields. See [Template syntax](#template-syntax) for more information.
+Use the Observability Pipelines Worker to send your processed logs and metrics to different destinations. Most Observability Pipelines destinations send events in batches to the downstream integration. See [Event batching](#event-batching) for more information. Some Observability Pipelines destinations also have fields that support template syntax, so you can set these fields based on specific fields. See [Template syntax](#template-syntax) for more information.
Select a destination in the left navigation menu to see more information about it.
@@ -79,7 +84,6 @@ These are the available destinations:
- [Elasticsearch][2]
- [HTTP/S Client][3]
- [OpenTelemetry][5]
-- [Prometheus][6]
- [Splunk HEC][4]
[1]: /observability_pipelines/destinations/datadog_metrics/
@@ -87,18 +91,6 @@ These are the available destinations:
[3]: /observability_pipelines/destinations/http_client/
[4]: /observability_pipelines/destinations/splunk_hec/metrics
[5]: /observability_pipelines/destinations/opentelemetry/metrics
-[6]: /observability_pipelines/destinations/prometheus/
-
-{{% /tab %}}
-{{% tab "Traces" %}}
-
-Traces Pipeline is in Preview. Contact your account manager to request access.
-
-- [Datadog APM][1]
-- [OpenTelemetry Traces][2]
-
-[1]: /observability_pipelines/destinations/datadog_apm/
-[2]: /observability_pipelines/destinations/opentelemetry/traces
{{% /tab %}}
{{< /tabs >}}
diff --git a/hugo/content/en/observability_pipelines/destinations/datadog_apm.md b/hugo/content/en/observability_pipelines/destinations/datadog_apm.md
deleted file mode 100644
index 49666f8abbd..00000000000
--- a/hugo/content/en/observability_pipelines/destinations/datadog_apm.md
+++ /dev/null
@@ -1,67 +0,0 @@
----
-title: Datadog APM Destination
-description: Learn how to send traces to Datadog using the Observability Pipelines Worker.
-disable_toc: false
-products:
-- name: Traces
- icon: apm
- url: /observability_pipelines/configuration/?tab=traces#pipeline-types
----
-
-{{< product-availability >}}
-
-{{< callout url="#"
- btn_hidden="true" header="false">}}
-The Datadog APM destination is in Preview. Contact your account manager to request access.
-{{< /callout >}}
-
-## Overview
-
-Use Observability Pipelines' Datadog APM destination to send traces to Datadog.
-
-## Setup
-
-Configure the Datadog APM destination when you [set up a pipeline][1] in the UI.
-
-### Optional buffering
-
-{{% observability_pipelines/destination_buffer %}}
-
-## Secret defaults
-
-{{% observability_pipelines/set_secrets_intro %}}
-
-{{< tabs >}}
-{{% tab "Secrets Management" %}}
-
-There are no secret identifiers for this destination.
-
-{{% /tab %}}
-
-{{% tab "Environment Variables" %}}
-
-{{% observability_pipelines/configure_existing_pipelines/destination_env_vars/datadog %}}
-
-{{% /tab %}}
-{{< /tabs >}}
-
-## AWS PrivateLink
-
-To send traces from Observability Pipelines to Datadog using AWS PrivateLink, see [Connect to Datadog over AWS PrivateLink][7] for setup instructions. The two endpoints you need to set up are:
-
-- Traces: {{< region-param key=traces_endpoint_private_link code="true" >}}
-- Remote Configuration: {{< region-param key=remote_config_endpoint_private_link code="true" >}}
-
-**Note**: The `obpipeline-intake.datadoghq.com` endpoint is used for Live Capture and is not available as a PrivateLink endpoint.
-
-## Health metrics
-
-See [Component metrics][5] and [Destination buffer metrics][6] for more information on metrics emitted by all destinations.
-
-[1]: /observability_pipelines/configuration/set_up_pipelines/
-[2]: https://app.datadoghq.com/observability-pipelines
-[3]: /api/latest/observability-pipelines/
-[4]: https://registry.terraform.io/providers/datadog/datadog/latest/docs/resources/observability_pipeline
-[5]: /observability_pipelines/monitoring_and_troubleshooting/pipeline_usage_metrics/#component-metrics
-[6]: /observability_pipelines/monitoring_and_troubleshooting/pipeline_usage_metrics/?tab=destinations#buffer
-[7]: /agent/guide/private-link/?tab=crossregionprivatelinkendpoints
diff --git a/hugo/content/en/observability_pipelines/destinations/opentelemetry/_index.md b/hugo/content/en/observability_pipelines/destinations/opentelemetry/_index.md
deleted file mode 100644
index c97b5fb4002..00000000000
--- a/hugo/content/en/observability_pipelines/destinations/opentelemetry/_index.md
+++ /dev/null
@@ -1,5 +0,0 @@
----
-title: OpenTelemetry
-type: multi-code-lang
-external_redirect: /observability_pipelines/destinations/opentelemetry/metrics/
----
\ No newline at end of file
diff --git a/hugo/content/en/observability_pipelines/destinations/opentelemetry/metrics.md b/hugo/content/en/observability_pipelines/destinations/opentelemetry/metrics.md
deleted file mode 100644
index 490b17f6608..00000000000
--- a/hugo/content/en/observability_pipelines/destinations/opentelemetry/metrics.md
+++ /dev/null
@@ -1,103 +0,0 @@
----
-title: OpenTelemetry Metrics Destination
-disable_toc: false
-code_lang: metrics
-type: multi-code-lang
-weight: 1
----
-
-{{< callout url="#"
- btn_hidden="true" header="false">}}
-The OpenTelemetry destination is in Preview. Contact your account manager to request access.
-{{< /callout >}}
-
-## Overview
-
-Use Observability Pipelines' OpenTelemetry destination to send metrics over HTTP/S to an OpenTelemetry (OTel) Collector or another OpenTelemetry Protocol (OTLP)-compatible endpoint.
-
-## Set up destination
-
-For Secrets Management: Only enter the identifier for the HTTP/S Client URI and, if applicable, the TLS key pass. Do not enter the actual values.
-
-Configure the OpenTelemetry destination when you [set up a pipeline][3]. You can set up a pipeline in the [UI][1], using the [API][4], or with [Terraform][5]. The steps in this section are configured in the UI.
-
-After you select the OpenTelemetry destination in the pipeline UI, enter the identifier for your HTTP/S Client URI. An example of the HTTP/S URI endpoint that the identifier references: `http://localhost:4319/v1/metrics`. If you leave the identifier field blank, the [default](#secret-defaults) is used.
-
-**Notes**:
-- The Worker can only send counter, gauge, and histogram metrics to OpenTelemetry. OpenTelemetry does not support other metrics types, so the Worker drops them. See [Filter out unsupported metrics](#filter-out-unsupported-metrics) for more information.
-- Datadog recommends setting your OTLP receiver to allow out-of-order samples because the Worker doesn't reorder metrics and some OTLP receivers reject out-of-order samples. See [Allow out-of-order samples](#allow-out-of-order-samples) for more information.
-- If you enter secret identifiers and then choose to use environment variables, the environment variable is the identifier entered and prepended with `DD_OP_`. For example, if you entered `PASSWORD_1` for a password identifier, the environment variable for that password is `DD_OP_PASSWORD_1`.
-
-### Optional settings
-
-#### Enable TLS
-
-{{% observability_pipelines/tls_settings %}}
-
-#### Buffering
-
-{{% observability_pipelines/destination_buffer %}}
-
-## Filter out unsupported metrics
-
-The Worker can only send counter, gauge, and histogram metrics to OpenTelemetry. The following Datadog metrics are not supported because they cannot be converted to OTLP format:
-
-- StatsD-type metrics
-- Distribution metrics
-- Sketch metrics
-
-If one of these metrics is in a batch to be encoded and sent to OpenTelemetry, the Worker drops the unsupported metric, logs an error, and updates the `component_error_total` metric. Datadog recommends using a [filter processor][9] to filter out unsupported metric types.
-
-## Allow out-of-order samples
-
-The Worker doesn't always send metrics in the correct order for a given series because it doesn't reorder metrics. For example, if the first batch of metrics contains metrics with timestamps: `10:03`, `10:04`, `10:05` and the second batch contains metrics with timestamps: `10:01`, `10:02`, `10:06`, the Worker does not reorder those metrics before sending them out.
-
-Because some OTLP receivers, such as the Prometheus OTLP receiver, reject out-of-order samples, the second batch of metrics gets rejected by the receiver. As a result, the Worker logs a Bad Request (`400`) error and the entire batch that was rejected gets dropped, even if the OTLP receiver accepted some of the valid metrics in the batch.
-
-Datadog recommends setting your OTLP receiver to allow out-of-order samples to prevent out-of-order samples from getting dropped.
-
-## Secret defaults
-
-{{% observability_pipelines/set_secrets_intro %}}
-
-{{< tabs >}}
-{{% tab "Secrets Management" %}}
-
-- HTTP/S Client URI endpoint identifier
- - References the HTTP/S URI endpoint to which the Worker sends OpenTelemetry data. An example of the HTTP/S URI endpoint that the identifier references: `http://localhost:4319/v1/metrics`.
- - The default identifier is `DESTINATION_OTEL_HTTP_CLIENT_URI`.
-- HTTP/S Client TLS passphrase identifier (when TLS is enabled):
- - The default identifier is `DESTINATION_OTEL_HTTP_CLIENT_KEY_PASS`.
-
-{{% /tab %}}
-
-{{% tab "Environment Variables" %}}
-
-{{% observability_pipelines/configure_existing_pipelines/destination_env_vars/opentelemetry_metrics %}}
-
-{{% /tab %}}
-{{< /tabs >}}
-
-## Metrics
-
-For [component metrics][6] and [destination buffer metrics][7] emitted by all destinations, see the [Pipelines Usage Metrics][8] documentation. To filter or group by OpenTelemetry destination metrics, use the tag `component_type:opentelemetry`.
-
-## How the destination works
-
-### Event batching
-
-A batch of events is flushed when one of these conditions occurs. See [event batching][2] for more information.
-
-| Maximum Events | Maximum Size (MB) | Timeout (seconds) |
-|----------------|-------------------|---------------------|
-| N/A | 10 | 1 |
-
-[1]: https://app.datadoghq.com/observability-pipelines
-[2]: /observability_pipelines/destinations/#event-batching
-[3]: /observability_pipelines/configuration/set_up_pipelines/
-[4]: /api/latest/observability-pipelines/
-[5]: https://registry.terraform.io/providers/datadog/datadog/latest/docs/resources/observability_pipeline
-[6]: /observability_pipelines/monitoring_and_troubleshooting/pipeline_usage_metrics/#component-metrics
-[7]: /observability_pipelines/monitoring_and_troubleshooting/pipeline_usage_metrics/#destination-buffer-metrics
-[8]: /observability_pipelines/monitoring_and_troubleshooting/pipeline_usage_metrics/
-[9]: /observability_pipelines/processors/filter/
diff --git a/hugo/content/en/observability_pipelines/destinations/opentelemetry/traces.md b/hugo/content/en/observability_pipelines/destinations/opentelemetry/traces.md
deleted file mode 100644
index 0fa55873f6f..00000000000
--- a/hugo/content/en/observability_pipelines/destinations/opentelemetry/traces.md
+++ /dev/null
@@ -1,90 +0,0 @@
----
-title: OpenTelemetry Traces Destination
-description: Learn how to send traces to an OpenTelemetry Collector using the Observability Pipelines Worker.
-disable_toc: false
-code_lang: traces
-type: multi-code-lang
-weight: 2
----
-
-{{< callout url="#"
- btn_hidden="true" header="false">}}
-The OpenTelemetry Traces destination is in Preview. Contact your account manager to request access.
-{{< /callout >}}
-
-## Overview
-
-Use Observability Pipelines' OpenTelemetry Traces destination to send traces to an OpenTelemetry (OTel) Collector.
-
-You must use an OpenTelemetry source to use the OpenTelemetry Traces destination.
-
-## Set up destination
-
-For Secrets Management: Only enter the identifier for the HTTP/S Client URI and, if applicable, the TLS key pass. Do not enter the actual values.
-
-Configure the OpenTelemetry Traces destination when you [set up a pipeline][3]. This section covers how to do so in the [UI][1], but you can also set up a pipeline using the [API][4] or with [Terraform][5].
-
-After you select the OpenTelemetry Traces destination in the pipeline UI, enter the identifier for your HTTP/S Client URI Key. An example of the URI endpoint the identifier references: `http://localhost:4319/v1/traces`. If you leave the identifier field blank, the [default](#secret-defaults) is used.
-
-{{% observability_pipelines/secrets_env_var_note %}}
-
-### Optional settings
-
-#### Enable TLS
-
-{{% observability_pipelines/tls_settings %}}
-
-#### Buffering
-
-{{% observability_pipelines/destination_buffer %}}
-
-## Allow out-of-order samples
-
-The Worker doesn't always send metrics in the correct order for a given series because it doesn't reorder metrics. For example, if the first batch of metrics contains metrics with timestamps: `10:03`, `10:04`, `10:05` and the second batch contains metrics with timestamps: `10:01`, `10:02`, `10:06`, the Worker does not reorder those metrics before sending them out.
-
-Because the OTLP receiver rejects out-of-order samples, the Worker logs a Bad Request (`400`) error and the entire second batch of metrics gets dropped, even if the OTLP receiver accepted some of the valid metrics in the batch.
-
-Datadog recommends setting your OTLP receiver to allow out-of-order samples to prevent out-of-order samples from getting dropped.
-
-## Troubleshooting
-
-### Debug error logs
-
-If you see `400` or `500` error logs from this destination, you can enable debug logs to see the response returned by the server. To enable logs for this HTTP-based destination only and not every Worker module, set `VECTOR_LOG` to `info,vector::sinks::util::http=debug`:
-
-```
-docker run -i -e DD_API_KEY= \
- -e DD_OP_PIPELINE_ID= \
- -e VECTOR_LOG=info,vector::sinks::util::http=debug \
- datadog/observability-pipelines-worker run
-```
-
-See [Enable debug logs][6] for instruction on enabling full debug logs.
-
-## Secret defaults
-
-{{% observability_pipelines/set_secrets_intro %}}
-
-{{< tabs >}}
-{{% tab "Secrets Management" %}}
-
-- HTTP/S Client URI endpoint identifier:
- - References the HTTP/S URI endpoint to which the Worker sends OpenTelemetry data. An example of the URI endpoint the identifier references: `http://localhost:4319/v1/traces`.
- - The default identifier is `DESTINATION_OTEL_HTTP_CLIENT_URI`.
-- OpenTelemetry Traces TLS passphrase identifier (when TLS is enabled):
- - The default identifier is `DESTINATION_OTEL_KEY_PASS`.
-
-{{% /tab %}}
-
-{{% tab "Environment Variables" %}}
-
-{{% observability_pipelines/configure_existing_pipelines/destination_env_vars/opentelemetry_traces %}}
-
-{{% /tab %}}
-{{< /tabs >}}
-
-[1]: https://app.datadoghq.com/observability-pipelines
-[3]: /observability_pipelines/configuration/set_up_pipelines/
-[4]: /api/latest/observability-pipelines/
-[5]: https://registry.terraform.io/providers/datadog/datadog/latest/docs/resources/observability_pipeline
-[6]: /observability_pipelines/monitoring_and_troubleshooting/troubleshooting/#enable-debug-logs
diff --git a/hugo/content/en/observability_pipelines/destinations/prometheus.md b/hugo/content/en/observability_pipelines/destinations/prometheus.md
deleted file mode 100644
index 913f8214cda..00000000000
--- a/hugo/content/en/observability_pipelines/destinations/prometheus.md
+++ /dev/null
@@ -1,115 +0,0 @@
----
-title: Prometheus Destination
-description: Learn how to send metrics to Prometheus using the Observability Pipelines Worker.
-disable_toc: false
-products:
-- name: Metrics
- icon: metrics
- url: /observability_pipelines/configuration/?tab=metrics#pipeline-types
----
-
-{{< product-availability >}}
-
-{{< callout url="#"
- btn_hidden="true" header="false">}}
-The Prometheus destination is in Preview. Contact your account manager to request access.
-{{< /callout >}}
-
-## Overview
-
-Use Observability Pipelines' Prometheus destination to send metrics to Prometheus.
-
-## Set up destination
-
-For Secrets Management: Only enter the identifiers for the endpoint URL and, if applicable, the username and password for basic authorization, the bearer token, and the TLS key pass. Do not enter the actual values.
-
-Configure the Prometheus destination when you [set up a pipeline][1]. You can set up a pipeline in the [UI][2], using the [API][3], or with [Terraform][4]. The steps in this section are configured in the UI.
-
-After you select the Prometheus destination in the pipeline UI:
-
-1. Enter the identifier for your {{< ui >}}Remote Write Endpoint URL{{< /ui >}}. An example of an endpoint URL the identifier references: `http://localhost:9090/api/v1/write`. If you leave the identifier field blank, the [default](#secret-defaults) is used.
-1. Select your authorization strategy ({{< ui >}}None{{< /ui >}}, {{< ui >}}Basic{{< /ui >}}, or {{< ui >}}Bearer{{< /ui >}}). If you selected:
- - {{< ui >}}Basic{{< /ui >}}: Enter the identifier for your username and password. If you leave it blank, the [default](#secret-defaults) is used.
- - {{< ui >}}Bearer{{< /ui >}}: Enter the identifier for your bearer token. If you leave it blank, the [default](#secret-defaults) is used.
-
-{{% observability_pipelines/secrets_env_var_note %}}
-
-### Optional settings
-
-#### Default namespace
-
-Enter the default namespace for any metrics sent. This namespace is only used if a metric has no existing namespace. It is added as a prefix to the metric name, separated by an underscore (`_`). The namespace must follow the [Prometheus naming convention][5].
-
-#### Tenant ID
-
-Enter the tenant ID to add to outgoing requests. This field supports [template syntax][6], but the template must have a literal prefix, such as `prefix-{{ tenant_id }}` or `prefix/{{ tenant_id }}`. Templates without a literal prefix, such as `{{ tenant_id }}`, are rejected; the Worker logs an error, and the pipeline isn't started.
-
-#### Enable TLS
-
-{{% observability_pipelines/tls_settings %}}
-- (Optional) Enter the server name to use for certificate validation. If left blank, the hostname from the endpoint URL is used.
-
-#### Buffering
-
-{{% observability_pipelines/destination_buffer %}}
-
-## Allow out-of-order samples
-
-The Worker doesn't always send metrics in the correct order for a given series because it doesn't reorder metrics. For example, if the first batch of metrics contains metrics with timestamps: `10:03`, `10:04`, `10:05` and the second batch contains metrics with timestamps: `10:01`, `10:02`, `10:06`, the Worker does not reorder those metrics before sending them out.
-
-Because the Prometheus OTLP receiver rejects out-of-order samples, the Worker logs a Bad Request (`400`) error and the entire second batch of metrics gets dropped, even if the OTLP receiver accepted some of the valid metrics in the batch.
-
-Datadog recommends setting your OTLP receiver to allow out-of-order samples to prevent out-of-order samples from getting dropped.
-
-## Troubleshooting
-
-### Debug error logs
-
-If you see `400` or `500` error logs from this destination, you can enable debug logs to see the response returned by the server. To enable logs for this HTTP-based destination only and not every Worker module, set `VECTOR_LOG` to `info,vector::sinks::util::http=debug`:
-
-```
-docker run -i -e DD_API_KEY= \
- -e DD_OP_PIPELINE_ID= \
- -e VECTOR_LOG=info,vector::sinks::util::http=debug \
- datadog/observability-pipelines-worker run
-```
-
-See [Enable debug logs][7] for instruction on enabling full debug logs.
-
-## Secret defaults
-
-{{% observability_pipelines/set_secrets_intro %}}
-
-{{< tabs >}}
-{{% tab "Secrets Management" %}}
-
-- Remote Write endpoint URL identifier:
- - References the Remote Write endpoint URL. An example of an endpoint URL the identifier references: `http://localhost:9090/api/v1/write`.
- - The default identifier is `DESTINATION_PROMETHEUS_ENDPOINT`.
-- Prometheus TLS passphrase identifier (when TLS is enabled):
- - The default identifier is `DESTINATION_PROMETHEUS_KEY_PASS`.
-- If you are using basic authentication:
- - Prometheus username identifier:
- - The default identifier is `DESTINATION_PROMETHEUS_USERNAME`.
- - Prometheus password identifier:
- - The default identifier is `DESTINATION_PROMETHEUS_PASSWORD`.
-- If you are using bearer authentication:
- - Prometheus bearer token identifier:
- - The default identifier is `DESTINATION_PROMETHEUS_BEARER_TOKEN`.
-
-{{% /tab %}}
-
-{{% tab "Environment Variables" %}}
-
-{{% observability_pipelines/configure_existing_pipelines/destination_env_vars/prometheus %}}
-
-{{% /tab %}}
-{{< /tabs >}}
-
-[1]: /observability_pipelines/configuration/set_up_pipelines/
-[2]: https://app.datadoghq.com/observability-pipelines
-[3]: /api/latest/observability-pipelines/
-[4]: https://registry.terraform.io/providers/datadog/datadog/latest/docs/resources/observability_pipeline
-[5]: https://prometheus.io/docs/practices/naming/
-[6]: /observability_pipelines/destinations/#template-syntax
-[7]: /observability_pipelines/monitoring_and_troubleshooting/troubleshooting/#enable-debug-logs
diff --git a/hugo/content/en/observability_pipelines/guide/environment_variables.md b/hugo/content/en/observability_pipelines/guide/environment_variables.md
index a6b14530b38..64fd46708b2 100644
--- a/hugo/content/en/observability_pipelines/guide/environment_variables.md
+++ b/hugo/content/en/observability_pipelines/guide/environment_variables.md
@@ -50,9 +50,6 @@ Some Observability Pipelines components require setting up environment variables
### OpenTelemetry
{{% observability_pipelines/configure_existing_pipelines/source_env_vars/opentelemetry %}}
-### Prometheus
-{{% observability_pipelines/configure_existing_pipelines/source_env_vars/prometheus %}}
-
### Socket
{{% observability_pipelines/configure_existing_pipelines/source_env_vars/socket %}}
@@ -132,17 +129,6 @@ Some Observability Pipelines components require setting up environment variables
### OpenSearch
{{% observability_pipelines/configure_existing_pipelines/destination_env_vars/opensearch %}}
-### OpenTelemetry
-
-**Metrics**
-{{% observability_pipelines/configure_existing_pipelines/destination_env_vars/opentelemetry_metrics %}}
-
-**Traces**
-{{% observability_pipelines/configure_existing_pipelines/destination_env_vars/opentelemetry_traces %}}
-
-### Prometheus
-{{% observability_pipelines/configure_existing_pipelines/destination_env_vars/prometheus %}}
-
### SentinelOne
{{% observability_pipelines/configure_existing_pipelines/destination_env_vars/sentinelone %}}
diff --git a/hugo/content/en/observability_pipelines/processors/_index.md b/hugo/content/en/observability_pipelines/processors/_index.md
index ec6927da7f3..86590d3cd29 100644
--- a/hugo/content/en/observability_pipelines/processors/_index.md
+++ b/hugo/content/en/observability_pipelines/processors/_index.md
@@ -1,6 +1,8 @@
---
title: Processors
disable_toc: false
+aliases:
+ - /observability_pipelines/processors/tail_based_sampling/
further_reading:
- link: https://www.datadoghq.com/blog/rehydrate-archived-logs-with-observability-pipelines
tag: Blog
@@ -14,7 +16,7 @@ further_reading:
The processors outlined in this documentation are specific to on-premises logging environments. To parse, structure, and enrich cloud-based logs, see the
Log Management documentation.
Traces Pipeline is in Preview. Contact your account manager to request access.
-
-- [Custom Processor][1]
-- [Filter][2]
-- [Sample][3]
-- [Sensitive Data Scanner][4]
-- [Tail-Based Sampling][5]
-
-[1]: /observability_pipelines/processors/custom_processor/
-[2]: /observability_pipelines/processors/filter/
-[3]: /observability_pipelines/processors/sample/
-[4]: /observability_pipelines/processors/sensitive_data_scanner/
-[5]: /observability_pipelines/processors/tail_based_sampling/
-
{{% /tab %}}
{{< /tabs >}}
diff --git a/hugo/content/en/observability_pipelines/processors/custom_processor.md b/hugo/content/en/observability_pipelines/processors/custom_processor.md
index 13e272590d2..0f0093ae25e 100644
--- a/hugo/content/en/observability_pipelines/processors/custom_processor.md
+++ b/hugo/content/en/observability_pipelines/processors/custom_processor.md
@@ -18,16 +18,13 @@ products:
- name: Metrics
icon: metrics
url: /observability_pipelines/configuration/?tab=metrics#pipeline-types
-- name: Traces
- icon: apm
- url: /observability_pipelines/configuration/?tab=traces#pipeline-types
---
{{< product-availability >}}
## Overview
-Use this processor with Vector Remap Language (VRL) to modify and enrich your logs, metrics, or traces ({{< tooltip text="in Preview" tooltip="Traces Pipeline is in Preview. Contact your account manager to request access." >}}). VRL is an expression-oriented, domain specific language designed for transforming data. It features built-in functions for observability use cases. You can use custom functions in the following ways:
+Use this processor with Vector Remap Language (VRL) to modify and enrich your logs or metrics. VRL is an expression-oriented, domain specific language designed for transforming data. It features built-in functions for observability use cases. You can use custom functions in the following ways:
- Manipulate [arrays](#array), [strings](#string), and other data types.
- Encode and decode values using [Codec](#codec).
diff --git a/hugo/content/en/observability_pipelines/processors/filter.md b/hugo/content/en/observability_pipelines/processors/filter.md
index 814854cfad7..065068c2564 100644
--- a/hugo/content/en/observability_pipelines/processors/filter.md
+++ b/hugo/content/en/observability_pipelines/processors/filter.md
@@ -15,16 +15,13 @@ products:
- name: Metrics
icon: metrics
url: /observability_pipelines/configuration/?tab=metrics#pipeline-types
-- name: Traces
- icon: apm
- url: /observability_pipelines/configuration/?tab=traces#pipeline-types
---
{{< product-availability >}}
## Overview
-This processor sends all logs, metrics, or traces ({{< tooltip text="in Preview" tooltip="Traces Pipeline is in Preview. Contact your account manager to request access." >}}) that match the filter query to the next step in the pipeline. Events that do not match the filter query are dropped and are not sent to any subsequent processors or destinations.
+This processor sends all logs or metrics that match the filter query to the next step in the pipeline. Events that do not match the filter query are dropped and are not sent to any subsequent processors or destinations.
**Note**: For all other processor queries, events that don't match the query are sent to the subsequent step in the pipeline. They are not dropped.
@@ -32,7 +29,7 @@ This processor sends all logs, metrics, or traces ({{< tooltip text="in Preview"
To set up the filter processor:
-- Define a {{< ui >}}filter query{{< /ui >}}. See [Logs Search Syntax][1], [Metrics Search Syntax][2], or [APM Query Syntax][6] for more information.
+- Define a {{< ui >}}filter query{{< /ui >}}. See [Logs Search Syntax][1] or [Metrics Search Syntax][2] for more information.
- Events that match the query are sent to the next component.
- Events that don't match the query are dropped.
@@ -45,7 +42,6 @@ For [component metrics][3] and [processor buffer metrics][4] emitted by all proc
[3]: /observability_pipelines/monitoring_and_troubleshooting/pipeline_usage_metrics/#component-metrics
[4]: /observability_pipelines/monitoring_and_troubleshooting/pipeline_usage_metrics/#processor-buffer-metrics
[5]: /observability_pipelines/monitoring_and_troubleshooting/pipeline_usage_metrics/
-[6]: /tracing/trace_explorer/query_syntax/
## Further reading
diff --git a/hugo/content/en/observability_pipelines/processors/sample.md b/hugo/content/en/observability_pipelines/processors/sample.md
index e7ead001823..2a2424f8ce7 100644
--- a/hugo/content/en/observability_pipelines/processors/sample.md
+++ b/hugo/content/en/observability_pipelines/processors/sample.md
@@ -5,23 +5,20 @@ products:
- name: Logs
icon: logs
url: /observability_pipelines/configuration/?tab=logs#pipeline-types
-- name: Traces
- icon: apm
- url: /observability_pipelines/configuration/?tab=traces#pipeline-types
---
{{< product-availability >}}
## Overview
-This processor samples your logs or traces ({{< tooltip text="in Preview" tooltip="Traces Pipeline is in Preview. Contact your account manager to request access." >}}) for a representative subset at the rate that you define, dropping the remaining events. As an example, you can use this processor to sample 20% of events from a noisy non-critical service.
+This processor samples your logs for a representative subset at the rate that you define, dropping the remaining events. As an example, you can use this processor to sample 20% of events from a noisy non-critical service.
The sampling only applies to events that match your filter query and does not impact other events. If an event is dropped at this processor, the event is not sent to subsequent processors.
## Setup
To set up the sample processor:
-1. Define a {{< ui >}}filter query{{< /ui >}}. See [Logs Search Syntax][1] or [APM Query Syntax][2] for more information.
+1. Define a {{< ui >}}filter query{{< /ui >}}. See [Logs Search Syntax][1] for more information.
- Only events that match the specified filter query are sampled at the specified retention rate.
- The sampled events and the events that do not match the filter query are sent to the next step in the pipeline.
1. Enter your desired sampling rate in the {{< ui >}}Retain{{< /ui >}} field. For example, entering `2` means 2% of events are retained out of all events that match the filter query.
@@ -44,4 +41,3 @@ Then, 40% of events for each unique combination of `status` and `service` from `
- 40% of events with `status:error` and `service:core-web` are retained.
[1]: /observability_pipelines/search_syntax/logs/
-[2]: /tracing/trace_explorer/query_syntax/
diff --git a/hugo/content/en/observability_pipelines/processors/sensitive_data_scanner.md b/hugo/content/en/observability_pipelines/processors/sensitive_data_scanner.md
index 8576ded6b10..b82ec0407df 100644
--- a/hugo/content/en/observability_pipelines/processors/sensitive_data_scanner.md
+++ b/hugo/content/en/observability_pipelines/processors/sensitive_data_scanner.md
@@ -12,16 +12,13 @@ products:
- name: Logs
icon: logs
url: /observability_pipelines/configuration/?tab=logs#pipeline-types
-- name: Traces
- icon: apm
- url: /observability_pipelines/configuration/?tab=traces#pipeline-types
---
{{< product-availability >}}
## Overview
-The Sensitive Data Scanner processor scans logs or traces ({{< tooltip text="in Preview" tooltip="Traces Pipeline is in Preview. Contact your account manager to request access." >}}) to detect and redact or hash sensitive information such as PII, PCI, and custom sensitive data. You can pick from Datadog's library of predefined rules, or input custom Regex rules to scan for sensitive data.
+The Sensitive Data Scanner processor scans logs to detect and redact or hash sensitive information such as PII, PCI, and custom sensitive data. You can pick from Datadog's library of predefined rules, or input custom Regex rules to scan for sensitive data.
You can set up the pipeline and processor in the [UI](#set-up-the-processor-in-the-ui), [API][10], or [Terraform](#set-up-the-processor-using-terraform).
@@ -31,7 +28,7 @@ See [Best practices to optimize performance](#best-practices-to-optimize-perform
To set up the processor:
-1. Define a {{< ui >}}filter query{{< /ui >}}. See [Logs Search Syntax][1] or [APM Query Syntax][17] for more information.
+1. Define a {{< ui >}}filter query{{< /ui >}}. See [Logs Search Syntax][1] for more information.
- Only events matching the filter are scanned and processed.
- All events, regardless of whether they match the filter query, are sent to the next step in the pipeline.
1. Click {{< ui >}}Add Scanning Rule{{< /ui >}}.
@@ -411,4 +408,3 @@ For [component metrics][13] and [processor buffer metrics][14] emitted by all pr
[14]: /observability_pipelines/monitoring_and_troubleshooting/pipeline_usage_metrics/#processor-buffer-metrics
[15]: /observability_pipelines/monitoring_and_troubleshooting/pipeline_usage_metrics/
[16]: https://app.datadoghq.com/dash/integration/32326/observability-pipelines-overview
-[17]: /tracing/trace_explorer/query_syntax/
diff --git a/hugo/content/en/observability_pipelines/processors/tail_based_sampling.md b/hugo/content/en/observability_pipelines/processors/tail_based_sampling.md
deleted file mode 100644
index ec24aed1733..00000000000
--- a/hugo/content/en/observability_pipelines/processors/tail_based_sampling.md
+++ /dev/null
@@ -1,48 +0,0 @@
----
-title: Tail-Based Sampling Processor
-disable_toc: false
-products:
-- name: Traces
- icon: apm
- url: /observability_pipelines/configuration/?tab=traces#pipeline-types
----
-
-{{< product-availability >}}
-
-{{< callout url="#"
- btn_hidden="true" header="false">}}
-The Tail-Based Sampling processor is in Preview. Contact your account manager to request access.
-{{< /callout >}}
-
-## Overview
-
-The Tail-based Sampling processor determines whether the Worker keeps a completed trace based on sampling policies that you define. This processor can sample based on the full context of a trace, such as its status code, latency, or associated events.
-
-## Setup
-
-If you are using the Filter processor, do not filter out spans belonging to a trace because this can lead to incomplete traces and incorrect sampling.
-
-To set up the tail-based sampling processor:
-
-1. Define a {{< ui >}}filter query{{< /ui >}}. See [APM Query Syntax][1] for more information.
- - Only traces that match the specified filter query are evaluated against your sampling policy groups.
-1. Click {{< ui >}}Manage sampling policies{{< /ui >}} to add at least one policy group.
-
-### Add a sampling policy group
-
-1. Enter a {{< ui >}}Policy Group Name{{< /ui >}}.
-1. Click to add a policy, and select a {{< ui >}}Policy Type{{< /ui >}}:
- - {{< ui >}}Condition{{< /ui >}}: Keep traces when any associated event matches the specified query.
- - {{< ui >}}Status code{{< /ui >}}: Keep traces that have the selected status code.
- - {{< ui >}}Latency{{< /ui >}}: Keep traces whose total duration falls within the specified bounds.
- - {{< ui >}}Sampling rate{{< /ui >}}: Keep this percentage of traces, sampled consistently by trace ID.
-1. (Optional) Click to add more policies to the policy group. A trace matches a policy group only if it satisfies all of the policies within that group.
-1. (Optional) Repeat these steps to add more policy groups.
-1. Click **Save**.
-
-**Notes**:
-- A trace is only sampled if it matches any of the configured policy groups.
-- If a trace matches a policy group, it's kept and sent to the next step in the pipeline. If a trace doesn't match any policy groups, it's dropped.
-- If there are multiple policies in a policy group, a trace must match all policies to be sampled.
-
-[1]: /tracing/trace_explorer/query_syntax/
diff --git a/hugo/content/en/observability_pipelines/sources/_index.md b/hugo/content/en/observability_pipelines/sources/_index.md
index 6c218483003..5fcaea0312a 100644
--- a/hugo/content/en/observability_pipelines/sources/_index.md
+++ b/hugo/content/en/observability_pipelines/sources/_index.md
@@ -1,6 +1,8 @@
---
title: Sources
description: Learn about the sources available for the Observability Pipelines Worker.
+aliases:
+ - /observability_pipelines/sources/prometheus
disable_toc: false
further_reading:
- link: "/observability_pipelines/configuration/set_up_pipelines/"
@@ -80,19 +82,6 @@ These are the available sources:
{{% /tab %}}
{{% tab "Metrics" %}}
-- [Datadog Agent][1]
-- [OpenTelemetry][2]
-- [Prometheus][25]
-
-[1]: /observability_pipelines/sources/datadog_agent/
-[2]: /observability_pipelines/sources/opentelemetry/
-[25]: /observability_pipelines/sources/prometheus/
-
-{{% /tab %}}
-{{% tab "Traces" %}}
-
-Traces Pipeline is in Preview. Contact your account manager to request access.
-
- [Datadog Agent][1]
- [OpenTelemetry][2]
diff --git a/hugo/content/en/observability_pipelines/sources/datadog_agent.md b/hugo/content/en/observability_pipelines/sources/datadog_agent.md
index 25847806b45..83fb73b5f14 100644
--- a/hugo/content/en/observability_pipelines/sources/datadog_agent.md
+++ b/hugo/content/en/observability_pipelines/sources/datadog_agent.md
@@ -9,9 +9,6 @@ products:
- name: Metrics
icon: metrics
url: /observability_pipelines/configuration/?tab=metrics#pipeline-types
-- name: Traces
- icon: apm
- url: /observability_pipelines/configuration/?tab=traces#pipeline-types
further_reading:
- link: https://www.datadoghq.com/blog/manage-metrics-cost-control-with-observability-pipelines
tag: Blog
@@ -22,7 +19,7 @@ further_reading:
## Overview
-Use Observability Pipelines' Datadog Agent source to receive logs, metrics, or traces ({{< tooltip text="in Preview" tooltip="Traces Pipeline is in Preview. Contact your account manager to request access." >}}) from the Datadog Agent.
+Use Observability Pipelines' Datadog Agent source to receive logs or metrics from the Datadog Agent.
**Notes**:
- If you are using the Datadog Distribution of OpenTelemetry (DDOT) Collector to collect logs or metrics, you must [use the OpenTelemetry source to send that data to Observability Pipelines][4].
@@ -152,63 +149,6 @@ datadog:
[1]: /containers/docker/data_collected/
[2]: /containers/guide/container-discovery-management/?tab=helm#setting-environment-variables
-{{% /tab %}}
-
-{{% tab "Traces" %}}
-
-Traces Pipeline is in Preview. Contact your account manager to request access.
-
-Use the Agent configuration file or the Agent Helm chart values file to connect the Datadog Agent to the Observability Pipelines Worker.
-
-{{% collapse-content title="Agent configuration file" level="h4" expanded=false id="traces-agent-config-file" %}}
-
-To send Datadog Agent traces to the Observability Pipelines Worker, update your [Agent configuration file][1] with the following:
-
-```
-observability_pipelines_worker:
- traces:
- enabled: true
- url: "http://:8484"
-
-```
-
-`` is the host IP address or the load balancer URL associated with the Observability Pipelines Worker.
-- For CloudFormation installs, use the `LoadBalancerDNS` CloudFormation output for the URL.
-- For Kubernetes installs, you can use the internal DNS record of the Observability Pipelines Worker service. For example: `http://opw-observability-pipelines-worker.default.svc.cluster.local:`.
-
-**Note**: If the Worker is listening for logs or metrics on ports 8282 or 8383, you must use another port for traces, such as 8484.
-
-After you [restart the Agent][2], your observability data is sent to the Worker, processed by the pipeline, and delivered to Datadog.
-
-[1]: /agent/configuration/agent-configuration-files/
-[2]: /agent/configuration/agent-commands/#restart-the-agent
-
-{{% /collapse-content %}}
-
-{{% collapse-content title="Agent Helm values file" level="h4" expanded=false id="traces-agent-helm-values-file" %}}
-
-To send Datadog Agent traces to the Observability Pipelines Worker, update your Datadog Helm chart [datadog-values.yaml][1] with the following environment variables. See [Agent Environment Variables][2] for more information.
-
-```
-datadog:
- env:
- - name: DD_OBSERVABILITY_PIPELINES_WORKER_TRACES_ENABLED
- value: true
- - name: DD_OBSERVABILITY_PIPELINES_WORKER_TRACES_URL
- value: "http://:8484"
-```
-
-`` is the host IP address or the load balancer URL associated with the Observability Pipelines Worker.
-
- For Kubernetes installs, you can use the internal DNS record of the Observability Pipelines Worker service. For example: `http://opw-observability-pipelines-worker.default.svc.cluster.local:`.
-
-**Note**: If the Worker is listening for logs or metrics on ports 8282 or 8383, you must use another port for traces, such as 8484.
-
-[1]: https://github.com/DataDog/helm-charts/blob/main/charts/datadog/values.yaml
-[2]: https://docs.datadoghq.com/agent/guide/environment-variables/
-
-{{% /collapse-content %}}
-
{{% /tab %}}
{{< /tabs >}}
diff --git a/hugo/content/en/observability_pipelines/sources/opentelemetry.md b/hugo/content/en/observability_pipelines/sources/opentelemetry.md
index 73098f0e5d3..ca8589d5aa2 100644
--- a/hugo/content/en/observability_pipelines/sources/opentelemetry.md
+++ b/hugo/content/en/observability_pipelines/sources/opentelemetry.md
@@ -16,16 +16,13 @@ products:
- name: Metrics
icon: metrics
url: /observability_pipelines/configuration/?tab=metrics#pipeline-types
-- name: Traces
- icon: apm
- url: /observability_pipelines/configuration/?tab=traces#pipeline-types
---
{{< product-availability >}}
## Overview
-Use Observability Pipelines' OpenTelemetry (OTel) source to collect logs, metrics, or traces ({{< tooltip text="in Preview" tooltip="Traces Pipeline is in Preview. Contact your account manager to request access." >}}) from your OTel Collector through HTTP or gRPC.
+Use Observability Pipelines' OpenTelemetry (OTel) source to collect logs or metrics from your OTel Collector through HTTP or gRPC.
**Notes**:
- If you are using the Datadog Distribution of OpenTelemetry (DDOT) Collector, use the OpenTelemetry source to [send data to Observability Pipelines](#send-data-from-the-datadog-distribution-of-opentelemetry-collector-to-observability-pipelines).
@@ -164,42 +161,6 @@ Set the listener address environment variables to the following default values.
- HTTP listener address: `worker:4318`
- gRPC listener address: `worker:4317`
-{{% /tab %}}
-{{% tab "Traces" %}}
-
-Traces Pipeline is in Preview. Contact your account manager to request access.
-
-### HTTP configuration example
-
-The Worker exposes the HTTP endpoint on port 4318, which is the default port. You can configure the port value in the Worker.
-
-For example, to configure an OTel trace exporter over HTTP in Python:
-
-```python
- from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
- http_exporter = OTLPSpanExporter(
- endpoint="http://worker:4318/v1/traces"
- )
-```
-
-### gRPC configuration example
-
-The Worker exposes the gRPC endpoint on port 4317, which is the default port. You can configure the port value in the Worker.
-
-For example, to configure an OTel trace exporter over gRPC in Python:
-
-```python
- from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
- grpc_exporter = OTLPSpanExporter(
- endpoint="grpc://worker:4317"
- )
-```
-
-Set the listener address environment variables to the following default values. If you configured different port values in the Worker, use those instead.
-
-- HTTP listener address: `worker:4318`
-- gRPC listener address: `worker:4317`
-
{{% /tab %}}
{{< /tabs >}}
@@ -290,47 +251,6 @@ To send metrics from the Datadog Distribution of the OpenTelemetry (DDOT) Collec
[8]: /observability_pipelines/processors/custom_processor
[9]: https://docs.datadoghq.com/opentelemetry/setup/ddot_collector/install/kubernetes_daemonset/?tab=helm#configure-the-opentelemetry-collector
-{{% /tab %}}
-{{% tab "Traces" %}}
-
-Traces Pipeline is in Preview. Contact your account manager to request access.
-
-To send traces from the Datadog Distribution of the OpenTelemetry (DDOT) Collector:
-1. Deploy the DDOT Collector using Helm. See [Install the DDOT Collector as a Kubernetes DaemonSet][5] for instructions.
-1. [Set up a pipeline][6] on Observability Pipelines using the [OpenTelemetry source](#set-up-the-source-in-the-pipeline-ui).
- 1. (Optional) Datadog recommends adding an [Edit Fields processor][7] to the pipeline that appends the field `op_otel_ddot:true`.
- 1. When you install the Worker, for the OpenTelemetry source environment variables:
- 1. Set your HTTP listener to `0.0.0.0:4318`.
- 1. Set your gRPC listener to `0.0.0.0:4317`.
- 1. After you install the Worker and deployed the pipeline, update the OpenTelemetry Collector's [`otel-config.yaml`][9] to include an exporter that sends traces to Observability Pipelines. For example:
- ```
- exporters:
- otlphttp:
- endpoint: http://opw-observability-pipelines-worker..svc.cluster.local:4318
- ...
- service:
- pipelines:
- traces:
- exporters: [otlphttp]
- ```
- Replace `` with the Kubernetes namespace where the Observability Pipelines Worker is deployed (for example, `default`).
- 1. Redeploy the Datadog Agent with the updated [`otel-config.yaml`][9]. For example, if the Agent is installed in Kubernetes:
- ```
- helm upgrade --install datadog-agent datadog/datadog \
- --values ./agent.yaml \
- --set-file datadog.otelCollector.config=./otel-config.yaml
- ```
-
-**Notes**:
-- Traces sent from DDOT might have nested objects that prevent Datadog from parsing the traces correctly. To resolve this, Datadog recommends using the [Custom Processor][8] to flatten the nested `resource` object.
-- If the DDOT Collector and the Observability Pipelines Worker are running on the same host, their default OTLP receiver ports (4317/4318) may conflict. In a typical Kubernetes deployment, the Collector and the Worker run in separate pods, so this is not an issue.
-
-[5]: /opentelemetry/setup/ddot_collector/install/kubernetes_daemonset/?tab=datadogoperator
-[6]: /observability_pipelines/configuration/set_up_pipelines/
-[7]: /observability_pipelines/processors/edit_fields#add-field
-[8]: /observability_pipelines/processors/custom_processor
-[9]: https://docs.datadoghq.com/opentelemetry/setup/ddot_collector/install/kubernetes_daemonset/?tab=helm#configure-the-opentelemetry-collector
-
{{% /tab %}}
{{< /tabs >}}
diff --git a/hugo/content/en/observability_pipelines/sources/prometheus.md b/hugo/content/en/observability_pipelines/sources/prometheus.md
deleted file mode 100644
index f860c6a03c1..00000000000
--- a/hugo/content/en/observability_pipelines/sources/prometheus.md
+++ /dev/null
@@ -1,89 +0,0 @@
----
-title: Prometheus Source
-description: Learn how to collect metrics pushed by Prometheus clients using the Observability Pipelines Worker.
-disable_toc: false
-products:
-- name: Metrics
- icon: metrics
- url: /observability_pipelines/configuration/?tab=metrics#pipeline-types
----
-
-{{< product-availability >}}
-
-{{< callout url="#"
- btn_hidden="true" header="false">}}
-The Prometheus source is in Preview. Contact your account manager to request access.
-{{< /callout >}}
-
-## Overview
-
-Use Observability Pipelines' Prometheus source to receive metrics pushed by your Prometheus clients.
-
-## Setup
-
-For Secrets Management: Only enter the identifiers for the Prometheus address and, if applicable, the username and password for plain (also known as basic) authorization. Do not enter the actual values.
-
-Set up this source when you [set up a pipeline][1]. You can set up a pipeline in the [UI][2], using the [API][3], or with [Terraform][4]. The instructions in this section are for setting up the source in the UI.
-
-After you select the Prometheus source in the pipeline UI:
-
-1. Enter the identifier for your Prometheus address. An example of the socket address that the identifier references: `0.0.0.0:9091`. If you leave the identifier field blank, the [default](#secret-defaults) is used.
-1. Select your authorization strategy. If you selected {{< ui >}}Plain{{< /ui >}}:
- - Enter the identifiers for your Prometheus username and password. If you leave them blank, the [defaults](#secret-defaults) are used.
-
-{{% observability_pipelines/secrets_env_var_note %}}
-
-### Optional settings
-
-#### Configure authentication tokens
-
-If you store tokens as credentials in your Prometheus client's authorization header, you can configure the Worker to check if incoming requests have a valid token. Request events that do not have a valid token are dropped. The Worker can also look up an endpoint path or an IP address instead of a header.
-
-{{% observability_pipelines/configure_authentication_tokens %}}
-
-#### Aggregate metrics
-
-Select {{< ui >}}Aggregate metrics{{< /ui >}} to combine metrics that share the same name, tags, and timestamp before they are sent downstream.
-
-#### Configure keepalive
-
-To configure keepalive settings for connections to the source, enable the {{< ui >}}Configure keepalive{{< /ui >}} toggle:
-
-- {{< ui >}}Max connection age{{< /ui >}}: The maximum number of seconds after which a connection is closed. The default is `300` seconds.
-- {{< ui >}}Max connection age jitter factor{{< /ui >}}: The factor used to randomize the max connection age, so connections don't all close simultaneously. The default is `0.1`.
-
-#### Enable TLS
-
-{{% observability_pipelines/tls_settings %}}
-
-{{% observability_pipelines/tls_settings_mtls %}}
-
-## Secret defaults
-
-{{% observability_pipelines/set_secrets_intro %}}
-
-{{< tabs >}}
-{{% tab "Secrets Management" %}}
-
-- Prometheus address identifier:
- - References the socket address on which the Observability Pipelines Worker listens for Prometheus metrics. An example of the socket address that the identifier references: `0.0.0.0:9091`.
- - The default identifier is `SOURCE_PROMETHEUS_ADDRESS`.
-- If you are using plain authentication:
- - Prometheus username identifier:
- - The default identifier is `SOURCE_PROMETHEUS_USERNAME`.
- - Prometheus password identifier:
- - The default identifier is `SOURCE_PROMETHEUS_PASSWORD`.
-
-{{% /tab %}}
-
-{{% tab "Environment Variables" %}}
-
-{{% observability_pipelines/configure_existing_pipelines/source_env_vars/prometheus %}}
-
-{{% /tab %}}
-{{< /tabs >}}
-
-[1]: /observability_pipelines/configuration/set_up_pipelines/
-[2]: https://app.datadoghq.com/observability-pipelines
-[3]: /api/latest/observability-pipelines/
-[4]: https://registry.terraform.io/providers/datadog/datadog/latest/docs/resources/observability_pipeline