From a4b718abbe47db21965b2772571f99645ddd2c1b Mon Sep 17 00:00:00 2001 From: Dineth Date: Fri, 26 Jun 2026 13:56:30 +0530 Subject: [PATCH 01/21] Add support for capturing request and response headers in analytics - Introduced configuration options for enabling header capture in analytics events. - Added a new Log publisher to output analytics events to stdout. - Updated analytics system policy to handle request and response headers. - Enhanced tests to validate header capture functionality. Enhance analytics configuration with max payload size and ignored path prefixes Remove pretty Refactor analytics and traffic logging configuration to use a unified collector - Migrate analytics configuration to a new collector structure, consolidating settings for analytics and traffic logging. - Update tests to reflect changes in configuration structure and ensure compatibility with the new collector. - Deprecate old analytics settings and provide migration paths for existing configurations. - Adjust access log service server initialization to utilize the new collector settings. - Ensure that both analytics and traffic logging consumers require the collector to be enabled for proper functionality. Refactor access log configuration to unify ALS settings under [collector.als] - Updated CollectorConfig and AnalyticsConfig to replace deprecated grpc_event_server and access_logs_service fields with als. - Adjusted validation and migration logic to reflect new configuration structure. - Modified error messages to reference the new [collector.als] section. - Enhanced traffic logging functionality by introducing TrafficLog and TrafficLogDirective types, allowing for per-API logging configurations. - Implemented tests to ensure correct behavior of traffic logging, including handling of excluded headers and field selection. - Updated access logger server initialization to utilize new configuration keys. Add auto activate for collector Refactor traffic logging configuration to support max payload size and ignored path prefixes Refactor latency calculations and enhance Latencies struct with new duration fields --- gateway/configs/config-template.toml | 100 +++++- gateway/configs/config.toml | 36 +- .../gateway-controller/pkg/config/config.go | 223 ++++++++---- .../pkg/config/config_test.go | 117 +++--- .../pkg/utils/system_policies.go | 22 +- .../pkg/utils/system_policies_test.go | 60 +++- .../gateway-controller/pkg/xds/translator.go | 18 +- .../pkg/xds/translator_test.go | 10 +- .../policy-engine/cmd/policy-engine/main.go | 8 +- .../internal/analytics/analytics.go | 92 +++-- .../internal/analytics/analytics_test.go | 112 +++++- .../internal/analytics/dto/event.go | 34 ++ .../internal/analytics/dto/latencies.go | 7 +- .../internal/analytics/publishers/log.go | 310 ++++++++++++++++ .../internal/analytics/publishers/log_test.go | 334 ++++++++++++++++++ .../policy-engine/internal/config/config.go | 245 +++++++++---- .../internal/config/config_test.go | 104 ++++-- .../internal/utils/access_logger_server.go | 18 +- .../utils/access_logger_server_test.go | 6 +- .../system-policies/analytics/analytics.go | 76 +++- .../analytics/analytics_headers_test.go | 63 ++++ .../analytics/policy-definition.yaml | 24 ++ 22 files changed, 1706 insertions(+), 313 deletions(-) create mode 100644 gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go create mode 100644 gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go create mode 100644 gateway/system-policies/analytics/analytics_headers_test.go diff --git a/gateway/configs/config-template.toml b/gateway/configs/config-template.toml index d11d5ebb66..a9c3965cce 100644 --- a/gateway/configs/config-template.toml +++ b/gateway/configs/config-template.toml @@ -281,18 +281,61 @@ port = 9010 host = "localhost" # ============================================================================= -# ANALYTICS CONFIGURATION +# COLLECTOR CONFIGURATION # ============================================================================= -[analytics] +# The collector is the shared data-capture pipeline: it injects the analytics +# system policy and ships request/response headers and bodies to the policy-engine +# over ALS. It is a PREREQUISITE for every consumer below — analytics and +# traffic_logging both require collector.enabled = true (enabling a consumer with +# the collector off is a startup error). +[collector] enabled = false -# Deprecated: allow_payloads is preserved for backward compatibility. When true -# and both send_request_body and send_response_body are false, both directions -# are enabled (bool fields cannot distinguish "unset" from explicitly false). -# To disable payloads entirely, set allow_payloads = false or remove it; do not -# rely on send_*_body = false while allow_payloads remains true. -allow_payloads = false +# Capture request/response payloads (bodies) into the collected event. Bodies are +# captured in full; per-consumer size caps are applied on output (see +# traffic_logging.max_payload_size). send_request_body = false send_response_body = false +# Capture ALL request/response headers into the collected event (request_headers/ +# response_headers). Applies to every API via the collector system policy, so no +# per-API header policy is needed. Sensitive values are redacted on output by each +# consumer (e.g. traffic_logging.masked_headers). +send_request_headers = false +send_response_headers = false + +# ALS transport tuning (advanced; defaults are sensible). One section read by BOTH +# ends of the Envoy → policy-engine access-log stream: +# • gateway-controller — configures Envoy's gRPC access-log sink (the sender) +# • policy-engine — runs the receiving gRPC server +# Shared keys (server_port, TLS, message/header limits) must match on both ends and +# live here once. Envoy-sender-only keys (buffer_*, grpc_request_timeout) are read +# only by the controller and ignored by the policy-engine. +[collector.als] +mode = "uds" # "uds" (default) or "tcp" +server_port = 18090 # engine listens here; Envoy dials it (tcp mode) +buffer_flush_interval = 1000000000 # Envoy sender only (nanoseconds) +buffer_size_bytes = 16384 # Envoy sender only +grpc_request_timeout = 20000000000 # Envoy sender only (nanoseconds) +shutdown_timeout = "600s" +public_key_path = "" # TLS for the ALS connection (both ends) +private_key_path = "" +als_plain_text = true # plaintext gRPC (skip TLS) +max_message_size = 1000000000 +max_header_limit = 8192 + +# ============================================================================= +# ANALYTICS CONFIGURATION (consumer — requires [collector].enabled = true) +# ============================================================================= +[analytics] +enabled = false +# Deprecated: allow_payloads / send_request_body / send_response_body are preserved +# for backward compatibility and are mapped onto the [collector] body-capture flags +# during validation (with a warning). Prefer setting [collector] directly. +allow_payloads = false +# send_request_body = false +# send_response_body = false +# Publishers that receive analytics events. Supported: "moesif" (sends to the +# Moesif SaaS). For stdout JSON logging use [traffic_logging] instead of a "log" +# publisher here. enabled_publishers = ["moesif"] [analytics.publishers.moesif] @@ -303,17 +346,36 @@ event_queue_size = 10000 batch_size = 50 timer_wakeup_seconds = 3 -[analytics.grpc_event_server] -buffer_flush_interval = 1000000000 -buffer_size_bytes = 16384 -grpc_request_timeout = 20000000000 -server_port = 18090 -shutdown_timeout = "600s" -public_key_path = "" -private_key_path = "" -als_plain_text = true -max_message_size = 1000000000 -max_header_limit = 8192 +# ============================================================================= +# TRAFFIC LOGGING (consumer — requires [collector].enabled = true) +# ============================================================================= +# Writes collected events (rich API/application/AI/latency metadata, plus +# request/response headers and — when the collector's send_*_body is enabled — +# payloads) to stdout as a JSON line. Useful for log-scraping pipelines such as +# Fluent Bit/Loki/ELK; no external SaaS needed. +# +# Per-API (opt-in): a line is emitted only for APIs that attach the `log-message` +# policy with `destination = access-log` — enabling this section alone does NOT +# log every API. So an API is logged only when all three hold: [collector] +# .enabled = true, [traffic_logging].enabled = true, and the `log-message` policy +# (access-log destination) is attached to that API. The policy's request/response +# parameters (per-flow payload/headers toggles and excludeHeaders) shape each +# API's line; they filter/mask what the collector captured (they cannot enable +# capture the collector was not configured to perform). +[traffic_logging] +enabled = false +# Header names (case-insensitive) whose values are redacted as "****" in the +# logged requestHeaders/responseHeaders. Applies globally; the `log-message` +# policy's (access-log mode) per-API excludeHeaders drop additional headers entirely. +masked_headers = ["authorization"] +# Max bytes of request/response payload written to each log line (0 = no limit). +# Applied output-side, so the collector still captures full bodies and other +# consumers (e.g. Moesif) are unaffected. +max_payload_size = 0 +# Suppress traffic-log lines for requests whose original (pre-rewrite) path starts +# with any of these prefixes (e.g. health/readiness probes). Only affects this +# stdout publisher — other consumers still receive the event. +ignored_path_prefixes = [] # ============================================================================= # POLICY CONFIGURATIONS diff --git a/gateway/configs/config.toml b/gateway/configs/config.toml index cd4ea4897f..98238a3fc1 100644 --- a/gateway/configs/config.toml +++ b/gateway/configs/config.toml @@ -1,16 +1,34 @@ -[analytics] +# The collector is the shared data-capture pipeline. It must be enabled before any +# consumer (analytics, traffic_logging) can receive data. +[collector] enabled = true -# Deprecated: allow_payloads is preserved for backward compatibility. When true -# and when send_request_body/send_response_body are not explicitly set, both -# request and response bodies will be included in analytics events. -# allow_payloads = true -send_request_body = false -send_response_body = false +# Capture request/response payloads (bodies) into the collected event (full bodies; +# per-consumer size caps are applied on output). +send_request_body = true +send_response_body = true +# Capture ALL request/response headers (redacted on output by per-consumer masking). +send_request_headers = true +send_response_headers = true + +# Analytics consumer (e.g. Moesif). Requires [collector].enabled = true. +[analytics] +enabled = false enabled_publishers = ["moesif"] -[analytics.publishers.moesif] -application_id = "" +# [analytics.publishers.moesif] +# application_id = "" + +# Traffic-logging consumer: writes each collected event to stdout as a JSON line +# (no external service needed). Requires [collector].enabled = true. +[traffic_logging] +enabled = true +# Header names (case-insensitive) whose values are redacted as "****". +masked_headers = ["authorization"] +# Max bytes of request/response payload written to each log line (0 = no limit). +max_payload_size = 2048 +# Skip traffic-log lines for these path prefixes (e.g. health probes). +ignored_path_prefixes = ["/health", "/ready"] [router] gateway_host = "*" diff --git a/gateway/gateway-controller/pkg/config/config.go b/gateway/gateway-controller/pkg/config/config.go index 6402427984..8ed547a111 100644 --- a/gateway/gateway-controller/pkg/config/config.go +++ b/gateway/gateway-controller/pkg/config/config.go @@ -48,6 +48,7 @@ type Config struct { Router RouterConfig `koanf:"router"` PolicyEngine map[string]interface{} `koanf:"policy_engine"` PolicyConfigurations map[string]interface{} `koanf:"policy_configurations"` + Collector CollectorConfig `koanf:"collector"` Analytics AnalyticsConfig `koanf:"analytics"` TracingConfig TracingConfig `koanf:"tracing"` APIKey APIKeyConfig `koanf:"api_key"` @@ -57,19 +58,45 @@ type Config struct { ImmutableGateway ImmutableGatewayConfig `koanf:"immutable_gateway"` } +// CollectorConfig holds the data-collection ("collector") configuration. The +// collector is the shared capture pipeline (the analytics system policy plus the +// Envoy→policy-engine ALS transport) that gathers request/response headers and +// bodies. It is a prerequisite for every consumer of that data — analytics and +// traffic logging both require collector.enabled to be true. +type CollectorConfig struct { + // Enabled turns the collector on. When false, the analytics system policy is + // not injected and Envoy ships no access logs to the policy-engine. + Enabled bool `koanf:"enabled"` + // SendRequestBody / SendResponseBody capture request/response bodies into the + // collected event. + SendRequestBody bool `koanf:"send_request_body"` + SendResponseBody bool `koanf:"send_response_body"` + // SendRequestHeaders / SendResponseHeaders, when true, make the collector + // capture ALL request / response headers, so every API's headers flow into + // the collected event without attaching a per-API header policy. + SendRequestHeaders bool `koanf:"send_request_headers"` + SendResponseHeaders bool `koanf:"send_response_headers"` + // GRPCEventServerCfg tunes the Envoy→policy-engine gRPC access-log (ALS) + // transport that ships collected data. It is part of the collector and is + // configured under the shared [collector.als] section (the policy-engine reads + // the same section to configure its receiving ALS server). + GRPCEventServerCfg GRPCEventServerConfig `koanf:"als"` +} + // AnalyticsConfig holds analytics configuration type AnalyticsConfig struct { - Enabled bool `koanf:"enabled"` - EnabledPublishers []string `koanf:"enabled_publishers"` - Publishers AnalyticsPublishersConfig `koanf:"publishers"` - GRPCEventServerCfg GRPCEventServerConfig `koanf:"grpc_event_server"` - // AllowPayloads controls whether request and response bodies are captured - // into analytics metadata and forwarded to analytics publishers. - // Deprecated: use SendRequestBody and SendResponseBody instead. - // When true, validateAnalyticsConfig maps both SendRequestBody and SendResponseBody - // to true if both are false. Because bools cannot represent "unset", this also - // applies when both new flags are explicitly false; remove allow_payloads when - // migrating and set the directional flags directly. + Enabled bool `koanf:"enabled"` + EnabledPublishers []string `koanf:"enabled_publishers"` + Publishers AnalyticsPublishersConfig `koanf:"publishers"` + // GRPCEventServerCfg is a deprecated alias. ALS transport tuning moved to + // [collector.als]; when set here it is migrated onto the collector during + // validation (with a warning). Prefer [collector.als]. + GRPCEventServerCfg GRPCEventServerConfig `koanf:"grpc_event_server"` + // AllowPayloads, SendRequestBody and SendResponseBody are deprecated aliases. + // Body/header capture now lives under [collector]. When set, these are mapped + // onto collector.send_request_body / collector.send_response_body during + // validation (with a warning) for backward compatibility. Prefer the + // [collector] fields directly. AllowPayloads bool `koanf:"allow_payloads"` SendRequestBody bool `koanf:"send_request_body"` SendResponseBody bool `koanf:"send_response_body"` @@ -690,6 +717,26 @@ func LoadConfig(configPath string) (*Config, error) { return cfg, nil } +// defaultGRPCEventServerConfig returns the default Envoy→policy-engine ALS +// transport tuning. Shared by the collector (canonical) and the deprecated +// [analytics].grpc_event_server alias so a partial alias override migrates cleanly. +func defaultGRPCEventServerConfig() GRPCEventServerConfig { + return GRPCEventServerConfig{ + Mode: "uds", // UDS mode by default + Port: 18090, // Only used in TCP mode + ServerPort: 18090, // ALS server port + BufferFlushInterval: 1000000000, // 1 second + BufferSizeBytes: 16384, // 16 KiB + GRPCRequestTimeout: 20000000000, // 20 seconds + ShutdownTimeout: 600 * time.Second, + PublicKeyPath: "", + PrivateKeyPath: "", + ALSPlainText: true, + MaxMessageSize: 1000000000, + MaxHeaderLimit: 8192, + } +} + // defaultConfig returns a Config struct with default configuration values func defaultConfig() *Config { return &Config{ @@ -908,23 +955,20 @@ func defaultConfig() *Config { TimerWakeupSeconds: 3, }, }, - GRPCEventServerCfg: GRPCEventServerConfig{ - Mode: "uds", // UDS mode by default - Port: 18090, // Only used in TCP mode - ServerPort: 18090, // ALS server port - BufferFlushInterval: 1000000000, // 1 second - BufferSizeBytes: 16384, // 16 KiB - GRPCRequestTimeout: 20000000000, // 20 seconds - ShutdownTimeout: 600 * time.Second, - PublicKeyPath: "", - PrivateKeyPath: "", - ALSPlainText: true, - MaxMessageSize: 1000000000, - MaxHeaderLimit: 8192, - }, - AllowPayloads: false, - SendRequestBody: false, - SendResponseBody: false, + // Deprecated alias: default mirrors the collector so a partial + // [analytics.grpc_event_server] override migrates cleanly. + GRPCEventServerCfg: defaultGRPCEventServerConfig(), + AllowPayloads: false, + SendRequestBody: false, + SendResponseBody: false, + }, + Collector: CollectorConfig{ + Enabled: false, + SendRequestBody: false, + SendResponseBody: false, + SendRequestHeaders: false, + SendResponseHeaders: false, + GRPCEventServerCfg: defaultGRPCEventServerConfig(), }, TracingConfig: TracingConfig{ Enabled: false, @@ -1291,7 +1335,7 @@ func (c *Config) Validate() error { return err } - if err := c.validateAnalyticsConfig(); err != nil { + if err := c.validateCollectorConfig(); err != nil { return err } @@ -1697,51 +1741,96 @@ func validateDomains(field string, domains []string) error { return nil } -// validateAnalyticsConfig validates the analytics configuration -func (c *Config) validateAnalyticsConfig() error { - // Validate analytics configuration - if c.Analytics.Enabled { - // Migration path for deprecated analytics.allow_payloads. - // Runs when both directional flags are false, which is indistinguishable - // from "not set" because bool fields cannot represent unset vs explicit false. - if c.Analytics.AllowPayloads { - slog.Warn("analytics.allow_payloads is deprecated; use analytics.send_request_body and analytics.send_response_body instead") - if !c.Analytics.SendRequestBody && !c.Analytics.SendResponseBody { - c.Analytics.SendRequestBody = true - c.Analytics.SendResponseBody = true - } +// validateCollectorConfig migrates deprecated analytics aliases onto the collector +// and enforces the collector prerequisite: analytics (a consumer) requires the +// collector that feeds it. For backward compatibility this is a soft prerequisite — +// if a consumer is enabled without the collector, the collector is auto-enabled with +// a warning rather than failing. It also validates the ALS transport tuning when the +// collector is enabled. +func (c *Config) validateCollectorConfig() error { + c.migrateDeprecatedAnalyticsCapture() + c.migrateDeprecatedAnalyticsTransport() + + // Backward-compat bridge: a consumer cannot run without the collector that feeds + // it, but rather than fail an existing config that only set analytics.enabled + // (valid before the collector split), auto-enable the collector with a warning. + if c.Analytics.Enabled && !c.Collector.Enabled { + slog.Warn("analytics.enabled requires the collector; enabling collector.enabled automatically for backward compatibility. Set collector.enabled = true explicitly to silence this warning.") + c.Collector.Enabled = true + } + if c.Collector.Enabled { + if err := validateGRPCEventServerConfig(c.Collector.GRPCEventServerCfg); err != nil { + return err } + } + return nil +} - // Validate gRPC event server configuration - grpcEventServerCfg := c.Analytics.GRPCEventServerCfg - - // Validate connection mode - switch grpcEventServerCfg.Mode { - case "uds", "": - // UDS mode (default) - port is unused for Envoy connection - case "tcp": - // TCP mode - validate port (host is derived from policy_engine.host) - if grpcEventServerCfg.Port <= 0 || grpcEventServerCfg.Port > 65535 { - return fmt.Errorf("analytics.grpc_event_server.port must be between 1 and 65535 when mode is tcp, got %d", grpcEventServerCfg.Port) - } - default: - return fmt.Errorf("analytics.grpc_event_server.mode must be 'uds' or 'tcp', got: %s", grpcEventServerCfg.Mode) +// migrateDeprecatedAnalyticsTransport maps a deprecated [analytics].grpc_event_server +// override onto the collector when the collector's transport tuning is still at its +// default, so existing configs keep working after the transport moved to [collector]. +func (c *Config) migrateDeprecatedAnalyticsTransport() { + def := defaultGRPCEventServerConfig() + if c.Analytics.GRPCEventServerCfg != def { + slog.Warn("analytics.grpc_event_server is deprecated; use collector.als instead") + if c.Collector.GRPCEventServerCfg == def { + c.Collector.GRPCEventServerCfg = c.Analytics.GRPCEventServerCfg } + } +} - // Validate buffer and timeout settings - if grpcEventServerCfg.BufferFlushInterval <= 0 || grpcEventServerCfg.BufferSizeBytes <= 0 || grpcEventServerCfg.GRPCRequestTimeout <= 0 { - return fmt.Errorf( - "invalid gRPC event server configuration: bufferFlushInterval=%d, bufferSizeBytes=%d, grpcRequestTimeout=%d (all must be > 0)", - grpcEventServerCfg.BufferFlushInterval, - grpcEventServerCfg.BufferSizeBytes, - grpcEventServerCfg.GRPCRequestTimeout, - ) +// migrateDeprecatedAnalyticsCapture maps the deprecated analytics.allow_payloads / +// analytics.send_request_body / analytics.send_response_body onto the collector's +// body-capture flags (when the collector flag is not already set), so existing +// configs keep working after capture settings moved under [collector]. +func (c *Config) migrateDeprecatedAnalyticsCapture() { + // Directional aliases take precedence over allow_payloads. + if c.Analytics.SendRequestBody && !c.Collector.SendRequestBody { + slog.Warn("analytics.send_request_body is deprecated; use collector.send_request_body instead") + c.Collector.SendRequestBody = true + } + if c.Analytics.SendResponseBody && !c.Collector.SendResponseBody { + slog.Warn("analytics.send_response_body is deprecated; use collector.send_response_body instead") + c.Collector.SendResponseBody = true + } + // allow_payloads only fills in when no directional body capture is configured. + if c.Analytics.AllowPayloads { + slog.Warn("analytics.allow_payloads is deprecated; use collector.send_request_body and collector.send_response_body instead") + if !c.Collector.SendRequestBody && !c.Collector.SendResponseBody { + c.Collector.SendRequestBody = true + c.Collector.SendResponseBody = true } + } +} - // Validate server port - if grpcEventServerCfg.ServerPort <= 0 || grpcEventServerCfg.ServerPort > 65535 { - return fmt.Errorf("analytics.grpc_event_server.server_port must be between 1 and 65535, got %d", grpcEventServerCfg.ServerPort) +// validateGRPCEventServerConfig validates the Envoy→policy-engine ALS transport tuning. +func validateGRPCEventServerConfig(cfg GRPCEventServerConfig) error { + // Validate connection mode + switch cfg.Mode { + case "uds", "": + // UDS mode (default) - port is unused for Envoy connection + case "tcp": + // TCP mode - validate port (host is derived from policy_engine.host) + if cfg.Port <= 0 || cfg.Port > 65535 { + return fmt.Errorf("collector.als.port must be between 1 and 65535 when mode is tcp, got %d", cfg.Port) } + default: + return fmt.Errorf("collector.als.mode must be 'uds' or 'tcp', got: %s", cfg.Mode) + } + + // Validate buffer and timeout settings + if cfg.BufferFlushInterval <= 0 || cfg.BufferSizeBytes <= 0 || cfg.GRPCRequestTimeout <= 0 { + return fmt.Errorf( + "invalid gRPC event server configuration: bufferFlushInterval=%d, bufferSizeBytes=%d, grpcRequestTimeout=%d (all must be > 0)", + cfg.BufferFlushInterval, + cfg.BufferSizeBytes, + cfg.GRPCRequestTimeout, + ) + } + + // Validate server port + if cfg.ServerPort <= 0 || cfg.ServerPort > 65535 { + return fmt.Errorf("collector.als.server_port must be between 1 and 65535, got %d", cfg.ServerPort) } return nil } diff --git a/gateway/gateway-controller/pkg/config/config_test.go b/gateway/gateway-controller/pkg/config/config_test.go index 91e53733a4..e17039112d 100644 --- a/gateway/gateway-controller/pkg/config/config_test.go +++ b/gateway/gateway-controller/pkg/config/config_test.go @@ -102,6 +102,14 @@ func validConfig() *Config { ServerHeaderTransformation: commonconstants.OVERWRITE, }, }, + // Transport defaults mirror production so collector-gated grpc validation + // passes and the deprecated [analytics] alias stays neutral. + Collector: CollectorConfig{ + GRPCEventServerCfg: defaultGRPCEventServerConfig(), + }, + Analytics: AnalyticsConfig{ + GRPCEventServerCfg: defaultGRPCEventServerConfig(), + }, } } @@ -1205,11 +1213,11 @@ func TestConfig_ValidateAnalyticsConfig(t *testing.T) { name: "Analytics enabled with valid UDS config (default mode)", enabled: true, setupConfig: func(cfg *Config) { - cfg.Analytics.GRPCEventServerCfg.Mode = "uds" - cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 1000 - cfg.Analytics.GRPCEventServerCfg.BufferSizeBytes = 16384 - cfg.Analytics.GRPCEventServerCfg.GRPCRequestTimeout = 5000 - cfg.Analytics.GRPCEventServerCfg.ServerPort = 18090 + cfg.Collector.GRPCEventServerCfg.Mode = "uds" + cfg.Collector.GRPCEventServerCfg.BufferFlushInterval = 1000 + cfg.Collector.GRPCEventServerCfg.BufferSizeBytes = 16384 + cfg.Collector.GRPCEventServerCfg.GRPCRequestTimeout = 5000 + cfg.Collector.GRPCEventServerCfg.ServerPort = 18090 }, wantErr: false, }, @@ -1217,12 +1225,12 @@ func TestConfig_ValidateAnalyticsConfig(t *testing.T) { name: "Analytics enabled with valid TCP config", enabled: true, setupConfig: func(cfg *Config) { - cfg.Analytics.GRPCEventServerCfg.Mode = "tcp" - cfg.Analytics.GRPCEventServerCfg.Port = 18090 - cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 1000 - cfg.Analytics.GRPCEventServerCfg.BufferSizeBytes = 16384 - cfg.Analytics.GRPCEventServerCfg.GRPCRequestTimeout = 5000 - cfg.Analytics.GRPCEventServerCfg.ServerPort = 18090 + cfg.Collector.GRPCEventServerCfg.Mode = "tcp" + cfg.Collector.GRPCEventServerCfg.Port = 18090 + cfg.Collector.GRPCEventServerCfg.BufferFlushInterval = 1000 + cfg.Collector.GRPCEventServerCfg.BufferSizeBytes = 16384 + cfg.Collector.GRPCEventServerCfg.GRPCRequestTimeout = 5000 + cfg.Collector.GRPCEventServerCfg.ServerPort = 18090 }, wantErr: false, }, @@ -1230,11 +1238,11 @@ func TestConfig_ValidateAnalyticsConfig(t *testing.T) { name: "Analytics enabled with empty mode defaults to UDS", enabled: true, setupConfig: func(cfg *Config) { - cfg.Analytics.GRPCEventServerCfg.Mode = "" - cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 1000 - cfg.Analytics.GRPCEventServerCfg.BufferSizeBytes = 16384 - cfg.Analytics.GRPCEventServerCfg.GRPCRequestTimeout = 5000 - cfg.Analytics.GRPCEventServerCfg.ServerPort = 18090 + cfg.Collector.GRPCEventServerCfg.Mode = "" + cfg.Collector.GRPCEventServerCfg.BufferFlushInterval = 1000 + cfg.Collector.GRPCEventServerCfg.BufferSizeBytes = 16384 + cfg.Collector.GRPCEventServerCfg.GRPCRequestTimeout = 5000 + cfg.Collector.GRPCEventServerCfg.ServerPort = 18090 }, wantErr: false, }, @@ -1242,49 +1250,49 @@ func TestConfig_ValidateAnalyticsConfig(t *testing.T) { name: "Invalid mode value", enabled: true, setupConfig: func(cfg *Config) { - cfg.Analytics.GRPCEventServerCfg.Mode = "invalid" - cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 1000 - cfg.Analytics.GRPCEventServerCfg.BufferSizeBytes = 16384 - cfg.Analytics.GRPCEventServerCfg.GRPCRequestTimeout = 5000 - cfg.Analytics.GRPCEventServerCfg.ServerPort = 18090 + cfg.Collector.GRPCEventServerCfg.Mode = "invalid" + cfg.Collector.GRPCEventServerCfg.BufferFlushInterval = 1000 + cfg.Collector.GRPCEventServerCfg.BufferSizeBytes = 16384 + cfg.Collector.GRPCEventServerCfg.GRPCRequestTimeout = 5000 + cfg.Collector.GRPCEventServerCfg.ServerPort = 18090 }, wantErr: true, - errContains: "grpc_event_server.mode must be 'uds' or 'tcp'", + errContains: "collector.als.mode must be 'uds' or 'tcp'", }, { name: "TCP mode - invalid port", enabled: true, setupConfig: func(cfg *Config) { - cfg.Analytics.GRPCEventServerCfg.Mode = "tcp" - cfg.Analytics.GRPCEventServerCfg.Port = 0 - cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 1000 - cfg.Analytics.GRPCEventServerCfg.BufferSizeBytes = 16384 - cfg.Analytics.GRPCEventServerCfg.GRPCRequestTimeout = 5000 - cfg.Analytics.GRPCEventServerCfg.ServerPort = 18090 + cfg.Collector.GRPCEventServerCfg.Mode = "tcp" + cfg.Collector.GRPCEventServerCfg.Port = 0 + cfg.Collector.GRPCEventServerCfg.BufferFlushInterval = 1000 + cfg.Collector.GRPCEventServerCfg.BufferSizeBytes = 16384 + cfg.Collector.GRPCEventServerCfg.GRPCRequestTimeout = 5000 + cfg.Collector.GRPCEventServerCfg.ServerPort = 18090 }, wantErr: true, - errContains: "grpc_event_server.port must be between 1 and 65535", + errContains: "collector.als.port must be between 1 and 65535", }, { name: "Invalid server port", enabled: true, setupConfig: func(cfg *Config) { - cfg.Analytics.GRPCEventServerCfg.Mode = "uds" - cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 1000 - cfg.Analytics.GRPCEventServerCfg.BufferSizeBytes = 16384 - cfg.Analytics.GRPCEventServerCfg.GRPCRequestTimeout = 5000 - cfg.Analytics.GRPCEventServerCfg.ServerPort = 0 + cfg.Collector.GRPCEventServerCfg.Mode = "uds" + cfg.Collector.GRPCEventServerCfg.BufferFlushInterval = 1000 + cfg.Collector.GRPCEventServerCfg.BufferSizeBytes = 16384 + cfg.Collector.GRPCEventServerCfg.GRPCRequestTimeout = 5000 + cfg.Collector.GRPCEventServerCfg.ServerPort = 0 }, wantErr: true, - errContains: "grpc_event_server.server_port must be between 1 and 65535", + errContains: "collector.als.server_port must be between 1 and 65535", }, { name: "Invalid buffer flush interval", enabled: true, setupConfig: func(cfg *Config) { - cfg.Analytics.GRPCEventServerCfg.Mode = "uds" - cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 0 - cfg.Analytics.GRPCEventServerCfg.ServerPort = 18090 + cfg.Collector.GRPCEventServerCfg.Mode = "uds" + cfg.Collector.GRPCEventServerCfg.BufferFlushInterval = 0 + cfg.Collector.GRPCEventServerCfg.ServerPort = 18090 }, wantErr: true, errContains: "invalid gRPC event server configuration", @@ -1295,6 +1303,9 @@ func TestConfig_ValidateAnalyticsConfig(t *testing.T) { t.Run(tt.name, func(t *testing.T) { cfg := validConfig() cfg.Analytics.Enabled = tt.enabled + // Analytics is a consumer; enable the collector it depends on so these + // tests exercise analytics validation rather than the prerequisite check. + cfg.Collector.Enabled = tt.enabled if tt.setupConfig != nil { tt.setupConfig(cfg) } @@ -1309,9 +1320,34 @@ func TestConfig_ValidateAnalyticsConfig(t *testing.T) { } } +func TestConfig_CollectorPrerequisite(t *testing.T) { + t.Run("analytics enabled without collector auto-enables the collector", func(t *testing.T) { + cfg := validConfig() + cfg.Collector.Enabled = false + cfg.Analytics.Enabled = true + cfg.Analytics.EnabledPublishers = []string{} + // Deprecated transport alias with valid values migrates onto the auto-enabled collector. + cfg.Analytics.GRPCEventServerCfg.Mode = "uds" + cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 1000 + cfg.Analytics.GRPCEventServerCfg.BufferSizeBytes = 16384 + cfg.Analytics.GRPCEventServerCfg.GRPCRequestTimeout = 5000 + cfg.Analytics.GRPCEventServerCfg.ServerPort = 18090 + require.NoError(t, cfg.Validate()) + assert.True(t, cfg.Collector.Enabled, "collector should be auto-enabled for backward compatibility") + }) + + t.Run("collector enabled with no consumers is valid", func(t *testing.T) { + cfg := validConfig() + cfg.Collector.Enabled = true + require.NoError(t, cfg.Validate()) + }) + +} + func TestConfig_ValidateAnalyticsPayloadMigration(t *testing.T) { setValidAnalyticsGRPC := func(cfg *Config) { cfg.Analytics.Enabled = true + cfg.Collector.Enabled = true // analytics is a consumer; the collector must be on cfg.Analytics.GRPCEventServerCfg.Mode = "uds" cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 1000 cfg.Analytics.GRPCEventServerCfg.BufferSizeBytes = 16384 @@ -1371,8 +1407,9 @@ func TestConfig_ValidateAnalyticsPayloadMigration(t *testing.T) { err := cfg.Validate() require.NoError(t, err) - assert.Equal(t, tt.wantSendReq, cfg.Analytics.SendRequestBody) - assert.Equal(t, tt.wantSendResp, cfg.Analytics.SendResponseBody) + // Deprecated analytics body aliases now migrate onto the collector. + assert.Equal(t, tt.wantSendReq, cfg.Collector.SendRequestBody) + assert.Equal(t, tt.wantSendResp, cfg.Collector.SendResponseBody) }) } } diff --git a/gateway/gateway-controller/pkg/utils/system_policies.go b/gateway/gateway-controller/pkg/utils/system_policies.go index 8ad1da067f..25a598a634 100644 --- a/gateway/gateway-controller/pkg/utils/system_policies.go +++ b/gateway/gateway-controller/pkg/utils/system_policies.go @@ -84,13 +84,18 @@ var defaultSystemPolicies = []systemPolicyConfig{ if cfg == nil { return false } - slog.Debug("Analytics state -> ", "state", cfg.Analytics.Enabled) - return cfg.Analytics.Enabled + // The analytics system policy is the collector: it is injected whenever + // the collector is enabled, regardless of which consumer (analytics, + // traffic logging) ultimately reads the collected data. + slog.Debug("Collector state -> ", "state", cfg.Collector.Enabled) + return cfg.Collector.Enabled }, // Default parameters (can be overridden via additionalProps) Parameters: map[string]interface{}{ - "send_request_body": false, - "send_response_body": false, + "send_request_body": false, + "send_response_body": false, + "send_request_headers": false, + "send_response_headers": false, }, ExecutionCondition: nil, }, @@ -197,10 +202,13 @@ func InjectSystemPolicies(policies []policyenginev1.PolicyInstance, cfg *config. for k, v := range sysPol.Parameters { effectiveDefaults[k] = v } - // For the analytics system policy, propagate the payload flags from runtime config. + // For the analytics (collector) system policy, propagate the payload and + // header capture flags from the collector config. if sysPol.Name == constants.ANALYTICS_SYSTEM_POLICY_NAME { - effectiveDefaults["send_request_body"] = cfg.Analytics.SendRequestBody - effectiveDefaults["send_response_body"] = cfg.Analytics.SendResponseBody + effectiveDefaults["send_request_body"] = cfg.Collector.SendRequestBody + effectiveDefaults["send_response_body"] = cfg.Collector.SendResponseBody + effectiveDefaults["send_request_headers"] = cfg.Collector.SendRequestHeaders + effectiveDefaults["send_response_headers"] = cfg.Collector.SendResponseHeaders } // Merge parameters efficiently diff --git a/gateway/gateway-controller/pkg/utils/system_policies_test.go b/gateway/gateway-controller/pkg/utils/system_policies_test.go index d0a0f1a61e..f324db0377 100644 --- a/gateway/gateway-controller/pkg/utils/system_policies_test.go +++ b/gateway/gateway-controller/pkg/utils/system_policies_test.go @@ -124,9 +124,9 @@ func TestInjectSystemPolicies_NilConfig(t *testing.T) { assert.Equal(t, policies, result) } -func TestInjectSystemPolicies_AnalyticsDisabled(t *testing.T) { +func TestInjectSystemPolicies_CollectorDisabled(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ + Collector: config.CollectorConfig{ Enabled: false, }, } @@ -139,9 +139,9 @@ func TestInjectSystemPolicies_AnalyticsDisabled(t *testing.T) { assert.Equal(t, "existing", result[0].Name) } -func TestInjectSystemPolicies_AnalyticsEnabled(t *testing.T) { +func TestInjectSystemPolicies_CollectorEnabled(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ + Collector: config.CollectorConfig{ Enabled: true, }, } @@ -159,11 +159,11 @@ func TestInjectSystemPolicies_AnalyticsEnabled(t *testing.T) { assert.Equal(t, "existing", result[1].Name) } -func TestInjectSystemPolicies_AllowPayloadsTrue(t *testing.T) { +func TestInjectSystemPolicies_BodyFlagsPropagated(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ - Enabled: true, - SendRequestBody: true, + Collector: config.CollectorConfig{ + Enabled: true, + SendRequestBody: true, SendResponseBody: true, }, } @@ -175,11 +175,11 @@ func TestInjectSystemPolicies_AllowPayloadsTrue(t *testing.T) { assert.Equal(t, true, result[0].Parameters["send_response_body"]) } -func TestInjectSystemPolicies_AllowPayloadsFalse(t *testing.T) { +func TestInjectSystemPolicies_BodyFlagsDefaultFalse(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ - Enabled: true, - SendRequestBody: false, + Collector: config.CollectorConfig{ + Enabled: true, + SendRequestBody: false, SendResponseBody: false, }, } @@ -190,9 +190,37 @@ func TestInjectSystemPolicies_AllowPayloadsFalse(t *testing.T) { assert.Equal(t, false, result[0].Parameters["send_response_body"]) } +func TestInjectSystemPolicies_HeaderFlagsPropagated(t *testing.T) { + cfg := &config.Config{ + Collector: config.CollectorConfig{ + Enabled: true, + SendRequestHeaders: true, + SendResponseHeaders: true, + }, + } + + result := InjectSystemPolicies(nil, cfg, nil) + assert.Len(t, result, 1) + assert.Equal(t, constants.ANALYTICS_SYSTEM_POLICY_NAME, result[0].Name) + assert.Equal(t, true, result[0].Parameters["send_request_headers"]) + assert.Equal(t, true, result[0].Parameters["send_response_headers"]) +} + +func TestInjectSystemPolicies_HeaderFlagsDefaultFalse(t *testing.T) { + cfg := &config.Config{ + Collector: config.CollectorConfig{Enabled: true}, + } + + result := InjectSystemPolicies(nil, cfg, nil) + assert.Len(t, result, 1) + assert.Equal(t, false, result[0].Parameters["send_request_headers"]) + assert.Equal(t, false, result[0].Parameters["send_response_headers"]) +} + + func TestInjectSystemPolicies_WithAdditionalProps(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ + Collector: config.CollectorConfig{ Enabled: true, }, } @@ -209,7 +237,7 @@ func TestInjectSystemPolicies_WithAdditionalProps(t *testing.T) { func TestInjectSystemPolicies_WithSharedParams(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ + Collector: config.CollectorConfig{ Enabled: true, }, } @@ -226,7 +254,7 @@ func TestInjectSystemPolicies_WithSharedParams(t *testing.T) { func TestInjectSystemPolicies_EmptyPolicies(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ + Collector: config.CollectorConfig{ Enabled: true, }, } @@ -238,7 +266,7 @@ func TestInjectSystemPolicies_EmptyPolicies(t *testing.T) { func TestInjectSystemPolicies_PreservesExistingPolicies(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ + Collector: config.CollectorConfig{ Enabled: true, }, } diff --git a/gateway/gateway-controller/pkg/xds/translator.go b/gateway/gateway-controller/pkg/xds/translator.go index 408a6dda7a..7ae5387236 100644 --- a/gateway/gateway-controller/pkg/xds/translator.go +++ b/gateway/gateway-controller/pkg/xds/translator.go @@ -561,10 +561,10 @@ func (t *Translator) TranslateConfigs( policyEngineCluster := t.createPolicyEngineCluster() clusters = append(clusters, policyEngineCluster) - // Add ALS cluster if gRPC access log is enabled - log.Debug("gRPC event server config", slog.Any("config", t.config.Analytics.GRPCEventServerCfg)) - if t.config.Analytics.Enabled { - log.Info("gRPC access log is enabled, creating ALS cluster") + // Add ALS cluster if the collector is enabled (it ships access logs over gRPC) + log.Debug("gRPC event server config", slog.Any("config", t.config.Collector.GRPCEventServerCfg)) + if t.config.Collector.Enabled { + log.Info("collector is enabled, creating ALS cluster") alsCluster := t.createALSCluster() clusters = append(clusters, alsCluster) } @@ -2156,7 +2156,7 @@ func (t *Translator) createPolicyEngineCluster() *cluster.Cluster { // createALSCluster creates an Envoy cluster for the gRPC access log service func (t *Translator) createALSCluster() *cluster.Cluster { - grpcConfig := t.config.Analytics.GRPCEventServerCfg + grpcConfig := t.config.Collector.GRPCEventServerCfg // Build the endpoint address (UDS or TCP) var address *core.Address @@ -2785,8 +2785,8 @@ func (t *Translator) createAccessLogConfig() ([]*accesslog.AccessLog, error) { }, }) - // If gRPC access log is enabled, create the configuration and append to existing access logs - if t.config.Analytics.Enabled { + // If the collector is enabled, create the gRPC access log config and append to existing access logs + if t.config.Collector.Enabled { t.logger.Info("Creating gRPC access log configuration") grpcAccessLog, err := t.createGRPCAccessLog() if err != nil { @@ -2802,8 +2802,8 @@ func (t *Translator) createAccessLogConfig() ([]*accesslog.AccessLog, error) { // createGRPCAccessLog creates a gRPC access log configuration for the gateway controller func (t *Translator) createGRPCAccessLog() (*accesslog.AccessLog, error) { - grpcConfig := t.config.Analytics.GRPCEventServerCfg - bufferSizeBytes, err := checkedUInt32FromPositiveInt("analytics.grpc_event_server.buffer_size_bytes", grpcConfig.BufferSizeBytes) + grpcConfig := t.config.Collector.GRPCEventServerCfg + bufferSizeBytes, err := checkedUInt32FromPositiveInt("collector.als.buffer_size_bytes", grpcConfig.BufferSizeBytes) if err != nil { return nil, err } diff --git a/gateway/gateway-controller/pkg/xds/translator_test.go b/gateway/gateway-controller/pkg/xds/translator_test.go index dc637583e8..4a99e0036d 100644 --- a/gateway/gateway-controller/pkg/xds/translator_test.go +++ b/gateway/gateway-controller/pkg/xds/translator_test.go @@ -1330,7 +1330,7 @@ func TestTranslator_CreateALSCluster(t *testing.T) { routerCfg := testRouterConfig() cfg := testConfig() cfg.Analytics.Enabled = true - cfg.Analytics.GRPCEventServerCfg = config.GRPCEventServerConfig{ + cfg.Collector.GRPCEventServerCfg = config.GRPCEventServerConfig{ Mode: "uds", BufferFlushInterval: 1000000000, BufferSizeBytes: 16384, @@ -1357,7 +1357,7 @@ func TestTranslator_CreateALSCluster(t *testing.T) { routerCfg := testRouterConfig() cfg := testConfig() cfg.Analytics.Enabled = true - cfg.Analytics.GRPCEventServerCfg = config.GRPCEventServerConfig{ + cfg.Collector.GRPCEventServerCfg = config.GRPCEventServerConfig{ Mode: "", BufferFlushInterval: 1000000000, BufferSizeBytes: 16384, @@ -1383,7 +1383,7 @@ func TestTranslator_CreateALSCluster(t *testing.T) { routerCfg := testRouterConfig() cfg := testConfig() cfg.Analytics.Enabled = true - cfg.Analytics.GRPCEventServerCfg = config.GRPCEventServerConfig{ + cfg.Collector.GRPCEventServerCfg = config.GRPCEventServerConfig{ Mode: "tcp", Port: 18090, BufferFlushInterval: 1000000000, @@ -1415,7 +1415,7 @@ func TestTranslator_CreateGRPCAccessLog(t *testing.T) { logger := createTestLogger() routerCfg := testRouterConfig() cfg := testConfig() - cfg.Analytics.GRPCEventServerCfg = config.GRPCEventServerConfig{ + cfg.Collector.GRPCEventServerCfg = config.GRPCEventServerConfig{ Mode: "tcp", Port: 18090, BufferFlushInterval: 1000, @@ -1433,7 +1433,7 @@ func TestTranslator_CreateGRPCAccessLog_BufferSizeOverflow(t *testing.T) { logger := createTestLogger() routerCfg := testRouterConfig() cfg := testConfig() - cfg.Analytics.GRPCEventServerCfg = config.GRPCEventServerConfig{ + cfg.Collector.GRPCEventServerCfg = config.GRPCEventServerConfig{ Mode: "tcp", Port: 18090, BufferFlushInterval: 1000, diff --git a/gateway/gateway-runtime/policy-engine/cmd/policy-engine/main.go b/gateway/gateway-runtime/policy-engine/cmd/policy-engine/main.go index 031f527745..b7116fd01c 100644 --- a/gateway/gateway-runtime/policy-engine/cmd/policy-engine/main.go +++ b/gateway/gateway-runtime/policy-engine/cmd/policy-engine/main.go @@ -268,10 +268,12 @@ func main() { metrics.StartMemoryMetricsUpdater(ctx, 15*time.Second) } - // Start access log service server if enabled + // Start the access log service server when the collector is enabled. The + // collector is the shared transport that carries collected data to its + // consumers (analytics, traffic logging). var alsServer *grpc.Server - slog.DebugContext(ctx, "Policy engine ALS server config", "config", cfg.Analytics.AccessLogsServiceCfg) - if cfg.Analytics.Enabled { + slog.DebugContext(ctx, "Policy engine ALS server config", "config", cfg.Collector.AccessLogsServiceCfg) + if cfg.Collector.Enabled { // Start the access log service server slog.Info("Starting the ALS gRPC server...") alsServer = utils.StartAccessLogServiceServer(cfg) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go index 2b84536570..8960cfba2b 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go @@ -59,11 +59,19 @@ const ( DefaultAnalyticsPublisher = "default" // MoesifAnalyticsPublisher represents the Moesif analytics publisher. MoesifAnalyticsPublisher = "moesif" + // LogAnalyticsPublisher represents the stdout/log analytics publisher. + LogAnalyticsPublisher = "log" // HeaderKeys represents the header keys. RequestHeadersKey = "request_headers" ResponseHeadersKey = "response_headers" + // TrafficLogMetadataKey is the analytics-metadata key under which the + // log-message policy (in access-log mode) stamps its per-API opt-in marker. + // Its presence gates the stdout traffic-logging publisher; its value (a JSON + // directive) shapes the emitted line. Must match the key used by that policy. + TrafficLogMetadataKey = "traffic_log" + // PromptTokenCountMetadataKey represents the prompt token count metadata key. PromptTokenCountMetadataKey string = "aitoken:prompttokencount" // CompletionTokenCountMetadataKey represents the completion token count metadata key. @@ -90,27 +98,39 @@ type Analytics struct { publishers []analytics_publisher.Publisher } -// NewAnalytics creates a new instance of Analytics. +// NewAnalytics creates a new instance of Analytics. Publishers are assembled from +// each independently-configured consumer of the collected data: the analytics +// consumer ([analytics], e.g. Moesif) and the traffic-logging consumer +// ([traffic_logging], stdout JSON). Both rely on the collector being enabled to +// receive any events. func NewAnalytics(cfg *config.Config) *Analytics { analyticsCfg := cfg.Analytics publishers := make([]analytics_publisher.Publisher, 0) if analyticsCfg.Enabled { for _, publisherName := range analyticsCfg.EnabledPublishers { switch publisherName { - case "moesif": + case MoesifAnalyticsPublisher: publisher := analytics_publisher.NewMoesif(&analyticsCfg.Publishers.Moesif) if publisher != nil { publishers = append(publishers, publisher) slog.Info("Moesif publisher added") } + case LogAnalyticsPublisher: + slog.Warn("\"log\" in analytics.enabled_publishers is no longer supported; enable stdout traffic logging via [traffic_logging] instead") default: slog.Warn("Unknown publisher type", "type", publisherName) } } } + // Traffic logging is a standalone consumer, independent of analytics. + if cfg.TrafficLogging.Enabled { + publishers = append(publishers, analytics_publisher.NewLog(&cfg.TrafficLogging)) + slog.Info("Traffic logging (stdout) publisher added") + } + if len(publishers) == 0 { - slog.Debug("No analytics publishers found. Analytics will not be published.") + slog.Debug("No analytics publishers found. Collected events will not be published.") } return &Analytics{ cfg: cfg, @@ -133,7 +153,9 @@ func (c *Analytics) Process(event *v3.HTTPAccessLogEntry) { return } - // Add logic to publish the event + // Path-based suppression is a per-consumer presentation concern: it is applied + // by the stdout traffic-logging publisher (traffic_logging.ignored_path_prefixes), + // not here, so other consumers still receive every event. analyticEvent := c.prepareAnalyticEvent(event) for _, publisher := range c.publishers { publisher.Publish(analyticEvent) @@ -197,6 +219,23 @@ func (c *Analytics) prepareAnalyticEvent(logEntry *v3.HTTPAccessLogEntry) *dto.E for key, value := range keyValuePairsFromMetadata { slog.Debug(fmt.Sprintf("Metadata key: %v -> value: %+v", key, value)) } + + // Extract the per-API traffic-log opt-in marker stamped by the log-message + // policy (access-log mode). Its presence marks the API as opted in to stdout + // traffic logging; + // its value shapes the emitted line. It is intentionally kept off + // event.Properties so it never reaches the serialized output or other + // publishers (e.g. Moesif). A malformed marker still opts the API in with + // default (empty) presentation, since attachment alone signals intent. + if raw, exists := keyValuePairsFromMetadata[TrafficLogMetadataKey]; exists { + dir := &dto.TrafficLogDirective{} + if raw != "" { + if err := json.Unmarshal([]byte(raw), dir); err != nil { + slog.Warn("Failed to parse traffic_log directive; opting in with defaults", "error", err) + } + } + event.TrafficLog = dir + } // Prepare extended API extendedAPI := dto.ExtendedAPI{} extendedAPI.APIType = keyValuePairsFromMetadata[APITypeKey] @@ -247,33 +286,36 @@ func (c *Analytics) prepareAnalyticEvent(logEntry *v3.HTTPAccessLogEntry) *dto.E properties.TimeToFirstUpstreamTxByte != nil && properties.TimeToFirstUpstreamRxByte != nil && properties.TimeToLastUpstreamRxByte != nil && properties.TimeToLastDownstreamTxByte != nil { - lastRx := - (properties.TimeToLastRxByte.Seconds * 1000) + - (int64(properties.TimeToLastRxByte.Nanos) / 1_000_000) - - firstUpTx := - (properties.TimeToFirstUpstreamTxByte.Seconds * 1000) + - (int64(properties.TimeToFirstUpstreamTxByte.Nanos) / 1_000_000) - - firstUpRx := - (properties.TimeToFirstUpstreamRxByte.Seconds * 1000) + - (int64(properties.TimeToFirstUpstreamRxByte.Nanos) / 1_000_000) - - lastUpRx := - (properties.TimeToLastUpstreamRxByte.Seconds * 1000) + - (int64(properties.TimeToLastUpstreamRxByte.Nanos) / 1_000_000) + toMs := func(secs int64, nanos int32) int64 { + return (secs * 1000) + int64(nanos)/1_000_000 + } - lastDownTx := - (properties.TimeToLastDownstreamTxByte.Seconds * 1000) + - (int64(properties.TimeToLastDownstreamTxByte.Nanos) / 1_000_000) + lastRx := toMs(properties.TimeToLastRxByte.Seconds, properties.TimeToLastRxByte.Nanos) + firstUpTx := toMs(properties.TimeToFirstUpstreamTxByte.Seconds, properties.TimeToFirstUpstreamTxByte.Nanos) + firstUpRx := toMs(properties.TimeToFirstUpstreamRxByte.Seconds, properties.TimeToFirstUpstreamRxByte.Nanos) + lastUpRx := toMs(properties.TimeToLastUpstreamRxByte.Seconds, properties.TimeToLastUpstreamRxByte.Nanos) + lastDownTx := toMs(properties.TimeToLastDownstreamTxByte.Seconds, properties.TimeToLastDownstreamTxByte.Nanos) latencies := dto.Latencies{ + Duration: lastDownTx, BackendLatency: lastUpRx - firstUpTx, RequestMediationLatency: firstUpTx - lastRx, ResponseLatency: lastDownTx - firstUpRx, ResponseMediationLatency: lastDownTx - lastUpRx, } + // US_TX_END → US_RX_BEG: time the backend spent before sending the first response byte (TTFB). + if properties.TimeToLastUpstreamTxByte != nil { + lastUpTx := toMs(properties.TimeToLastUpstreamTxByte.Seconds, properties.TimeToLastUpstreamTxByte.Nanos) + latencies.BackendProcDuration = firstUpRx - lastUpTx + } + + // US_RX_BEG → DS_TX_BEG: gateway overhead processing the first response byte before writing downstream. + if properties.TimeToFirstDownstreamTxByte != nil { + firstDownTx := toMs(properties.TimeToFirstDownstreamTxByte.Seconds, properties.TimeToFirstDownstreamTxByte.Nanos) + latencies.ResponseProcDuration = firstDownTx - firstUpRx + } + event.Latencies = &latencies } @@ -445,14 +487,14 @@ func (c *Analytics) prepareAnalyticEvent(logEntry *v3.HTTPAccessLogEntry) *dto.E event.Properties["responseHeaders"] = responseHeaders } - // Optionally attach request and response payloads when enabled via configuration. - if c.cfg.Analytics.SendRequestBody { + // Optionally attach request and response payloads when enabled via the collector. + if c.cfg.Collector.SendRequestBody { if requestPayload, ok := keyValuePairsFromMetadata["request_payload"]; ok && requestPayload != "" { event.Properties["request_payload"] = requestPayload slog.Debug("Analytics request payload captured", "size_bytes", len(requestPayload)) } } - if c.cfg.Analytics.SendResponseBody { + if c.cfg.Collector.SendResponseBody { if responsePayload, ok := keyValuePairsFromMetadata["response_payload"]; ok && responsePayload != "" { event.Properties["response_payload"] = responsePayload slog.Debug("Analytics response payload captured", "size_bytes", len(responsePayload)) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go index 99e6b4122e..4cc2ecf5d2 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go @@ -64,7 +64,9 @@ func validAnalyticsConfigForValidation(analytics config.AnalyticsConfig) *config Analytics: analytics, } cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = config.AccessLogsServiceConfig{ + // Analytics is a consumer; the collector must be enabled for it to validate. + cfg.Collector.Enabled = true + cfg.Collector.AccessLogsServiceCfg = config.AccessLogsServiceConfig{ Mode: "uds", ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -138,6 +140,34 @@ func TestNewAnalytics_EnabledWithUnknownPublisherType(t *testing.T) { assert.Empty(t, analytics.publishers) // Unknown type should not be added } +func TestNewAnalytics_TrafficLoggingEnabled(t *testing.T) { + // Traffic logging is a standalone consumer, independent of analytics. + cfg := &config.Config{ + TrafficLogging: config.TrafficLoggingConfig{Enabled: true}, + } + + analytics := NewAnalytics(cfg) + + require.NotNil(t, analytics) + assert.Len(t, analytics.publishers, 1) // traffic-logging publisher should be registered +} + +func TestNewAnalytics_LogInEnabledPublishersIgnored(t *testing.T) { + // "log" in analytics.enabled_publishers is no longer supported; it must not + // register a publisher (use [traffic_logging] instead). + cfg := &config.Config{ + Analytics: config.AnalyticsConfig{ + Enabled: true, + EnabledPublishers: []string{LogAnalyticsPublisher}, + }, + } + + analytics := NewAnalytics(cfg) + + require.NotNil(t, analytics) + assert.Empty(t, analytics.publishers) +} + // ============================================================================= // isInvalid Tests // ============================================================================= @@ -253,6 +283,24 @@ func TestProcess_WithMockPublisher(t *testing.T) { assert.Equal(t, "TestAPI", mockPub.event.API.APIName) } +// Path-based suppression moved to the traffic-logging publisher, so +// Analytics.Process no longer filters by path — every valid event reaches the +// publishers (each decides what to do with it). Ignored-path coverage lives in +// publishers/log_test.go. +func TestProcess_PublishesEvent(t *testing.T) { + analytics := NewAnalytics(&config.Config{}) + mockPub := &mockPublisher{} + analytics.publishers = append(analytics.publishers, mockPub) + + logEntry := &v3.HTTPAccessLogEntry{ + Response: &v3.HTTPResponseProperties{ResponseCode: wrapperspb.UInt32(200)}, + Request: &v3.HTTPRequestProperties{OriginalPath: "/api/v1/orders", RequestMethod: corev3.RequestMethod_GET}, + } + analytics.Process(logEntry) + + assert.True(t, mockPub.called, "publisher must be called") +} + func TestProcess_PanicRecovery(t *testing.T) { cfg := &config.Config{} analytics := NewAnalytics(cfg) @@ -373,6 +421,60 @@ func TestPrepareAnalyticEvent_WithFilterMetadata(t *testing.T) { assert.Equal(t, "TestApp", event.Application.ApplicationName) } +func TestPrepareAnalyticEvent_TrafficLogMarker(t *testing.T) { + cfg := &config.Config{} + analytics := NewAnalytics(cfg) + + logEntry := createLogEntryWithMetadata(map[string]string{ + APINameKey: "TestAPI", + TrafficLogMetadataKey: `{"request":{"payload":false,"headers":true,"excludeHeaders":["x-key"]}}`, + }) + + event := analytics.prepareAnalyticEvent(logEntry) + + require.NotNil(t, event) + require.NotNil(t, event.TrafficLog, "traffic_log marker should populate event.TrafficLog") + require.NotNil(t, event.TrafficLog.Request) + assert.True(t, event.TrafficLog.Request.Headers) + assert.False(t, event.TrafficLog.Request.Payload) + assert.Equal(t, []string{"x-key"}, event.TrafficLog.Request.ExcludeHeaders) + assert.Nil(t, event.TrafficLog.Response, "unset flow stays nil") + + // The marker must never leak into serialized properties (or other publishers). + _, inProps := event.Properties[TrafficLogMetadataKey] + assert.False(t, inProps, "traffic_log must not appear in event.Properties") +} + +func TestPrepareAnalyticEvent_NoTrafficLogMarker(t *testing.T) { + cfg := &config.Config{} + analytics := NewAnalytics(cfg) + + logEntry := createLogEntryWithMetadata(map[string]string{APINameKey: "TestAPI"}) + + event := analytics.prepareAnalyticEvent(logEntry) + + require.NotNil(t, event) + assert.Nil(t, event.TrafficLog, "no marker -> event.TrafficLog stays nil (publisher skips)") +} + +func TestPrepareAnalyticEvent_MalformedTrafficLogMarker(t *testing.T) { + cfg := &config.Config{} + analytics := NewAnalytics(cfg) + + logEntry := createLogEntryWithMetadata(map[string]string{ + TrafficLogMetadataKey: "not-json", + }) + + event := analytics.prepareAnalyticEvent(logEntry) + + require.NotNil(t, event) + // Presence alone is the opt-in signal; a malformed marker still opts in with + // default (empty) presentation. + require.NotNil(t, event.TrafficLog) + assert.Nil(t, event.TrafficLog.Request) + assert.Nil(t, event.TrafficLog.Response) +} + func TestPrepareAnalyticEvent_WithAnonymousApp(t *testing.T) { cfg := &config.Config{} analytics := NewAnalytics(cfg) @@ -525,7 +627,7 @@ func TestPrepareAnalyticEvent_WithRequestResponseHeaders(t *testing.T) { func TestPrepareAnalyticEvent_WithPayloadsEnabled(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ + Collector: config.CollectorConfig{ SendRequestBody: true, SendResponseBody: true, }, @@ -551,7 +653,7 @@ func TestPrepareAnalyticEvent_WithPayloadsEnabled(t *testing.T) { func TestPrepareAnalyticEvent_WithPayloadsDisabled(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ + Collector: config.CollectorConfig{ SendRequestBody: false, SendResponseBody: false, }, @@ -574,7 +676,7 @@ func TestPrepareAnalyticEvent_WithPayloadsDisabled(t *testing.T) { func TestPrepareAnalyticEvent_RequestPayloadOnly(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ + Collector: config.CollectorConfig{ SendRequestBody: true, SendResponseBody: false, }, @@ -598,7 +700,7 @@ func TestPrepareAnalyticEvent_RequestPayloadOnly(t *testing.T) { func TestPrepareAnalyticEvent_ResponsePayloadOnly(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ + Collector: config.CollectorConfig{ SendRequestBody: false, SendResponseBody: true, }, diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go index 6b2f05829b..75566db7e4 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go @@ -36,4 +36,38 @@ type Event struct { UserIP string `json:"userIp,omitempty" bson:"user_ip"` ErrorType string `json:"errorType,omitempty" bson:"error_type"` Properties map[string]interface{} `json:"properties,omitempty" bson:"properties"` + + // TrafficLog carries the per-API stdout traffic-logging opt-in marker stamped + // by the log-message policy (access-log mode). When nil, the API has not opted + // in and the stdout traffic-logging publisher skips the event. It is + // gating/presentation state only and is never serialized (json:"-") nor sent + // to other publishers. + TrafficLog *TrafficLogDirective `json:"-" bson:"-"` +} + +// TrafficLogDirective is the presentation config carried in the traffic-log +// marker. Field names mirror the policy's marker JSON so it round-trips. A nil +// flow means that flow was not configured. +type TrafficLogDirective struct { + Request *TrafficLogFlow `json:"request,omitempty"` + Response *TrafficLogFlow `json:"response,omitempty"` + Fields *TrafficLogFields `json:"fields,omitempty"` +} + +// TrafficLogFlow is the per-flow (request or response) presentation config. +type TrafficLogFlow struct { + Payload bool `json:"payload"` + Headers bool `json:"headers"` + ExcludeHeaders []string `json:"excludeHeaders,omitempty"` +} + +// TrafficLogFields selects which fields appear in the emitted line. When set it is +// authoritative over field presence: the per-flow Payload/Headers booleans are +// ignored (per-flow ExcludeHeaders and global masking still apply). Names are +// top-level keys (e.g. "latencies", "target") or dotted property paths +// (e.g. "properties.requestHeaders"). Mode "exclude" drops the named keys; any +// other value (default "include") keeps only the named keys. +type TrafficLogFields struct { + Mode string `json:"mode,omitempty"` + Names []string `json:"names,omitempty"` } diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/latencies.go b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/latencies.go index 412f95f460..e9e4818f0e 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/latencies.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/latencies.go @@ -23,7 +23,12 @@ type Latencies struct { BackendLatency int64 `json:"backendLatency"` RequestMediationLatency int64 `json:"requestMediationLatency"` ResponseMediationLatency int64 `json:"responseMediationLatency"` - Duration int64 `json:"duration"` + // Duration is the total request duration: downstream request received → downstream response sent (ms). + Duration int64 `json:"duration"` + // BackendProcDuration is the backend TTFB: upstream request fully sent → first upstream response byte (ms). + BackendProcDuration int64 `json:"backendProcDuration"` + // ResponseProcDuration is the gateway response overhead: first upstream response byte → first downstream response byte (ms). + ResponseProcDuration int64 `json:"responseProcDuration"` } // GetResponseLatency returns the response latency. diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go new file mode 100644 index 0000000000..c76cdf3db9 --- /dev/null +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go @@ -0,0 +1,310 @@ +/* + * Copyright (c) 2025, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package publishers + +import ( + "encoding/json" + "fmt" + "log/slog" + "maps" + "os" + "strings" + "sync" + + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/analytics/dto" + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/config" +) + +// maskedHeaderValue is the placeholder written in place of a masked header value. +const maskedHeaderValue = "****" + +// Log is an analytics publisher that writes each enriched analytics event to +// stdout as a single JSON line. It is intended for log-scraping pipelines +// (Fluent Bit, Loki, ELK, etc.) and as a lightweight alternative to a SaaS +// analytics backend. The event already carries the rich metadata, headers and +// (when send_request_body/send_response_body are enabled) payloads attached by +// the analytics engine, so this publisher only serializes it. +type Log struct { + // maskedHeaders holds lower-cased header names whose values are redacted in + // the requestHeaders/responseHeaders properties before logging. + maskedHeaders map[string]bool + // ignoredPathPrefixes: skip emitting a line when the event's original request + // path starts with any of these (per-consumer — other publishers are + // unaffected). Trimmed of blanks at construction. + ignoredPathPrefixes []string + // maxPayloadSize caps the number of request/response payload bytes written to + // the log line (0 = no limit). Truncation is output-side only. + maxPayloadSize int + // mu serializes writes to stdout so concurrent ALS streams do not interleave. + mu sync.Mutex + // out is the destination writer; defaults to os.Stdout (overridable in tests). + out *os.File +} + +// NewLog creates a new stdout traffic-logging publisher. +func NewLog(logCfg *config.TrafficLoggingConfig) *Log { + if logCfg == nil { + logCfg = &config.TrafficLoggingConfig{} + } + + masked := make(map[string]bool, len(logCfg.MaskedHeaders)) + for _, h := range logCfg.MaskedHeaders { + h = strings.ToLower(strings.TrimSpace(h)) + if h != "" { + masked[h] = true + } + } + + prefixes := make([]string, 0, len(logCfg.IgnoredPathPrefixes)) + for _, p := range logCfg.IgnoredPathPrefixes { + if p = strings.TrimSpace(p); p != "" { + prefixes = append(prefixes, p) + } + } + + return &Log{ + maskedHeaders: masked, + ignoredPathPrefixes: prefixes, + maxPayloadSize: logCfg.MaxPayloadSize, + out: os.Stdout, + } +} + +// Publish writes the event to stdout as JSON. Traffic logging is per-API: only +// events carrying a traffic-log directive (stamped by the log-message policy in +// access-log mode on APIs that opted in) are emitted; all others are skipped. +func (l *Log) Publish(event *dto.Event) { + if event == nil || event.TrafficLog == nil { + return + } + if l.isPathIgnored(event) { + return + } + + out := l.shapeEvent(event) + + data, err := json.Marshal(out) + if err != nil { + slog.Error("Failed to marshal analytics event for log publisher", "error", err) + return + } + + // When an explicit field selection is configured it is authoritative over which + // fields appear: project the serialized record down to (or removing) the named + // top-level keys and properties.* paths. + if fields := event.TrafficLog.Fields; fields != nil && len(fields.Names) > 0 { + if projected, perr := applyFieldsProjection(data, fields); perr != nil { + slog.Error("Failed to project traffic-log fields; emitting unprojected line", "error", perr) + } else { + data = projected + } + } + + l.mu.Lock() + defer l.mu.Unlock() + if _, err := fmt.Fprintln(l.out, string(data)); err != nil { + slog.Error("Failed to write analytics event to stdout", "error", err) + } +} + +// shapeEvent returns the event to serialize, applying the per-API traffic-log +// directive to a shallow copy with a cloned Properties map so the shared event +// observed by other publishers is left untouched. For each flow it drops the +// headers/payload the API did not request, removes per-flow excluded headers, and +// redacts globally masked headers. ALS-derived fields (latencies, status, timing) +// are always retained. +func (l *Log) shapeEvent(event *dto.Event) *dto.Event { + if event.Properties == nil { + return event + } + + props := make(map[string]interface{}, len(event.Properties)) + maps.Copy(props, event.Properties) + + dir := event.TrafficLog + // When an explicit field selection is set it is authoritative over presence, so + // the per-flow headers/payload booleans are not used for gating here; only header + // masking + per-flow excludeHeaders are applied, and the projection (in Publish) + // decides which fields survive. + gate := dir.Fields == nil || len(dir.Fields.Names) == 0 + l.applyFlow(props, dir.Request, "requestHeaders", "request_payload", gate) + l.applyFlow(props, dir.Response, "responseHeaders", "response_payload", gate) + + cp := *event + cp.Properties = props + return &cp +} + +// applyFlow enforces one flow's presentation rules on the cloned properties. +// Header content, when present, is always cleaned (per-flow excludeHeaders + global +// masking). When gate is true (no authoritative field selection), a nil flow or a +// disabled headers/payload field also drops the corresponding property entirely. +func (l *Log) applyFlow(props map[string]interface{}, flow *dto.TrafficLogFlow, headersKey, payloadKey string, gate bool) { + var exclude []string + if flow != nil { + exclude = flow.ExcludeHeaders + } + if raw, ok := props[headersKey].(string); ok { + props[headersKey] = l.filterHeaders(raw, exclude) + } + if raw, ok := props[payloadKey].(string); ok { + props[payloadKey] = l.truncatePayload(raw) + } + + if gate { + if flow == nil || !flow.Headers { + delete(props, headersKey) + } + if flow == nil || !flow.Payload { + delete(props, payloadKey) + } + } +} + +// isPathIgnored reports whether the event's original request path starts with any +// configured ignored prefix. It reads Operation.APIResourceTemplate, which carries +// the original (pre-rewrite) path. +func (l *Log) isPathIgnored(event *dto.Event) bool { + if len(l.ignoredPathPrefixes) == 0 || event.Operation == nil { + return false + } + path := event.Operation.APIResourceTemplate + if path == "" { + return false + } + for _, prefix := range l.ignoredPathPrefixes { + if strings.HasPrefix(path, prefix) { + return true + } + } + return false +} + +// truncatePayload returns up to maxPayloadSize bytes of the payload (0 = no +// limit). Truncation is on a byte boundary, matching the previous capture-time +// behavior. +func (l *Log) truncatePayload(s string) string { + if l.maxPayloadSize <= 0 || len(s) <= l.maxPayloadSize { + return s + } + return s[:l.maxPayloadSize] +} + +// applyFieldsProjection restricts the serialized event JSON to the configured +// fields. Names are top-level keys (e.g. "latencies") or dotted property paths +// (e.g. "properties.requestHeaders"). Mode "exclude" drops the named keys; any +// other value (default "include") keeps only the named keys. Naming the whole +// "properties" key keeps all of its subkeys. +func applyFieldsProjection(data []byte, fields *dto.TrafficLogFields) ([]byte, error) { + var m map[string]interface{} + if err := json.Unmarshal(data, &m); err != nil { + return nil, err + } + + topNames := make(map[string]bool) + propNames := make(map[string]bool) + propsReferenced := false + for _, name := range fields.Names { + if sub, ok := strings.CutPrefix(name, "properties."); ok { + if sub != "" { + propNames[sub] = true + propsReferenced = true + } + continue + } + if name == "properties" { + propsReferenced = true + } + if name != "" { + topNames[name] = true + } + } + + props, _ := m["properties"].(map[string]interface{}) + + if strings.EqualFold(fields.Mode, "exclude") { + for name := range topNames { + delete(m, name) + } + for sub := range propNames { + delete(props, sub) + } + if props != nil && len(props) == 0 { + delete(m, "properties") + } + return json.Marshal(m) + } + + // include (default): keep only referenced top-level keys. + for key := range m { + keep := topNames[key] || (key == "properties" && propsReferenced) + if !keep { + delete(m, key) + } + } + // When specific property subkeys were named (and not the whole "properties"), + // keep only those subkeys. + if props != nil && len(propNames) > 0 && !topNames["properties"] { + for sub := range props { + if !propNames[sub] { + delete(props, sub) + } + } + if len(props) == 0 { + delete(m, "properties") + } + } + return json.Marshal(m) +} + +// filterHeaders parses a JSON header map, drops any per-flow excluded headers, +// and redacts the values of globally masked headers (both case-insensitive). The +// raw string is returned unchanged if it is empty or not valid JSON. +func (l *Log) filterHeaders(raw string, excludeHeaders []string) string { + if raw == "" { + return raw + } + var headers map[string]interface{} + if err := json.Unmarshal([]byte(raw), &headers); err != nil { + return raw + } + + excluded := make(map[string]bool, len(excludeHeaders)) + for _, h := range excludeHeaders { + if h = strings.ToLower(strings.TrimSpace(h)); h != "" { + excluded[h] = true + } + } + + for name := range headers { + lower := strings.ToLower(name) + if excluded[lower] { + delete(headers, name) + continue + } + if l.maskedHeaders[lower] { + headers[name] = maskedHeaderValue + } + } + + out, err := json.Marshal(headers) + if err != nil { + return raw + } + return string(out) +} diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go new file mode 100644 index 0000000000..641d532aa0 --- /dev/null +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go @@ -0,0 +1,334 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package publishers + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/analytics/dto" + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/config" +) + +// newLogToFile builds a Log publisher that writes to a temp file, returning the +// publisher and a function that reads back what was written. +func newLogToFile(t *testing.T, cfg *config.TrafficLoggingConfig) (*Log, func() string) { + t.Helper() + path := filepath.Join(t.TempDir(), "out.log") + f, err := os.Create(path) + require.NoError(t, err) + t.Cleanup(func() { _ = f.Close() }) + + l := NewLog(cfg) + l.out = f + return l, func() string { + require.NoError(t, f.Sync()) + data, err := os.ReadFile(path) + require.NoError(t, err) + return string(data) + } +} + +// bothFlows returns a directive that opts in to logging both request and response +// headers and payloads — the "log everything captured" case. +func bothFlows() *dto.TrafficLogDirective { + return &dto.TrafficLogDirective{ + Request: &dto.TrafficLogFlow{Payload: true, Headers: true}, + Response: &dto.TrafficLogFlow{Payload: true, Headers: true}, + } +} + +// decodeProps runs the single JSON line and returns the decoded event + its properties. +func decodeLine(t *testing.T, out string) (map[string]interface{}, map[string]interface{}) { + t.Helper() + var decoded map[string]interface{} + require.NoError(t, json.Unmarshal([]byte(out), &decoded)) + props, _ := decoded["properties"].(map[string]interface{}) + return decoded, props +} + +func TestNewLog_NilConfig(t *testing.T) { + l := NewLog(nil) + require.NotNil(t, l) + assert.Empty(t, l.maskedHeaders) +} + +// Per-API gating: an event without a traffic-log directive is never emitted. +func TestLog_Publish_SkipsWhenNoDirective(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() // no TrafficLog set + event.Properties["requestHeaders"] = `{"x-foo":"bar"}` + + l.Publish(event) + + assert.Empty(t, read(), "event without a traffic-log directive must not be logged") +} + +func TestLog_Publish_WritesJSONLineWithLatencies(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.TrafficLog = bothFlows() + event.Properties["requestHeaders"] = `{"x-foo":"bar"}` + + l.Publish(event) + + out := read() + // Single line (one trailing newline). + assert.Equal(t, 1, strings.Count(strings.TrimRight(out, "\n"), "\n")+1) + + decoded, props := decodeLine(t, out) + api := decoded["api"].(map[string]interface{}) + assert.Equal(t, "test-api", api["apiName"]) + assert.Equal(t, `{"x-foo":"bar"}`, props["requestHeaders"]) + + // ALS-derived latencies are always present in the line — the key improvement + // over the inline log-message policy, which could never see them. + latencies := decoded["latencies"].(map[string]interface{}) + assert.Equal(t, float64(100), latencies["responseLatency"]) +} + +func TestLog_Publish_MasksHeaders(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{MaskedHeaders: []string{"Authorization"}}) + event := createBaseEvent() + event.TrafficLog = bothFlows() + event.Properties["requestHeaders"] = `{"Authorization":"Bearer secret","x-foo":"bar"}` + event.Properties["responseHeaders"] = `{"authorization":"Bearer secret2"}` + + l.Publish(event) + + _, props := decodeLine(t, read()) + + var reqH map[string]interface{} + require.NoError(t, json.Unmarshal([]byte(props["requestHeaders"].(string)), &reqH)) + assert.Equal(t, "****", reqH["Authorization"]) // masked + assert.Equal(t, "bar", reqH["x-foo"]) // untouched + + var resH map[string]interface{} + require.NoError(t, json.Unmarshal([]byte(props["responseHeaders"].(string)), &resH)) + assert.Equal(t, "****", resH["authorization"]) // case-insensitive match +} + +// Per-API excludeHeaders drops the header entirely (vs global masking which redacts). +func TestLog_Publish_ExcludeHeadersDrops(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{MaskedHeaders: []string{"authorization"}}) + event := createBaseEvent() + event.TrafficLog = &dto.TrafficLogDirective{ + Request: &dto.TrafficLogFlow{Headers: true, ExcludeHeaders: []string{"X-Secret"}}, + } + event.Properties["requestHeaders"] = `{"Authorization":"Bearer s","X-Secret":"top","x-foo":"bar"}` + + l.Publish(event) + + _, props := decodeLine(t, read()) + var reqH map[string]interface{} + require.NoError(t, json.Unmarshal([]byte(props["requestHeaders"].(string)), &reqH)) + + _, hasSecret := reqH["X-Secret"] + assert.False(t, hasSecret, "excluded header must be dropped") + assert.Equal(t, "****", reqH["Authorization"], "masked header still redacted") + assert.Equal(t, "bar", reqH["x-foo"]) +} + +// headers:false omits the headers property; a nil flow omits its whole side. +func TestLog_Publish_DisabledFieldsOmitted(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.TrafficLog = &dto.TrafficLogDirective{ + Request: &dto.TrafficLogFlow{Headers: false, Payload: true}, + // Response flow nil -> both response props dropped. + } + event.Properties["requestHeaders"] = `{"x-foo":"bar"}` + event.Properties["request_payload"] = "req-body" + event.Properties["responseHeaders"] = `{"x-bar":"baz"}` + event.Properties["response_payload"] = "resp-body" + + l.Publish(event) + + _, props := decodeLine(t, read()) + _, hasReqHeaders := props["requestHeaders"] + assert.False(t, hasReqHeaders, "request headers disabled -> omitted") + assert.Equal(t, "req-body", props["request_payload"], "request payload enabled -> kept") + + _, hasRespHeaders := props["responseHeaders"] + _, hasRespPayload := props["response_payload"] + assert.False(t, hasRespHeaders, "nil response flow -> response headers omitted") + assert.False(t, hasRespPayload, "nil response flow -> response payload omitted") +} + +func TestLog_Publish_DoesNotMutateSharedEvent(t *testing.T) { + l, _ := newLogToFile(t, &config.TrafficLoggingConfig{MaskedHeaders: []string{"authorization"}}) + event := createBaseEvent() + event.TrafficLog = bothFlows() + original := `{"authorization":"Bearer secret"}` + event.Properties["requestHeaders"] = original + + l.Publish(event) + + // The shared event (read by other publishers) must be untouched. + assert.Equal(t, original, event.Properties["requestHeaders"]) +} + +func TestLog_Publish_NilEvent(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + assert.NotPanics(t, func() { l.Publish(nil) }) + assert.Empty(t, read()) +} + +// Field selection (include): only named top-level keys and properties.* survive, +// and it is authoritative over presence (request.headers boolean is ignored, but +// excludeHeaders + masking still apply). +func TestLog_Publish_FieldsInclude(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{MaskedHeaders: []string{"authorization"}}) + event := createBaseEvent() + event.Properties["requestHeaders"] = `{"Authorization":"Bearer s","X-Drop":"d","X-Keep":"k"}` + event.Properties["responseHeaders"] = `{"x-bar":"baz"}` + event.TrafficLog = &dto.TrafficLogDirective{ + Request: &dto.TrafficLogFlow{Headers: false, ExcludeHeaders: []string{"X-Drop"}}, // boolean ignored + Fields: &dto.TrafficLogFields{Mode: "include", Names: []string{"latencies", "properties.requestHeaders"}}, + } + + l.Publish(event) + decoded, props := decodeLine(t, read()) + + _, hasAPI := decoded["api"] + _, hasOp := decoded["operation"] + assert.False(t, hasAPI, "api not listed -> dropped") + assert.False(t, hasOp, "operation not listed -> dropped") + assert.Contains(t, decoded, "latencies", "latencies listed -> kept") + require.NotNil(t, props, "properties kept (a properties.* path was listed)") + + _, hasResp := props["responseHeaders"] + assert.False(t, hasResp, "responseHeaders not listed -> dropped") + + reqRaw, ok := props["requestHeaders"].(string) + require.True(t, ok, "requestHeaders present (fields authoritative, boolean ignored)") + var reqH map[string]interface{} + require.NoError(t, json.Unmarshal([]byte(reqRaw), &reqH)) + assert.Equal(t, "****", reqH["Authorization"], "masking still applies") + _, hasDrop := reqH["X-Drop"] + assert.False(t, hasDrop, "excludeHeaders still applies") + assert.Equal(t, "k", reqH["X-Keep"]) +} + +// Field selection (exclude): named keys are dropped, everything else remains. +func TestLog_Publish_FieldsExclude(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.Properties["requestHeaders"] = `{"x-foo":"bar"}` + event.Properties["request_payload"] = "secret-body" + event.TrafficLog = &dto.TrafficLogDirective{ + Fields: &dto.TrafficLogFields{Mode: "exclude", Names: []string{"operation", "properties.request_payload"}}, + } + + l.Publish(event) + decoded, props := decodeLine(t, read()) + + _, hasOp := decoded["operation"] + assert.False(t, hasOp, "operation excluded") + assert.Contains(t, decoded, "api", "api kept") + assert.Contains(t, decoded, "latencies", "latencies kept") + require.NotNil(t, props) + _, hasPayload := props["request_payload"] + assert.False(t, hasPayload, "request_payload excluded") + assert.Contains(t, props, "requestHeaders", "requestHeaders kept (not excluded)") +} + +// Naming the whole "properties" key keeps all of its subkeys. +func TestLog_Publish_FieldsIncludeWholeProperties(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.Properties["requestHeaders"] = `{"x-foo":"bar"}` + event.Properties["responseHeaders"] = `{"x-bar":"baz"}` + event.TrafficLog = &dto.TrafficLogDirective{ + Fields: &dto.TrafficLogFields{Mode: "include", Names: []string{"properties"}}, + } + + l.Publish(event) + decoded, props := decodeLine(t, read()) + + _, hasAPI := decoded["api"] + assert.False(t, hasAPI, "api not listed -> dropped") + require.NotNil(t, props) + assert.Contains(t, props, "requestHeaders") + assert.Contains(t, props, "responseHeaders") +} + +// Per-consumer path ignore: the publisher skips ignored prefixes (matched on the +// event's original path in Operation.APIResourceTemplate); other consumers are +// unaffected. +func TestLog_Publish_IgnoredPathSkipped(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{IgnoredPathPrefixes: []string{"/health", "/ready"}}) + event := createBaseEvent() + event.TrafficLog = bothFlows() + event.Operation.APIResourceTemplate = "/health/live" + + l.Publish(event) + assert.Empty(t, read(), "ignored path must not be logged") +} + +func TestLog_Publish_NonIgnoredPathLogged(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{IgnoredPathPrefixes: []string{"/health"}}) + event := createBaseEvent() + event.TrafficLog = bothFlows() + event.Operation.APIResourceTemplate = "/api/v1/orders" + + l.Publish(event) + assert.NotEmpty(t, read(), "non-ignored path must be logged") +} + +// Output-side payload truncation (0 = no limit). +func TestLog_Publish_TruncatesPayload(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{MaxPayloadSize: 5}) + event := createBaseEvent() + event.TrafficLog = bothFlows() + event.Properties["request_payload"] = "hello world" + event.Properties["response_payload"] = "goodbye world" + + l.Publish(event) + _, props := decodeLine(t, read()) + assert.Equal(t, "hello", props["request_payload"]) + assert.Equal(t, "goodb", props["response_payload"]) +} + +func TestLog_Publish_NoTruncationWhenZero(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{MaxPayloadSize: 0}) + event := createBaseEvent() + event.TrafficLog = bothFlows() + event.Properties["request_payload"] = "hello world" + + l.Publish(event) + _, props := decodeLine(t, read()) + assert.Equal(t, "hello world", props["request_payload"]) +} + +func TestLog_Publish_InvalidHeaderJSONLeftAsIs(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{MaskedHeaders: []string{"authorization"}}) + event := createBaseEvent() + event.TrafficLog = bothFlows() + event.Properties["requestHeaders"] = "not-json" + + l.Publish(event) + + _, props := decodeLine(t, read()) + assert.Equal(t, "not-json", props["requestHeaders"]) +} diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config.go b/gateway/gateway-runtime/policy-engine/internal/config/config.go index 5dfdc688ed..00fe960056 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config.go @@ -42,24 +42,46 @@ type Config struct { PolicyEngine PolicyEngine `koanf:"policy_engine"` GatewayController map[string]interface{} `koanf:"gateway_controller"` PolicyConfigurations map[string]interface{} `koanf:"policy_configurations"` + Collector CollectorConfig `koanf:"collector"` Analytics AnalyticsConfig `koanf:"analytics"` + TrafficLogging TrafficLoggingConfig `koanf:"traffic_logging"` TracingConfig TracingConfig `koanf:"tracing"` } +// CollectorConfig holds the data-collection ("collector") configuration. The +// collector is the shared capture pipeline that gathers request/response headers +// and bodies and ships them to the policy-engine over ALS. It is a prerequisite +// for every consumer of that data — analytics and traffic logging both require +// collector.enabled to be true. +type CollectorConfig struct { + // Enabled turns the collector on. When false, the ALS server is not started + // and no events are produced for any consumer. + Enabled bool `koanf:"enabled"` + // SendRequestBody / SendResponseBody attach captured request/response bodies + // onto the collected event. + SendRequestBody bool `koanf:"send_request_body"` + SendResponseBody bool `koanf:"send_response_body"` + // AccessLogsServiceCfg tunes the policy-engine ALS receiver (the gRPC server + // that ingests collected access logs). It is part of the collector transport + // and is configured under the shared [collector.als] section (the controller + // reads the same section to configure Envoy's sender side). + AccessLogsServiceCfg AccessLogsServiceConfig `koanf:"als"` +} + // AnalyticsConfig holds analytics configuration type AnalyticsConfig struct { - Enabled bool `koanf:"enabled"` - EnabledPublishers []string `koanf:"enabled_publishers"` - Publishers AnalyticsPublishersConfig `koanf:"publishers"` - GRPCEventServerCfg map[string]interface{} `koanf:"grpc_event_server"` - AccessLogsServiceCfg AccessLogsServiceConfig `koanf:"access_logs_service"` - // AllowPayloads controls whether request and response bodies are captured - // into analytics metadata and forwarded to analytics publishers. - // Deprecated: use SendRequestBody and SendResponseBody instead. - // When true, validateAnalyticsConfig maps both SendRequestBody and SendResponseBody - // to true if both are false. Because bools cannot represent "unset", this also - // applies when both new flags are explicitly false; remove allow_payloads when - // migrating and set the directional flags directly. + Enabled bool `koanf:"enabled"` + EnabledPublishers []string `koanf:"enabled_publishers"` + Publishers AnalyticsPublishersConfig `koanf:"publishers"` + GRPCEventServerCfg map[string]interface{} `koanf:"grpc_event_server"` + // AccessLogsServiceCfg is a deprecated alias. ALS receiver tuning moved to + // [collector.als]; when set here it is migrated onto the collector during + // validation (with a warning). Prefer [collector.als]. + AccessLogsServiceCfg AccessLogsServiceConfig `koanf:"access_logs_service"` + // AllowPayloads, SendRequestBody and SendResponseBody are deprecated aliases. + // Body capture now lives under [collector]. When set, these are mapped onto + // collector.send_request_body / collector.send_response_body during validation + // (with a warning). Prefer the [collector] fields directly. AllowPayloads bool `koanf:"allow_payloads"` SendRequestBody bool `koanf:"send_request_body"` SendResponseBody bool `koanf:"send_response_body"` @@ -70,6 +92,27 @@ type AnalyticsPublishersConfig struct { Moesif MoesifPublisherConfig `koanf:"moesif"` } +// TrafficLoggingConfig holds configuration for the stdout traffic-logging feature, +// which writes each collected event to stdout as a JSON line. It is a consumer of +// the collector and requires collector.enabled to be true. +type TrafficLoggingConfig struct { + // Enabled turns stdout JSON traffic logging on. + Enabled bool `koanf:"enabled"` + // MaskedHeaders lists header names (case-insensitive) whose values are + // redacted in the logged requestHeaders/responseHeaders. + MaskedHeaders []string `koanf:"masked_headers"` + // MaxPayloadSize caps the number of bytes of request/response payload written + // to the log line (0 = no limit). Truncation is applied at the publisher, so + // the collector still captures the full body and other consumers (e.g. Moesif) + // are unaffected. + MaxPayloadSize int `koanf:"max_payload_size"` + // IgnoredPathPrefixes suppresses stdout traffic-log lines for requests whose + // original path starts with any of these prefixes (e.g. health/readiness + // probes). Only the traffic-logging publisher is affected; other consumers + // still receive the event. + IgnoredPathPrefixes []string `koanf:"ignored_path_prefixes"` +} + // MoesifPublisherConfig holds Moesif-specific configuration type MoesifPublisherConfig struct { ApplicationID string `koanf:"application_id"` @@ -304,6 +347,22 @@ func Load(configPath string) (*Config, error) { return cfg, nil } +// defaultAccessLogsServiceConfig returns the default policy-engine ALS receiver tuning. +// Shared by the collector (canonical) and the deprecated [analytics].access_logs_service +// alias so a partial alias override migrates cleanly. +func defaultAccessLogsServiceConfig() AccessLogsServiceConfig { + return AccessLogsServiceConfig{ + Mode: "", + ServerPort: 18090, + ShutdownTimeout: 600 * time.Second, + PublicKeyPath: "", + PrivateKeyPath: "", + ALSPlainText: true, + ExtProcMaxMessageSize: 1000000000, + ExtProcMaxHeaderLimit: 8192, + } +} + // defaultConfig returns a Config struct with default configuration values func defaultConfig() *Config { return &Config{ @@ -350,6 +409,18 @@ func defaultConfig() *Config { }, TracingServiceName: "policy-engine", }, + Collector: CollectorConfig{ + Enabled: false, + SendRequestBody: false, + SendResponseBody: false, + AccessLogsServiceCfg: defaultAccessLogsServiceConfig(), + }, + TrafficLogging: TrafficLoggingConfig{ + Enabled: false, + MaskedHeaders: []string{}, + MaxPayloadSize: 0, + IgnoredPathPrefixes: []string{}, + }, Analytics: AnalyticsConfig{ Enabled: false, EnabledPublishers: []string{"moesif"}, @@ -369,19 +440,12 @@ func defaultConfig() *Config { "buffer_size_bytes": 16384, "grpc_request_timeout": 20000000000, }, - AccessLogsServiceCfg: AccessLogsServiceConfig{ - Mode: "", - ServerPort: 18090, - ShutdownTimeout: 600 * time.Second, - PublicKeyPath: "", - PrivateKeyPath: "", - ALSPlainText: true, - ExtProcMaxMessageSize: 1000000000, - ExtProcMaxHeaderLimit: 8192, - }, - AllowPayloads: false, - SendRequestBody: false, - SendResponseBody: false, + // Deprecated alias: default mirrors the collector so a partial + // [analytics.access_logs_service] override migrates cleanly. + AccessLogsServiceCfg: defaultAccessLogsServiceConfig(), + AllowPayloads: false, + SendRequestBody: false, + SendResponseBody: false, }, TracingConfig: TracingConfig{ Enabled: false, @@ -480,6 +544,12 @@ func (c *Config) Validate() error { return fmt.Errorf("invalid logging.format: %s (must be json or text)", c.PolicyEngine.Logging.Format) } + if err := c.validateCollectorConfig(); err != nil { + return err + } + if c.TrafficLogging.MaxPayloadSize < 0 { + return fmt.Errorf("traffic_logging.max_payload_size must be >= 0, got %d", c.TrafficLogging.MaxPayloadSize) + } if c.Analytics.Enabled { if err := c.validateAnalyticsConfig(); err != nil { return fmt.Errorf("analytics configuration validation failed: %v", err) @@ -536,49 +606,98 @@ func (c *Config) validateXDSConfig() error { return nil } -// validateAnalyticsConfig validates the analytics configuration -func (c *Config) validateAnalyticsConfig() error { - // Validate analytics configuration - if c.Analytics.Enabled { - // Migration path for deprecated analytics.allow_payloads. - // Runs when both directional flags are false, which is indistinguishable - // from "not set" because bool fields cannot represent unset vs explicit false. - if c.Analytics.AllowPayloads { - slog.Warn("analytics.allow_payloads is deprecated; use analytics.send_request_body and analytics.send_response_body instead") - if !c.Analytics.SendRequestBody && !c.Analytics.SendResponseBody { - c.Analytics.SendRequestBody = true - c.Analytics.SendResponseBody = true - } +// validateCollectorConfig migrates deprecated analytics capture aliases onto the +// collector and enforces the collector prerequisite: a consumer (analytics or +// traffic logging) requires the collector that feeds it. For backward compatibility +// this is a soft prerequisite — if a consumer is enabled without the collector, the +// collector is auto-enabled with a warning rather than failing. +func (c *Config) validateCollectorConfig() error { + c.migrateDeprecatedAnalyticsCapture() + c.migrateDeprecatedAnalyticsTransport() + + // Backward-compat bridge: a consumer cannot run without the collector that feeds + // it, but rather than fail an existing config that only set analytics.enabled + // (valid before the collector split), auto-enable the collector with a warning. + if (c.Analytics.Enabled || c.TrafficLogging.Enabled) && !c.Collector.Enabled { + slog.Warn("a consumer (analytics.enabled or traffic_logging.enabled) requires the collector; enabling collector.enabled automatically for backward compatibility. Set collector.enabled = true explicitly to silence this warning.") + c.Collector.Enabled = true + } + if c.Collector.Enabled { + if err := validateAccessLogsServiceConfig(c.Collector.AccessLogsServiceCfg); err != nil { + return err } + } + return nil +} - // Validate ALS server config (policy-engine side) - als := c.Analytics.AccessLogsServiceCfg - - // Validate ALS connection mode - switch als.Mode { - case "uds", "": - // UDS mode (default) - port is unused - case "tcp": - // TCP mode - validate port - if als.ServerPort <= 0 || als.ServerPort > 65535 { - return fmt.Errorf("analytics.access_logs_service.server_port must be between 1 and 65535, got %d", als.ServerPort) - } - default: - return fmt.Errorf("analytics.access_logs_service.mode must be 'uds' or 'tcp', got: %s", als.Mode) - } - if als.ShutdownTimeout <= 0 { - return fmt.Errorf("analytics.access_logs_service.shutdown_timeout must be positive, got %s", als.ShutdownTimeout) - } - if als.ExtProcMaxMessageSize <= 0 { - return fmt.Errorf("analytics.access_logs_service.max_message_size must be positive, got %d", als.ExtProcMaxMessageSize) +// migrateDeprecatedAnalyticsTransport maps a deprecated [analytics].access_logs_service +// override onto the collector when the collector's receiver tuning is still at its +// default, so existing configs keep working after the transport moved to [collector]. +func (c *Config) migrateDeprecatedAnalyticsTransport() { + def := defaultAccessLogsServiceConfig() + if c.Analytics.AccessLogsServiceCfg != def { + slog.Warn("analytics.access_logs_service is deprecated; use collector.als instead") + if c.Collector.AccessLogsServiceCfg == def { + c.Collector.AccessLogsServiceCfg = c.Analytics.AccessLogsServiceCfg } - if als.ExtProcMaxHeaderLimit <= 0 { - return fmt.Errorf("analytics.access_logs_service.max_header_limit must be positive, got %d", als.ExtProcMaxHeaderLimit) + } +} + +// validateAccessLogsServiceConfig validates the policy-engine ALS receiver tuning. +func validateAccessLogsServiceConfig(als AccessLogsServiceConfig) error { + switch als.Mode { + case "uds", "": + // UDS mode (default) - port is unused + case "tcp": + if als.ServerPort <= 0 || als.ServerPort > 65535 { + return fmt.Errorf("collector.als.server_port must be between 1 and 65535, got %d", als.ServerPort) } - if als.ExtProcMaxHeaderLimit > math.MaxUint32 { - return fmt.Errorf("analytics.access_logs_service.max_header_limit must be <= %d, got %d", uint64(math.MaxUint32), als.ExtProcMaxHeaderLimit) + default: + return fmt.Errorf("collector.als.mode must be 'uds' or 'tcp', got: %s", als.Mode) + } + if als.ShutdownTimeout <= 0 { + return fmt.Errorf("collector.als.shutdown_timeout must be positive, got %s", als.ShutdownTimeout) + } + if als.ExtProcMaxMessageSize <= 0 { + return fmt.Errorf("collector.als.max_message_size must be positive, got %d", als.ExtProcMaxMessageSize) + } + if als.ExtProcMaxHeaderLimit <= 0 { + return fmt.Errorf("collector.als.max_header_limit must be positive, got %d", als.ExtProcMaxHeaderLimit) + } + if als.ExtProcMaxHeaderLimit > math.MaxUint32 { + return fmt.Errorf("collector.als.max_header_limit must be <= %d, got %d", uint64(math.MaxUint32), als.ExtProcMaxHeaderLimit) + } + return nil +} + +// migrateDeprecatedAnalyticsCapture maps the deprecated analytics.allow_payloads / +// analytics.send_request_body / analytics.send_response_body onto the collector's +// body-capture flags (when the collector flag is not already set), so existing +// configs keep working after capture settings moved under [collector]. +func (c *Config) migrateDeprecatedAnalyticsCapture() { + // Directional aliases take precedence over allow_payloads. + if c.Analytics.SendRequestBody && !c.Collector.SendRequestBody { + slog.Warn("analytics.send_request_body is deprecated; use collector.send_request_body instead") + c.Collector.SendRequestBody = true + } + if c.Analytics.SendResponseBody && !c.Collector.SendResponseBody { + slog.Warn("analytics.send_response_body is deprecated; use collector.send_response_body instead") + c.Collector.SendResponseBody = true + } + // allow_payloads only fills in when no directional body capture is configured. + if c.Analytics.AllowPayloads { + slog.Warn("analytics.allow_payloads is deprecated; use collector.send_request_body and collector.send_response_body instead") + if !c.Collector.SendRequestBody && !c.Collector.SendResponseBody { + c.Collector.SendRequestBody = true + c.Collector.SendResponseBody = true } + } +} +// validateAnalyticsConfig validates the analytics consumer configuration (publishers). +// ALS transport validation lives in validateCollectorConfig. +func (c *Config) validateAnalyticsConfig() error { + if c.Analytics.Enabled { // Validate enabled publishers for _, publisherName := range c.Analytics.EnabledPublishers { switch publisherName { @@ -595,6 +714,8 @@ func (c *Config) validateAnalyticsConfig() error { return fmt.Errorf("analytics.publishers.moesif.moesif_base_url must be a valid URL (e.g. https://api.moesif.net), got %q", moesifCfg.BaseURL) } } + case "log": + // The stdout/log publisher has no required configuration. default: return fmt.Errorf("unknown publisher type in enabled_publishers: %s", publisherName) } diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config_test.go b/gateway/gateway-runtime/policy-engine/internal/config/config_test.go index 3470b998c9..9be230088d 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config_test.go @@ -69,8 +69,17 @@ func validConfig() *Config { Timeout: 30 * time.Second, }, }, + // Collector enabled by default so tests that turn on a consumer + // (analytics / traffic logging) satisfy the collector prerequisite. + // ALS receiver defaults mirror production so transport validation passes + // and the deprecated alias stays neutral (no spurious migration). + Collector: CollectorConfig{ + Enabled: true, + AccessLogsServiceCfg: defaultAccessLogsServiceConfig(), + }, Analytics: AnalyticsConfig{ - Enabled: false, + Enabled: false, + AccessLogsServiceCfg: defaultAccessLogsServiceConfig(), }, TracingConfig: TracingConfig{ Enabled: false, @@ -948,7 +957,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - valid UDS config (default)", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ Mode: "uds", ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -961,7 +970,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - valid UDS config (empty mode defaults to uds)", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ Mode: "", ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -974,7 +983,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - valid TCP config", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ Mode: "tcp", ServerPort: 18090, ShutdownTimeout: 600 * time.Second, @@ -988,7 +997,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - invalid mode", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ Mode: "invalid", ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -1002,7 +1011,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - TCP mode invalid ALS port", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ Mode: "tcp", ServerPort: 0, ShutdownTimeout: 600 * time.Second, @@ -1017,7 +1026,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - UDS mode skips port validation", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ Mode: "uds", ServerPort: 0, // Invalid port, but irrelevant in UDS mode ShutdownTimeout: 600 * time.Second, @@ -1031,7 +1040,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - invalid shutdown timeout", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ ShutdownTimeout: 0, ExtProcMaxMessageSize: 1000000, ExtProcMaxHeaderLimit: 8192, @@ -1044,7 +1053,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - invalid max message size", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 0, ExtProcMaxHeaderLimit: 8192, @@ -1057,7 +1066,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - invalid max header limit", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, ExtProcMaxHeaderLimit: 0, @@ -1070,7 +1079,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - max header limit exceeds uint32", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, ExtProcMaxHeaderLimit: math.MaxInt, @@ -1100,7 +1109,8 @@ func TestValidate_AnalyticsConfig(t *testing.T) { func TestValidate_AnalyticsPayloadMigration(t *testing.T) { setValidAnalyticsALS := func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.Enabled = true // analytics is a consumer; the collector must be on + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ Mode: "uds", ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -1160,12 +1170,66 @@ func TestValidate_AnalyticsPayloadMigration(t *testing.T) { err := cfg.Validate() require.NoError(t, err) - assert.Equal(t, tt.wantSendReq, cfg.Analytics.SendRequestBody) - assert.Equal(t, tt.wantSendResp, cfg.Analytics.SendResponseBody) + // Deprecated analytics body aliases now migrate onto the collector. + assert.Equal(t, tt.wantSendReq, cfg.Collector.SendRequestBody) + assert.Equal(t, tt.wantSendResp, cfg.Collector.SendResponseBody) }) } } +// TestValidate_CollectorPrerequisite verifies that enabling a consumer (analytics, +// traffic logging) without the collector auto-enables the collector (a +// backward-compat soft prerequisite) rather than failing. +func TestValidate_CollectorPrerequisite(t *testing.T) { + validALS := AccessLogsServiceConfig{ + Mode: "uds", + ShutdownTimeout: 600 * time.Second, + ExtProcMaxMessageSize: 1000000, + ExtProcMaxHeaderLimit: 8192, + } + + t.Run("analytics enabled without collector auto-enables the collector", func(t *testing.T) { + cfg := validConfig() + cfg.Collector.Enabled = false + cfg.Analytics.Enabled = true + cfg.Collector.AccessLogsServiceCfg = validALS + cfg.Analytics.EnabledPublishers = []string{} + require.NoError(t, cfg.Validate()) + assert.True(t, cfg.Collector.Enabled, "collector should be auto-enabled for backward compatibility") + }) + + t.Run("traffic logging enabled without collector auto-enables the collector", func(t *testing.T) { + cfg := validConfig() + cfg.Collector.Enabled = false + cfg.TrafficLogging.Enabled = true + require.NoError(t, cfg.Validate()) + assert.True(t, cfg.Collector.Enabled, "collector should be auto-enabled for backward compatibility") + }) + + t.Run("traffic logging enabled with collector is valid", func(t *testing.T) { + cfg := validConfig() + cfg.Collector.Enabled = true + cfg.TrafficLogging.Enabled = true + require.NoError(t, cfg.Validate()) + }) + + t.Run("collector enabled with no consumers is valid", func(t *testing.T) { + cfg := validConfig() + cfg.Collector.Enabled = true + require.NoError(t, cfg.Validate()) + }) +} + +func TestValidate_TrafficLoggingMaxPayloadSize(t *testing.T) { + cfg := validConfig() + cfg.Collector.Enabled = true + cfg.TrafficLogging.Enabled = true + cfg.TrafficLogging.MaxPayloadSize = -1 + err := cfg.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "traffic_logging.max_payload_size") +} + // TestValidate_AnalyticsPublishers tests analytics publisher validation func TestValidate_AnalyticsPublishers(t *testing.T) { tests := []struct { @@ -1178,7 +1242,7 @@ func TestValidate_AnalyticsPublishers(t *testing.T) { name: "no publishers enabled", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ ServerPort: 18090, ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -1192,7 +1256,7 @@ func TestValidate_AnalyticsPublishers(t *testing.T) { name: "unknown publisher type", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ ServerPort: 18090, ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -1207,7 +1271,7 @@ func TestValidate_AnalyticsPublishers(t *testing.T) { name: "moesif publisher - missing application_id", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ ServerPort: 18090, ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -1223,7 +1287,7 @@ func TestValidate_AnalyticsPublishers(t *testing.T) { name: "moesif publisher - invalid publish_interval", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ ServerPort: 18090, ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -1240,7 +1304,7 @@ func TestValidate_AnalyticsPublishers(t *testing.T) { name: "moesif publisher - invalid base_url", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ ServerPort: 18090, ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -1258,7 +1322,7 @@ func TestValidate_AnalyticsPublishers(t *testing.T) { name: "moesif publisher - valid config", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ ServerPort: 18090, ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, diff --git a/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server.go b/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server.go index 87049d7d8a..b68babbe08 100644 --- a/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server.go +++ b/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server.go @@ -90,15 +90,15 @@ func StartAccessLogServiceServer(cfg *config.Config) *grpc.Server { Timeout: 20 * time.Second, } maxHeaderListSize, err := checkedUInt32FromPositiveInt( - "analytics.access_logs_service.max_header_limit", - cfg.Analytics.AccessLogsServiceCfg.ExtProcMaxHeaderLimit, + "collector.als.max_header_limit", + cfg.Collector.AccessLogsServiceCfg.ExtProcMaxHeaderLimit, ) if err != nil { panic(err) } - server, err := CreateGRPCServer(cfg.Analytics.AccessLogsServiceCfg.PublicKeyPath, - cfg.Analytics.AccessLogsServiceCfg.PrivateKeyPath, cfg.Analytics.AccessLogsServiceCfg.ALSPlainText, - grpc.MaxRecvMsgSize(cfg.Analytics.AccessLogsServiceCfg.ExtProcMaxMessageSize), + server, err := CreateGRPCServer(cfg.Collector.AccessLogsServiceCfg.PublicKeyPath, + cfg.Collector.AccessLogsServiceCfg.PrivateKeyPath, cfg.Collector.AccessLogsServiceCfg.ALSPlainText, + grpc.MaxRecvMsgSize(cfg.Collector.AccessLogsServiceCfg.ExtProcMaxMessageSize), grpc.MaxHeaderListSize(maxHeaderListSize), grpc.KeepaliveParams(kaParams)) if err != nil { @@ -109,7 +109,7 @@ func StartAccessLogServiceServer(cfg *config.Config) *grpc.Server { // Create listener based on mode (same pattern as ext_proc in main.go) var listener net.Listener - alsMode := cfg.Analytics.AccessLogsServiceCfg.Mode + alsMode := cfg.Collector.AccessLogsServiceCfg.Mode if alsMode == "" { alsMode = "uds" } @@ -139,14 +139,14 @@ func StartAccessLogServiceServer(cfg *config.Config) *grpc.Server { } }() case "tcp": - listener, err = net.Listen("tcp", fmt.Sprintf(":%d", cfg.Analytics.AccessLogsServiceCfg.ServerPort)) + listener, err = net.Listen("tcp", fmt.Sprintf(":%d", cfg.Collector.AccessLogsServiceCfg.ServerPort)) if err != nil { - slog.Error("Failed to listen on ALS TCP port", "port", cfg.Analytics.AccessLogsServiceCfg.ServerPort) + slog.Error("Failed to listen on ALS TCP port", "port", cfg.Collector.AccessLogsServiceCfg.ServerPort) panic(err) } go func() { - slog.Info("Starting to serve access log service server", "mode", "tcp", "port", cfg.Analytics.AccessLogsServiceCfg.ServerPort) + slog.Info("Starting to serve access log service server", "mode", "tcp", "port", cfg.Collector.AccessLogsServiceCfg.ServerPort) if err := server.Serve(listener); err != nil { slog.Error("ALS server exited", "error", err) } diff --git a/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server_test.go b/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server_test.go index fd6dba08d7..0178fc5356 100644 --- a/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server_test.go @@ -210,10 +210,8 @@ func TestStreamAccessLogs_MultipleMessages(t *testing.T) { func TestStartAccessLogServiceServer_TCP(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ - Enabled: false, - EnabledPublishers: []string{}, - Publishers: config.AnalyticsPublishersConfig{}, + Collector: config.CollectorConfig{ + Enabled: true, AccessLogsServiceCfg: config.AccessLogsServiceConfig{ Mode: "tcp", ServerPort: 19001, // Use non-standard port to avoid conflicts diff --git a/gateway/system-policies/analytics/analytics.go b/gateway/system-policies/analytics/analytics.go index 261cef76cd..7635f5268b 100644 --- a/gateway/system-policies/analytics/analytics.go +++ b/gateway/system-policies/analytics/analytics.go @@ -152,6 +152,14 @@ func (a *AnalyticsPolicy) OnRequestHeaders(_ context.Context, reqCtx *policy.Req } } + // Capture all request headers when enabled, so they flow into analytics events + // (and the stdout/log publisher) without attaching a per-API header policy. + if sendReqHeaders, _ := getHeaderFlags(params); sendReqHeaders && reqCtx.Headers != nil { + if headers := serializeHeaders(reqCtx.Headers); headers != "" { + analyticsMetadata["request_headers"] = headers + } + } + if len(analyticsMetadata) > 0 { return policy.UpstreamRequestHeaderModifications{AnalyticsMetadata: analyticsMetadata} } @@ -207,6 +215,13 @@ func (a *AnalyticsPolicy) OnResponseHeaders(_ context.Context, respCtx *policy.R } } + // Capture all response headers when enabled. + if _, sendRespHeaders := getHeaderFlags(params); sendRespHeaders && respCtx.ResponseHeaders != nil { + if headers := serializeHeaders(respCtx.ResponseHeaders); headers != "" { + analyticsMetadata["response_headers"] = headers + } + } + if len(analyticsMetadata) > 0 { return policy.DownstreamResponseHeaderModifications{AnalyticsMetadata: analyticsMetadata} } @@ -898,18 +913,6 @@ func getPayloadFlags(params map[string]interface{}) (sendRequestBody, sendRespon return false, false } - parseBoolLike := func(v interface{}) bool { - switch val := v.(type) { - case bool: - return val - case string: - lower := strings.ToLower(strings.TrimSpace(val)) - return lower == "true" || lower == "1" || lower == "yes" - default: - return false - } - } - hasReq, hasResp := false, false if raw, ok := params["send_request_body"]; ok { @@ -937,6 +940,55 @@ func getPayloadFlags(params map[string]interface{}) (sendRequestBody, sendRespon return false, false } +// parseBoolLike interprets bool and common string ("true"/"1"/"yes") representations +// of a boolean policy parameter. +func parseBoolLike(v interface{}) bool { + switch val := v.(type) { + case bool: + return val + case string: + lower := strings.ToLower(strings.TrimSpace(val)) + return lower == "true" || lower == "1" || lower == "yes" + default: + return false + } +} + +// getHeaderFlags derives per-direction header capture flags from policy parameters. +func getHeaderFlags(params map[string]interface{}) (sendRequestHeaders, sendResponseHeaders bool) { + if params == nil { + return false, false + } + if raw, ok := params["send_request_headers"]; ok { + sendRequestHeaders = parseBoolLike(raw) + } + if raw, ok := params["send_response_headers"]; ok { + sendResponseHeaders = parseBoolLike(raw) + } + return sendRequestHeaders, sendResponseHeaders +} + +// serializeHeaders renders all headers as a JSON object string ({"name":"v1, v2"}), +// matching the request_headers/response_headers format the analytics engine reads. +// Returns "" when there are no headers. Sensitive values are not masked here; the +// stdout/log publisher applies masked_headers on output. +func serializeHeaders(headers *policy.Headers) string { + all := headers.GetAll() + if len(all) == 0 { + return "" + } + flat := make(map[string]string, len(all)) + for name, values := range all { + flat[name] = strings.Join(values, ", ") + } + data, err := json.Marshal(flat) + if err != nil { + slog.Error("Failed to marshal headers for analytics", "error", err) + return "" + } + return string(data) +} + // Helper to extract string values via JSONPath func extractStringFromJsonpath(payload map[string]interface{}, path string) string { val, err := utils.ExtractValueFromJsonpath(payload, path) diff --git a/gateway/system-policies/analytics/analytics_headers_test.go b/gateway/system-policies/analytics/analytics_headers_test.go new file mode 100644 index 0000000000..aed92c9f6b --- /dev/null +++ b/gateway/system-policies/analytics/analytics_headers_test.go @@ -0,0 +1,63 @@ +package analytics + +import ( + "encoding/json" + "testing" + + policy "github.com/wso2/api-platform/sdk/core/policy/v1alpha2" +) + +func TestGetHeaderFlags(t *testing.T) { + cases := []struct { + name string + params map[string]interface{} + wantReq bool + wantResp bool + }{ + {"nil params", nil, false, false}, + {"absent", map[string]interface{}{}, false, false}, + {"bool true", map[string]interface{}{"send_request_headers": true, "send_response_headers": true}, true, true}, + {"string true", map[string]interface{}{"send_request_headers": "true"}, true, false}, + {"mixed", map[string]interface{}{"send_request_headers": false, "send_response_headers": "yes"}, false, true}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + gotReq, gotResp := getHeaderFlags(c.params) + if gotReq != c.wantReq || gotResp != c.wantResp { + t.Fatalf("getHeaderFlags(%v) = (%v, %v), want (%v, %v)", c.params, gotReq, gotResp, c.wantReq, c.wantResp) + } + }) + } +} + +func TestSerializeHeaders(t *testing.T) { + // Empty headers -> empty string. + if got := serializeHeaders(policy.NewHeaders(nil)); got != "" { + t.Fatalf("serializeHeaders(empty) = %q, want \"\"", got) + } + + h := policy.NewHeaders(map[string][]string{ + "Authorization": {"Bearer secret"}, + "X-Foo": {"a", "b"}, + }) + got := serializeHeaders(h) + if got == "" { + t.Fatal("serializeHeaders returned empty for non-empty headers") + } + + var decoded map[string]string + if err := json.Unmarshal([]byte(got), &decoded); err != nil { + t.Fatalf("output is not valid JSON: %v (%q)", err, got) + } + // NewHeaders lower-cases keys; multi-value headers are joined with ", ". + if decoded["authorization"] != "Bearer secret" { + t.Errorf("authorization = %q, want %q", decoded["authorization"], "Bearer secret") + } + if decoded["x-foo"] != "a, b" { + t.Errorf("x-foo = %q, want %q", decoded["x-foo"], "a, b") + } +} + +// Note: payload-size capping moved out of the capture path (the collector now +// captures full bodies); truncation is applied output-side by the traffic-logging +// publisher (traffic_logging.max_payload_size) and tested there. diff --git a/gateway/system-policies/analytics/policy-definition.yaml b/gateway/system-policies/analytics/policy-definition.yaml index 7fe915469d..eaed1b307e 100644 --- a/gateway/system-policies/analytics/policy-definition.yaml +++ b/gateway/system-policies/analytics/policy-definition.yaml @@ -25,6 +25,30 @@ parameters: into analytics metadata as response_payload, for both buffered and streaming responses, which is then forwarded through the analytics pipeline (including publishers like Moesif). + send_request_headers: + type: boolean + default: false + description: > + When true, the analytics system policy captures ALL request headers + into analytics metadata as request_headers, which is forwarded through + the analytics pipeline (including the stdout/log and Moesif publishers). + Sensitive values are not masked here; the log publisher's masked_headers + setting redacts them on output. + send_response_headers: + type: boolean + default: false + description: > + When true, the analytics system policy captures ALL response headers + into analytics metadata as response_headers, which is forwarded through + the analytics pipeline (including the stdout/log and Moesif publishers). + max_payload_size: + type: integer + default: 0 + description: > + Maximum number of bytes captured per request/response body when + send_request_body / send_response_body is enabled. 0 (default) means no + limit; a positive value truncates each body to that many bytes (silent, + byte-boundary truncation). allow_payloads: type: boolean default: false From 5179aeaf71bb15e4b42a04b58446409f84688856 Mon Sep 17 00:00:00 2001 From: Dineth Date: Sat, 4 Jul 2026 00:51:58 +0530 Subject: [PATCH 02/21] Refactor collector configuration to remove explicit enabled flag - Updated the collector configuration to remove the explicit `enabled` flag, making the collector implicitly active whenever a consumer (analytics or traffic logging) is enabled. - Adjusted related comments and documentation to reflect the new behavior. - Modified tests to ensure they validate the collector's state based on consumer configurations rather than an explicit flag. - Removed deprecated path ignoring functionality from traffic logging configuration. - Enhanced logging to include custom properties in traffic log events. --- gateway/configs/config-template.toml | 38 ++++----- gateway/configs/config.toml | 16 ++-- .../gateway-controller/pkg/config/config.go | 51 +++++++----- .../pkg/config/config_test.go | 35 ++++---- .../pkg/utils/system_policies.go | 8 +- .../pkg/utils/system_policies_test.go | 39 ++++----- .../gateway-controller/pkg/xds/translator.go | 8 +- .../policy-engine/cmd/policy-engine/main.go | 2 +- .../internal/analytics/analytics.go | 3 - .../internal/analytics/analytics_test.go | 4 +- .../internal/analytics/dto/event.go | 4 + .../internal/analytics/publishers/log.go | 53 +++--------- .../internal/analytics/publishers/log_test.go | 83 ++++++++++++++----- .../policy-engine/internal/config/config.go | 47 +++++------ .../internal/config/config_test.go | 52 ++++-------- .../utils/access_logger_server_test.go | 1 - 16 files changed, 208 insertions(+), 236 deletions(-) diff --git a/gateway/configs/config-template.toml b/gateway/configs/config-template.toml index a9c3965cce..4f3530ad12 100644 --- a/gateway/configs/config-template.toml +++ b/gateway/configs/config-template.toml @@ -285,11 +285,11 @@ host = "localhost" # ============================================================================= # The collector is the shared data-capture pipeline: it injects the analytics # system policy and ships request/response headers and bodies to the policy-engine -# over ALS. It is a PREREQUISITE for every consumer below — analytics and -# traffic_logging both require collector.enabled = true (enabling a consumer with -# the collector off is a startup error). +# over ALS. It has NO on/off switch of its own — it is implicit, turning on +# automatically whenever a consumer below is enabled ([analytics] or +# [traffic_logging]) and staying off otherwise. This section only tunes what the +# collector captures. [collector] -enabled = false # Capture request/response payloads (bodies) into the collected event. Bodies are # captured in full; per-consumer size caps are applied on output (see # traffic_logging.max_payload_size). @@ -298,9 +298,9 @@ send_response_body = false # Capture ALL request/response headers into the collected event (request_headers/ # response_headers). Applies to every API via the collector system policy, so no # per-API header policy is needed. Sensitive values are redacted on output by each -# consumer (e.g. traffic_logging.masked_headers). -send_request_headers = false -send_response_headers = false +# consumer (e.g. traffic_logging.masked_headers). On by default. +send_request_headers = true +send_response_headers = true # ALS transport tuning (advanced; defaults are sensible). One section read by BOTH # ends of the Envoy → policy-engine access-log stream: @@ -323,7 +323,7 @@ max_message_size = 1000000000 max_header_limit = 8192 # ============================================================================= -# ANALYTICS CONFIGURATION (consumer — requires [collector].enabled = true) +# ANALYTICS CONFIGURATION (consumer — enabling it activates the collector) # ============================================================================= [analytics] enabled = false @@ -347,7 +347,7 @@ batch_size = 50 timer_wakeup_seconds = 3 # ============================================================================= -# TRAFFIC LOGGING (consumer — requires [collector].enabled = true) +# TRAFFIC LOGGING (consumer — enabling it activates the collector) # ============================================================================= # Writes collected events (rich API/application/AI/latency metadata, plus # request/response headers and — when the collector's send_*_body is enabled — @@ -355,27 +355,23 @@ timer_wakeup_seconds = 3 # Fluent Bit/Loki/ELK; no external SaaS needed. # # Per-API (opt-in): a line is emitted only for APIs that attach the `log-message` -# policy with `destination = access-log` — enabling this section alone does NOT -# log every API. So an API is logged only when all three hold: [collector] -# .enabled = true, [traffic_logging].enabled = true, and the `log-message` policy -# (access-log destination) is attached to that API. The policy's request/response -# parameters (per-flow payload/headers toggles and excludeHeaders) shape each -# API's line; they filter/mask what the collector captured (they cannot enable -# capture the collector was not configured to perform). +# policy with `enableTrafficLogging: true` — enabling this section alone does NOT +# log every API. So an API is logged only when both hold: [traffic_logging].enabled +# = true, and the `log-message` policy (enableTrafficLogging: true) is attached to +# that API. The policy's request/response parameters (per-flow payload/headers +# toggles and excludeHeaders) shape each API's line; they filter/mask what the +# collector captured (they cannot enable capture the collector was not configured +# to perform). [traffic_logging] enabled = false # Header names (case-insensitive) whose values are redacted as "****" in the # logged requestHeaders/responseHeaders. Applies globally; the `log-message` -# policy's (access-log mode) per-API excludeHeaders drop additional headers entirely. +# policy's per-API excludeHeaders drop additional headers entirely. masked_headers = ["authorization"] # Max bytes of request/response payload written to each log line (0 = no limit). # Applied output-side, so the collector still captures full bodies and other # consumers (e.g. Moesif) are unaffected. max_payload_size = 0 -# Suppress traffic-log lines for requests whose original (pre-rewrite) path starts -# with any of these prefixes (e.g. health/readiness probes). Only affects this -# stdout publisher — other consumers still receive the event. -ignored_path_prefixes = [] # ============================================================================= # POLICY CONFIGURATIONS diff --git a/gateway/configs/config.toml b/gateway/configs/config.toml index 98238a3fc1..bc22d25487 100644 --- a/gateway/configs/config.toml +++ b/gateway/configs/config.toml @@ -1,17 +1,17 @@ -# The collector is the shared data-capture pipeline. It must be enabled before any -# consumer (analytics, traffic_logging) can receive data. +# The collector is the shared data-capture pipeline. It has no on/off switch of its +# own: it turns on automatically whenever a consumer (analytics or traffic_logging) +# is enabled, and stays off otherwise. This section only tunes what is captured. [collector] -enabled = true # Capture request/response payloads (bodies) into the collected event (full bodies; # per-consumer size caps are applied on output). -send_request_body = true -send_response_body = true +send_request_body = false +send_response_body = false # Capture ALL request/response headers (redacted on output by per-consumer masking). send_request_headers = true send_response_headers = true -# Analytics consumer (e.g. Moesif). Requires [collector].enabled = true. +# Analytics consumer (e.g. Moesif). Enabling it activates the collector automatically. [analytics] enabled = false enabled_publishers = ["moesif"] @@ -20,15 +20,13 @@ enabled_publishers = ["moesif"] # application_id = "" # Traffic-logging consumer: writes each collected event to stdout as a JSON line -# (no external service needed). Requires [collector].enabled = true. +# (no external service needed). Enabling it activates the collector automatically. [traffic_logging] enabled = true # Header names (case-insensitive) whose values are redacted as "****". masked_headers = ["authorization"] # Max bytes of request/response payload written to each log line (0 = no limit). max_payload_size = 2048 -# Skip traffic-log lines for these path prefixes (e.g. health probes). -ignored_path_prefixes = ["/health", "/ready"] [router] gateway_host = "*" diff --git a/gateway/gateway-controller/pkg/config/config.go b/gateway/gateway-controller/pkg/config/config.go index 8ed547a111..2530e073d0 100644 --- a/gateway/gateway-controller/pkg/config/config.go +++ b/gateway/gateway-controller/pkg/config/config.go @@ -50,6 +50,7 @@ type Config struct { PolicyConfigurations map[string]interface{} `koanf:"policy_configurations"` Collector CollectorConfig `koanf:"collector"` Analytics AnalyticsConfig `koanf:"analytics"` + TrafficLogging TrafficLoggingConfig `koanf:"traffic_logging"` TracingConfig TracingConfig `koanf:"tracing"` APIKey APIKeyConfig `koanf:"api_key"` // Subscriptions controls application-level subscription behaviour for APIs. @@ -61,12 +62,11 @@ type Config struct { // CollectorConfig holds the data-collection ("collector") configuration. The // collector is the shared capture pipeline (the analytics system policy plus the // Envoy→policy-engine ALS transport) that gathers request/response headers and -// bodies. It is a prerequisite for every consumer of that data — analytics and -// traffic logging both require collector.enabled to be true. +// bodies. It underpins every consumer of that data (analytics and traffic logging) +// and is implicitly active whenever a consumer is enabled — see +// Config.IsCollectorEnabled. This section tunes capture and transport; it has no +// on/off flag of its own. type CollectorConfig struct { - // Enabled turns the collector on. When false, the analytics system policy is - // not injected and Envoy ships no access logs to the policy-engine. - Enabled bool `koanf:"enabled"` // SendRequestBody / SendResponseBody capture request/response bodies into the // collected event. SendRequestBody bool `koanf:"send_request_body"` @@ -102,6 +102,17 @@ type AnalyticsConfig struct { SendResponseBody bool `koanf:"send_response_body"` } +// TrafficLoggingConfig mirrors the policy-engine's stdout traffic-logging consumer. +// The controller only needs to know whether it is enabled, so that the collector +// (system policy + ALS sink) is activated when traffic logging is on even if +// analytics is off. Presentation keys (masked_headers, max_payload_size) are +// policy-engine-only and intentionally not bound here. +type TrafficLoggingConfig struct { + // Enabled turns stdout JSON traffic logging on. Enabling it implicitly activates + // the collector (see Config.IsCollectorEnabled). + Enabled bool `koanf:"enabled"` +} + // SubscriptionsConfig holds configuration for application-level subscriptions. type SubscriptionsConfig struct { // EnableValidation toggles automatic injection of the subscriptionValidation @@ -963,11 +974,10 @@ func defaultConfig() *Config { SendResponseBody: false, }, Collector: CollectorConfig{ - Enabled: false, SendRequestBody: false, SendResponseBody: false, - SendRequestHeaders: false, - SendResponseHeaders: false, + SendRequestHeaders: true, + SendResponseHeaders: true, GRPCEventServerCfg: defaultGRPCEventServerConfig(), }, TracingConfig: TracingConfig{ @@ -1742,23 +1752,14 @@ func validateDomains(field string, domains []string) error { } // validateCollectorConfig migrates deprecated analytics aliases onto the collector -// and enforces the collector prerequisite: analytics (a consumer) requires the -// collector that feeds it. For backward compatibility this is a soft prerequisite — -// if a consumer is enabled without the collector, the collector is auto-enabled with -// a warning rather than failing. It also validates the ALS transport tuning when the -// collector is enabled. +// and validates the ALS transport tuning when the collector is active. The collector +// has no on/off flag of its own: it is implicitly active whenever a consumer is +// enabled (analytics or traffic logging) — see IsCollectorEnabled. func (c *Config) validateCollectorConfig() error { c.migrateDeprecatedAnalyticsCapture() c.migrateDeprecatedAnalyticsTransport() - // Backward-compat bridge: a consumer cannot run without the collector that feeds - // it, but rather than fail an existing config that only set analytics.enabled - // (valid before the collector split), auto-enable the collector with a warning. - if c.Analytics.Enabled && !c.Collector.Enabled { - slog.Warn("analytics.enabled requires the collector; enabling collector.enabled automatically for backward compatibility. Set collector.enabled = true explicitly to silence this warning.") - c.Collector.Enabled = true - } - if c.Collector.Enabled { + if c.IsCollectorEnabled() { if err := validateGRPCEventServerConfig(c.Collector.GRPCEventServerCfg); err != nil { return err } @@ -1766,6 +1767,14 @@ func (c *Config) validateCollectorConfig() error { return nil } +// IsCollectorEnabled reports whether the collector should run. The collector is +// implicit: it is active whenever any consumer of the collected data is enabled +// (analytics or stdout traffic logging), and off otherwise. When active, the +// controller injects the analytics system policy and configures Envoy's ALS sink. +func (c *Config) IsCollectorEnabled() bool { + return c.Analytics.Enabled || c.TrafficLogging.Enabled +} + // migrateDeprecatedAnalyticsTransport maps a deprecated [analytics].grpc_event_server // override onto the collector when the collector's transport tuning is still at its // default, so existing configs keep working after the transport moved to [collector]. diff --git a/gateway/gateway-controller/pkg/config/config_test.go b/gateway/gateway-controller/pkg/config/config_test.go index e17039112d..4b237ac386 100644 --- a/gateway/gateway-controller/pkg/config/config_test.go +++ b/gateway/gateway-controller/pkg/config/config_test.go @@ -1302,10 +1302,9 @@ func TestConfig_ValidateAnalyticsConfig(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { cfg := validConfig() + // Analytics is a consumer; enabling it makes the collector implicitly + // active so these tests exercise the collector transport validation. cfg.Analytics.Enabled = tt.enabled - // Analytics is a consumer; enable the collector it depends on so these - // tests exercise analytics validation rather than the prerequisite check. - cfg.Collector.Enabled = tt.enabled if tt.setupConfig != nil { tt.setupConfig(cfg) } @@ -1320,34 +1319,34 @@ func TestConfig_ValidateAnalyticsConfig(t *testing.T) { } } -func TestConfig_CollectorPrerequisite(t *testing.T) { - t.Run("analytics enabled without collector auto-enables the collector", func(t *testing.T) { +// TestConfig_IsCollectorEnabled covers the implicit collector: it is active iff a +// consumer (analytics or traffic logging) is enabled, and off otherwise. +func TestConfig_IsCollectorEnabled(t *testing.T) { + t.Run("no consumers -> off", func(t *testing.T) { + cfg := validConfig() + assert.False(t, cfg.IsCollectorEnabled()) + require.NoError(t, cfg.Validate()) + }) + + t.Run("analytics on -> collector on", func(t *testing.T) { cfg := validConfig() - cfg.Collector.Enabled = false cfg.Analytics.Enabled = true cfg.Analytics.EnabledPublishers = []string{} - // Deprecated transport alias with valid values migrates onto the auto-enabled collector. - cfg.Analytics.GRPCEventServerCfg.Mode = "uds" - cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 1000 - cfg.Analytics.GRPCEventServerCfg.BufferSizeBytes = 16384 - cfg.Analytics.GRPCEventServerCfg.GRPCRequestTimeout = 5000 - cfg.Analytics.GRPCEventServerCfg.ServerPort = 18090 + assert.True(t, cfg.IsCollectorEnabled()) require.NoError(t, cfg.Validate()) - assert.True(t, cfg.Collector.Enabled, "collector should be auto-enabled for backward compatibility") }) - t.Run("collector enabled with no consumers is valid", func(t *testing.T) { + t.Run("traffic logging on -> collector on", func(t *testing.T) { cfg := validConfig() - cfg.Collector.Enabled = true + cfg.TrafficLogging.Enabled = true + assert.True(t, cfg.IsCollectorEnabled()) require.NoError(t, cfg.Validate()) }) - } func TestConfig_ValidateAnalyticsPayloadMigration(t *testing.T) { setValidAnalyticsGRPC := func(cfg *Config) { - cfg.Analytics.Enabled = true - cfg.Collector.Enabled = true // analytics is a consumer; the collector must be on + cfg.Analytics.Enabled = true // a consumer being on makes the collector implicit cfg.Analytics.GRPCEventServerCfg.Mode = "uds" cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 1000 cfg.Analytics.GRPCEventServerCfg.BufferSizeBytes = 16384 diff --git a/gateway/gateway-controller/pkg/utils/system_policies.go b/gateway/gateway-controller/pkg/utils/system_policies.go index 25a598a634..4d469fdc76 100644 --- a/gateway/gateway-controller/pkg/utils/system_policies.go +++ b/gateway/gateway-controller/pkg/utils/system_policies.go @@ -85,10 +85,10 @@ var defaultSystemPolicies = []systemPolicyConfig{ return false } // The analytics system policy is the collector: it is injected whenever - // the collector is enabled, regardless of which consumer (analytics, - // traffic logging) ultimately reads the collected data. - slog.Debug("Collector state -> ", "state", cfg.Collector.Enabled) - return cfg.Collector.Enabled + // the collector is active — i.e. whenever any consumer (analytics or + // traffic logging) is enabled. + slog.Debug("Collector state -> ", "state", cfg.IsCollectorEnabled()) + return cfg.IsCollectorEnabled() }, // Default parameters (can be overridden via additionalProps) Parameters: map[string]interface{}{ diff --git a/gateway/gateway-controller/pkg/utils/system_policies_test.go b/gateway/gateway-controller/pkg/utils/system_policies_test.go index f324db0377..5f38555594 100644 --- a/gateway/gateway-controller/pkg/utils/system_policies_test.go +++ b/gateway/gateway-controller/pkg/utils/system_policies_test.go @@ -125,11 +125,8 @@ func TestInjectSystemPolicies_NilConfig(t *testing.T) { } func TestInjectSystemPolicies_CollectorDisabled(t *testing.T) { - cfg := &config.Config{ - Collector: config.CollectorConfig{ - Enabled: false, - }, - } + // No consumer enabled -> collector implicitly off -> no system policy injected. + cfg := &config.Config{} policies := []policyenginev1.PolicyInstance{ {Name: "existing", Version: "v1.0.0"}, } @@ -140,10 +137,9 @@ func TestInjectSystemPolicies_CollectorDisabled(t *testing.T) { } func TestInjectSystemPolicies_CollectorEnabled(t *testing.T) { + // A consumer enabled -> collector implicitly on -> system policy injected. cfg := &config.Config{ - Collector: config.CollectorConfig{ - Enabled: true, - }, + Analytics: config.AnalyticsConfig{Enabled: true}, } policies := []policyenginev1.PolicyInstance{ {Name: "existing", Version: "v1.0.0"}, @@ -161,8 +157,8 @@ func TestInjectSystemPolicies_CollectorEnabled(t *testing.T) { func TestInjectSystemPolicies_BodyFlagsPropagated(t *testing.T) { cfg := &config.Config{ + Analytics: config.AnalyticsConfig{Enabled: true}, Collector: config.CollectorConfig{ - Enabled: true, SendRequestBody: true, SendResponseBody: true, }, @@ -177,8 +173,8 @@ func TestInjectSystemPolicies_BodyFlagsPropagated(t *testing.T) { func TestInjectSystemPolicies_BodyFlagsDefaultFalse(t *testing.T) { cfg := &config.Config{ + Analytics: config.AnalyticsConfig{Enabled: true}, Collector: config.CollectorConfig{ - Enabled: true, SendRequestBody: false, SendResponseBody: false, }, @@ -192,8 +188,8 @@ func TestInjectSystemPolicies_BodyFlagsDefaultFalse(t *testing.T) { func TestInjectSystemPolicies_HeaderFlagsPropagated(t *testing.T) { cfg := &config.Config{ + Analytics: config.AnalyticsConfig{Enabled: true}, Collector: config.CollectorConfig{ - Enabled: true, SendRequestHeaders: true, SendResponseHeaders: true, }, @@ -207,8 +203,10 @@ func TestInjectSystemPolicies_HeaderFlagsPropagated(t *testing.T) { } func TestInjectSystemPolicies_HeaderFlagsDefaultFalse(t *testing.T) { + // Zero-value collector (headers unset) with a consumer on: propagation passes the + // struct's false through. (Production defaults these true; see defaultConfig.) cfg := &config.Config{ - Collector: config.CollectorConfig{Enabled: true}, + Analytics: config.AnalyticsConfig{Enabled: true}, } result := InjectSystemPolicies(nil, cfg, nil) @@ -217,12 +215,9 @@ func TestInjectSystemPolicies_HeaderFlagsDefaultFalse(t *testing.T) { assert.Equal(t, false, result[0].Parameters["send_response_headers"]) } - func TestInjectSystemPolicies_WithAdditionalProps(t *testing.T) { cfg := &config.Config{ - Collector: config.CollectorConfig{ - Enabled: true, - }, + Analytics: config.AnalyticsConfig{Enabled: true}, } additionalProps := map[string]any{ constants.ANALYTICS_SYSTEM_POLICY_NAME: map[string]interface{}{ @@ -237,9 +232,7 @@ func TestInjectSystemPolicies_WithAdditionalProps(t *testing.T) { func TestInjectSystemPolicies_WithSharedParams(t *testing.T) { cfg := &config.Config{ - Collector: config.CollectorConfig{ - Enabled: true, - }, + Analytics: config.AnalyticsConfig{Enabled: true}, } additionalProps := map[string]any{ SharedParamsKey: map[string]interface{}{ @@ -254,9 +247,7 @@ func TestInjectSystemPolicies_WithSharedParams(t *testing.T) { func TestInjectSystemPolicies_EmptyPolicies(t *testing.T) { cfg := &config.Config{ - Collector: config.CollectorConfig{ - Enabled: true, - }, + Analytics: config.AnalyticsConfig{Enabled: true}, } result := InjectSystemPolicies([]policyenginev1.PolicyInstance{}, cfg, nil) @@ -266,9 +257,7 @@ func TestInjectSystemPolicies_EmptyPolicies(t *testing.T) { func TestInjectSystemPolicies_PreservesExistingPolicies(t *testing.T) { cfg := &config.Config{ - Collector: config.CollectorConfig{ - Enabled: true, - }, + Analytics: config.AnalyticsConfig{Enabled: true}, } policies := []policyenginev1.PolicyInstance{ {Name: "policy1", Version: "v1.0.0"}, diff --git a/gateway/gateway-controller/pkg/xds/translator.go b/gateway/gateway-controller/pkg/xds/translator.go index 7ae5387236..f0b5708d8f 100644 --- a/gateway/gateway-controller/pkg/xds/translator.go +++ b/gateway/gateway-controller/pkg/xds/translator.go @@ -561,9 +561,9 @@ func (t *Translator) TranslateConfigs( policyEngineCluster := t.createPolicyEngineCluster() clusters = append(clusters, policyEngineCluster) - // Add ALS cluster if the collector is enabled (it ships access logs over gRPC) + // Add ALS cluster if the collector is active (it ships access logs over gRPC) log.Debug("gRPC event server config", slog.Any("config", t.config.Collector.GRPCEventServerCfg)) - if t.config.Collector.Enabled { + if t.config.IsCollectorEnabled() { log.Info("collector is enabled, creating ALS cluster") alsCluster := t.createALSCluster() clusters = append(clusters, alsCluster) @@ -2785,8 +2785,8 @@ func (t *Translator) createAccessLogConfig() ([]*accesslog.AccessLog, error) { }, }) - // If the collector is enabled, create the gRPC access log config and append to existing access logs - if t.config.Collector.Enabled { + // If the collector is active, create the gRPC access log config and append to existing access logs + if t.config.IsCollectorEnabled() { t.logger.Info("Creating gRPC access log configuration") grpcAccessLog, err := t.createGRPCAccessLog() if err != nil { diff --git a/gateway/gateway-runtime/policy-engine/cmd/policy-engine/main.go b/gateway/gateway-runtime/policy-engine/cmd/policy-engine/main.go index b7116fd01c..1281a7a519 100644 --- a/gateway/gateway-runtime/policy-engine/cmd/policy-engine/main.go +++ b/gateway/gateway-runtime/policy-engine/cmd/policy-engine/main.go @@ -273,7 +273,7 @@ func main() { // consumers (analytics, traffic logging). var alsServer *grpc.Server slog.DebugContext(ctx, "Policy engine ALS server config", "config", cfg.Collector.AccessLogsServiceCfg) - if cfg.Collector.Enabled { + if cfg.IsCollectorEnabled() { // Start the access log service server slog.Info("Starting the ALS gRPC server...") alsServer = utils.StartAccessLogServiceServer(cfg) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go index 8960cfba2b..c75ba9bf35 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go @@ -153,9 +153,6 @@ func (c *Analytics) Process(event *v3.HTTPAccessLogEntry) { return } - // Path-based suppression is a per-consumer presentation concern: it is applied - // by the stdout traffic-logging publisher (traffic_logging.ignored_path_prefixes), - // not here, so other consumers still receive every event. analyticEvent := c.prepareAnalyticEvent(event) for _, publisher := range c.publishers { publisher.Publish(analyticEvent) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go index 4cc2ecf5d2..43810b57c9 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go @@ -63,9 +63,7 @@ func validAnalyticsConfigForValidation(analytics config.AnalyticsConfig) *config }, Analytics: analytics, } - cfg.Analytics.Enabled = true - // Analytics is a consumer; the collector must be enabled for it to validate. - cfg.Collector.Enabled = true + cfg.Analytics.Enabled = true // a consumer being on makes the collector implicit cfg.Collector.AccessLogsServiceCfg = config.AccessLogsServiceConfig{ Mode: "uds", ShutdownTimeout: 600 * time.Second, diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go index 75566db7e4..4e6daf6106 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go @@ -52,6 +52,10 @@ type TrafficLogDirective struct { Request *TrafficLogFlow `json:"request,omitempty"` Response *TrafficLogFlow `json:"response,omitempty"` Fields *TrafficLogFields `json:"fields,omitempty"` + // Properties holds the policy's resolved customProperties (context references + // already expanded at request time). The Log publisher emits them under + // properties.custom on the log line. + Properties map[string]interface{} `json:"properties,omitempty"` } // TrafficLogFlow is the per-flow (request or response) presentation config. diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go index c76cdf3db9..508cef8076 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go @@ -43,10 +43,6 @@ type Log struct { // maskedHeaders holds lower-cased header names whose values are redacted in // the requestHeaders/responseHeaders properties before logging. maskedHeaders map[string]bool - // ignoredPathPrefixes: skip emitting a line when the event's original request - // path starts with any of these (per-consumer — other publishers are - // unaffected). Trimmed of blanks at construction. - ignoredPathPrefixes []string // maxPayloadSize caps the number of request/response payload bytes written to // the log line (0 = no limit). Truncation is output-side only. maxPayloadSize int @@ -70,18 +66,10 @@ func NewLog(logCfg *config.TrafficLoggingConfig) *Log { } } - prefixes := make([]string, 0, len(logCfg.IgnoredPathPrefixes)) - for _, p := range logCfg.IgnoredPathPrefixes { - if p = strings.TrimSpace(p); p != "" { - prefixes = append(prefixes, p) - } - } - return &Log{ - maskedHeaders: masked, - ignoredPathPrefixes: prefixes, - maxPayloadSize: logCfg.MaxPayloadSize, - out: os.Stdout, + maskedHeaders: masked, + maxPayloadSize: logCfg.MaxPayloadSize, + out: os.Stdout, } } @@ -92,9 +80,6 @@ func (l *Log) Publish(event *dto.Event) { if event == nil || event.TrafficLog == nil { return } - if l.isPathIgnored(event) { - return - } out := l.shapeEvent(event) @@ -129,14 +114,15 @@ func (l *Log) Publish(event *dto.Event) { // redacts globally masked headers. ALS-derived fields (latencies, status, timing) // are always retained. func (l *Log) shapeEvent(event *dto.Event) *dto.Event { - if event.Properties == nil { + dir := event.TrafficLog + hasCustom := dir != nil && len(dir.Properties) > 0 + if event.Properties == nil && !hasCustom { return event } - props := make(map[string]interface{}, len(event.Properties)) + props := make(map[string]interface{}, len(event.Properties)+1) maps.Copy(props, event.Properties) - dir := event.TrafficLog // When an explicit field selection is set it is authoritative over presence, so // the per-flow headers/payload booleans are not used for gating here; only header // masking + per-flow excludeHeaders are applied, and the projection (in Publish) @@ -145,6 +131,12 @@ func (l *Log) shapeEvent(event *dto.Event) *dto.Event { l.applyFlow(props, dir.Request, "requestHeaders", "request_payload", gate) l.applyFlow(props, dir.Response, "responseHeaders", "response_payload", gate) + // Attach the policy's resolved custom properties under a dedicated namespace, so + // they never collide with reserved keys and are projectable as "properties.custom". + if hasCustom { + props["custom"] = dir.Properties + } + cp := *event cp.Properties = props return &cp @@ -176,25 +168,6 @@ func (l *Log) applyFlow(props map[string]interface{}, flow *dto.TrafficLogFlow, } } -// isPathIgnored reports whether the event's original request path starts with any -// configured ignored prefix. It reads Operation.APIResourceTemplate, which carries -// the original (pre-rewrite) path. -func (l *Log) isPathIgnored(event *dto.Event) bool { - if len(l.ignoredPathPrefixes) == 0 || event.Operation == nil { - return false - } - path := event.Operation.APIResourceTemplate - if path == "" { - return false - } - for _, prefix := range l.ignoredPathPrefixes { - if strings.HasPrefix(path, prefix) { - return true - } - } - return false -} - // truncatePayload returns up to maxPayloadSize bytes of the payload (0 = no // limit). Truncation is on a byte boundary, matching the previous capture-time // behavior. diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go index 641d532aa0..a912257c18 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go @@ -107,6 +107,66 @@ func TestLog_Publish_WritesJSONLineWithLatencies(t *testing.T) { assert.Equal(t, float64(100), latencies["responseLatency"]) } +// Custom properties from the directive are emitted under properties.custom. +func TestLog_Publish_CustomPropertiesUnderCustom(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.TrafficLog = &dto.TrafficLogDirective{ + Request: &dto.TrafficLogFlow{Headers: true}, + Properties: map[string]interface{}{ + "who": "alice", + "authType": "jwt", + "retryCount": float64(3), + }, + } + event.Properties["requestHeaders"] = `{"x-foo":"bar"}` + + l.Publish(event) + + _, props := decodeLine(t, read()) + custom, ok := props["custom"].(map[string]interface{}) + require.True(t, ok, "expected properties.custom object, got %v", props["custom"]) + assert.Equal(t, "alice", custom["who"]) + assert.Equal(t, "jwt", custom["authType"]) + assert.Equal(t, float64(3), custom["retryCount"]) + // Reserved keys are untouched by the custom namespace. + assert.Equal(t, `{"x-foo":"bar"}`, props["requestHeaders"]) +} + +// A directive with no Properties emits no custom key. +func TestLog_Publish_NoCustomWhenAbsent(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.TrafficLog = bothFlows() + + l.Publish(event) + + _, props := decodeLine(t, read()) + _, present := props["custom"] + assert.False(t, present, "no custom key expected when directive has no Properties") +} + +// The fields projection can select properties.custom like any other property path. +func TestLog_Publish_CustomProjectableViaFields(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.TrafficLog = &dto.TrafficLogDirective{ + Properties: map[string]interface{}{"who": "alice"}, + Fields: &dto.TrafficLogFields{Mode: "include", Names: []string{"properties.custom"}}, + } + event.Properties["requestHeaders"] = `{"x-foo":"bar"}` + + l.Publish(event) + + _, props := decodeLine(t, read()) + custom, ok := props["custom"].(map[string]interface{}) + require.True(t, ok, "expected properties.custom retained by include projection") + assert.Equal(t, "alice", custom["who"]) + // Non-selected property dropped by the include projection. + _, hasHeaders := props["requestHeaders"] + assert.False(t, hasHeaders, "requestHeaders should be dropped by include projection") +} + func TestLog_Publish_MasksHeaders(t *testing.T) { l, read := newLogToFile(t, &config.TrafficLoggingConfig{MaskedHeaders: []string{"Authorization"}}) event := createBaseEvent() @@ -273,29 +333,6 @@ func TestLog_Publish_FieldsIncludeWholeProperties(t *testing.T) { assert.Contains(t, props, "responseHeaders") } -// Per-consumer path ignore: the publisher skips ignored prefixes (matched on the -// event's original path in Operation.APIResourceTemplate); other consumers are -// unaffected. -func TestLog_Publish_IgnoredPathSkipped(t *testing.T) { - l, read := newLogToFile(t, &config.TrafficLoggingConfig{IgnoredPathPrefixes: []string{"/health", "/ready"}}) - event := createBaseEvent() - event.TrafficLog = bothFlows() - event.Operation.APIResourceTemplate = "/health/live" - - l.Publish(event) - assert.Empty(t, read(), "ignored path must not be logged") -} - -func TestLog_Publish_NonIgnoredPathLogged(t *testing.T) { - l, read := newLogToFile(t, &config.TrafficLoggingConfig{IgnoredPathPrefixes: []string{"/health"}}) - event := createBaseEvent() - event.TrafficLog = bothFlows() - event.Operation.APIResourceTemplate = "/api/v1/orders" - - l.Publish(event) - assert.NotEmpty(t, read(), "non-ignored path must be logged") -} - // Output-side payload truncation (0 = no limit). func TestLog_Publish_TruncatesPayload(t *testing.T) { l, read := newLogToFile(t, &config.TrafficLoggingConfig{MaxPayloadSize: 5}) diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config.go b/gateway/gateway-runtime/policy-engine/internal/config/config.go index 00fe960056..2d6690f7a7 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config.go @@ -50,13 +50,11 @@ type Config struct { // CollectorConfig holds the data-collection ("collector") configuration. The // collector is the shared capture pipeline that gathers request/response headers -// and bodies and ships them to the policy-engine over ALS. It is a prerequisite -// for every consumer of that data — analytics and traffic logging both require -// collector.enabled to be true. +// and bodies and ships them to the policy-engine over ALS. It underpins every +// consumer of that data (analytics and traffic logging) and is implicitly active +// whenever a consumer is enabled — see Config.IsCollectorEnabled. This section +// tunes capture and transport; it has no on/off flag of its own. type CollectorConfig struct { - // Enabled turns the collector on. When false, the ALS server is not started - // and no events are produced for any consumer. - Enabled bool `koanf:"enabled"` // SendRequestBody / SendResponseBody attach captured request/response bodies // onto the collected event. SendRequestBody bool `koanf:"send_request_body"` @@ -94,7 +92,7 @@ type AnalyticsPublishersConfig struct { // TrafficLoggingConfig holds configuration for the stdout traffic-logging feature, // which writes each collected event to stdout as a JSON line. It is a consumer of -// the collector and requires collector.enabled to be true. +// the collector; enabling it implicitly activates the collector. type TrafficLoggingConfig struct { // Enabled turns stdout JSON traffic logging on. Enabled bool `koanf:"enabled"` @@ -106,11 +104,6 @@ type TrafficLoggingConfig struct { // the collector still captures the full body and other consumers (e.g. Moesif) // are unaffected. MaxPayloadSize int `koanf:"max_payload_size"` - // IgnoredPathPrefixes suppresses stdout traffic-log lines for requests whose - // original path starts with any of these prefixes (e.g. health/readiness - // probes). Only the traffic-logging publisher is affected; other consumers - // still receive the event. - IgnoredPathPrefixes []string `koanf:"ignored_path_prefixes"` } // MoesifPublisherConfig holds Moesif-specific configuration @@ -410,16 +403,14 @@ func defaultConfig() *Config { TracingServiceName: "policy-engine", }, Collector: CollectorConfig{ - Enabled: false, SendRequestBody: false, SendResponseBody: false, AccessLogsServiceCfg: defaultAccessLogsServiceConfig(), }, TrafficLogging: TrafficLoggingConfig{ - Enabled: false, - MaskedHeaders: []string{}, - MaxPayloadSize: 0, - IgnoredPathPrefixes: []string{}, + Enabled: false, + MaskedHeaders: []string{}, + MaxPayloadSize: 0, }, Analytics: AnalyticsConfig{ Enabled: false, @@ -608,21 +599,14 @@ func (c *Config) validateXDSConfig() error { // validateCollectorConfig migrates deprecated analytics capture aliases onto the // collector and enforces the collector prerequisite: a consumer (analytics or -// traffic logging) requires the collector that feeds it. For backward compatibility -// this is a soft prerequisite — if a consumer is enabled without the collector, the -// collector is auto-enabled with a warning rather than failing. +// traffic logging) requires the collector that feeds it. The collector has no +// on/off flag of its own: it is implicitly active whenever a consumer is enabled +// (see IsCollectorEnabled), so its transport is validated only in that case. func (c *Config) validateCollectorConfig() error { c.migrateDeprecatedAnalyticsCapture() c.migrateDeprecatedAnalyticsTransport() - // Backward-compat bridge: a consumer cannot run without the collector that feeds - // it, but rather than fail an existing config that only set analytics.enabled - // (valid before the collector split), auto-enable the collector with a warning. - if (c.Analytics.Enabled || c.TrafficLogging.Enabled) && !c.Collector.Enabled { - slog.Warn("a consumer (analytics.enabled or traffic_logging.enabled) requires the collector; enabling collector.enabled automatically for backward compatibility. Set collector.enabled = true explicitly to silence this warning.") - c.Collector.Enabled = true - } - if c.Collector.Enabled { + if c.IsCollectorEnabled() { if err := validateAccessLogsServiceConfig(c.Collector.AccessLogsServiceCfg); err != nil { return err } @@ -630,6 +614,13 @@ func (c *Config) validateCollectorConfig() error { return nil } +// IsCollectorEnabled reports whether the collector should run. The collector is +// implicit: it is active whenever any consumer of the collected data is enabled +// (analytics or stdout traffic logging), and off otherwise. +func (c *Config) IsCollectorEnabled() bool { + return c.Analytics.Enabled || c.TrafficLogging.Enabled +} + // migrateDeprecatedAnalyticsTransport maps a deprecated [analytics].access_logs_service // override onto the collector when the collector's receiver tuning is still at its // default, so existing configs keep working after the transport moved to [collector]. diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config_test.go b/gateway/gateway-runtime/policy-engine/internal/config/config_test.go index 9be230088d..b9371d6990 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config_test.go @@ -69,12 +69,10 @@ func validConfig() *Config { Timeout: 30 * time.Second, }, }, - // Collector enabled by default so tests that turn on a consumer - // (analytics / traffic logging) satisfy the collector prerequisite. - // ALS receiver defaults mirror production so transport validation passes - // and the deprecated alias stays neutral (no spurious migration). + // The collector is implicit (active whenever a consumer is enabled). ALS + // receiver defaults mirror production so transport validation passes and the + // deprecated alias stays neutral (no spurious migration). Collector: CollectorConfig{ - Enabled: true, AccessLogsServiceCfg: defaultAccessLogsServiceConfig(), }, Analytics: AnalyticsConfig{ @@ -1108,8 +1106,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { func TestValidate_AnalyticsPayloadMigration(t *testing.T) { setValidAnalyticsALS := func(cfg *Config) { - cfg.Analytics.Enabled = true - cfg.Collector.Enabled = true // analytics is a consumer; the collector must be on + cfg.Analytics.Enabled = true // a consumer being on makes the collector implicit cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ Mode: "uds", ShutdownTimeout: 600 * time.Second, @@ -1180,49 +1177,34 @@ func TestValidate_AnalyticsPayloadMigration(t *testing.T) { // TestValidate_CollectorPrerequisite verifies that enabling a consumer (analytics, // traffic logging) without the collector auto-enables the collector (a // backward-compat soft prerequisite) rather than failing. -func TestValidate_CollectorPrerequisite(t *testing.T) { - validALS := AccessLogsServiceConfig{ - Mode: "uds", - ShutdownTimeout: 600 * time.Second, - ExtProcMaxMessageSize: 1000000, - ExtProcMaxHeaderLimit: 8192, - } - - t.Run("analytics enabled without collector auto-enables the collector", func(t *testing.T) { +// TestIsCollectorEnabled covers the implicit collector: it is active iff a consumer +// (analytics or traffic logging) is enabled, and off otherwise. +func TestIsCollectorEnabled(t *testing.T) { + t.Run("no consumers -> off", func(t *testing.T) { cfg := validConfig() - cfg.Collector.Enabled = false - cfg.Analytics.Enabled = true - cfg.Collector.AccessLogsServiceCfg = validALS - cfg.Analytics.EnabledPublishers = []string{} - require.NoError(t, cfg.Validate()) - assert.True(t, cfg.Collector.Enabled, "collector should be auto-enabled for backward compatibility") + cfg.Analytics.Enabled = false + cfg.TrafficLogging.Enabled = false + assert.False(t, cfg.IsCollectorEnabled()) }) - t.Run("traffic logging enabled without collector auto-enables the collector", func(t *testing.T) { + t.Run("analytics on -> collector on", func(t *testing.T) { cfg := validConfig() - cfg.Collector.Enabled = false - cfg.TrafficLogging.Enabled = true + cfg.Analytics.Enabled = true + cfg.Analytics.EnabledPublishers = []string{} + assert.True(t, cfg.IsCollectorEnabled()) require.NoError(t, cfg.Validate()) - assert.True(t, cfg.Collector.Enabled, "collector should be auto-enabled for backward compatibility") }) - t.Run("traffic logging enabled with collector is valid", func(t *testing.T) { + t.Run("traffic logging on -> collector on", func(t *testing.T) { cfg := validConfig() - cfg.Collector.Enabled = true cfg.TrafficLogging.Enabled = true - require.NoError(t, cfg.Validate()) - }) - - t.Run("collector enabled with no consumers is valid", func(t *testing.T) { - cfg := validConfig() - cfg.Collector.Enabled = true + assert.True(t, cfg.IsCollectorEnabled()) require.NoError(t, cfg.Validate()) }) } func TestValidate_TrafficLoggingMaxPayloadSize(t *testing.T) { cfg := validConfig() - cfg.Collector.Enabled = true cfg.TrafficLogging.Enabled = true cfg.TrafficLogging.MaxPayloadSize = -1 err := cfg.Validate() diff --git a/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server_test.go b/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server_test.go index 0178fc5356..3176524241 100644 --- a/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server_test.go @@ -211,7 +211,6 @@ func TestStreamAccessLogs_MultipleMessages(t *testing.T) { func TestStartAccessLogServiceServer_TCP(t *testing.T) { cfg := &config.Config{ Collector: config.CollectorConfig{ - Enabled: true, AccessLogsServiceCfg: config.AccessLogsServiceConfig{ Mode: "tcp", ServerPort: 19001, // Use non-standard port to avoid conflicts From a3be700e45538bc4159c91aded82c33e72ea6246 Mon Sep 17 00:00:00 2001 From: Dineth Date: Mon, 6 Jul 2026 09:56:14 +0530 Subject: [PATCH 03/21] Enhance analytics configuration by adding support for additional masked headers and caching traffic log directives --- gateway/configs/config-template.toml | 2 +- gateway/configs/config.toml | 2 +- .../pkg/utils/system_policies.go | 10 ++-- .../internal/analytics/analytics.go | 47 ++++++++++++------- .../internal/analytics/analytics_test.go | 30 ++++++------ .../internal/analytics/dto/event.go | 10 ++++ .../internal/analytics/publishers/log.go | 4 +- .../policy-engine/internal/config/config.go | 2 - .../analytics/policy-definition.yaml | 8 ---- 9 files changed, 61 insertions(+), 54 deletions(-) diff --git a/gateway/configs/config-template.toml b/gateway/configs/config-template.toml index 4f3530ad12..ae83145e3a 100644 --- a/gateway/configs/config-template.toml +++ b/gateway/configs/config-template.toml @@ -367,7 +367,7 @@ enabled = false # Header names (case-insensitive) whose values are redacted as "****" in the # logged requestHeaders/responseHeaders. Applies globally; the `log-message` # policy's per-API excludeHeaders drop additional headers entirely. -masked_headers = ["authorization"] +masked_headers = ["authorization", "x-api-key", "x-jwt-assertion"] # Max bytes of request/response payload written to each log line (0 = no limit). # Applied output-side, so the collector still captures full bodies and other # consumers (e.g. Moesif) are unaffected. diff --git a/gateway/configs/config.toml b/gateway/configs/config.toml index bc22d25487..83107adfa6 100644 --- a/gateway/configs/config.toml +++ b/gateway/configs/config.toml @@ -24,7 +24,7 @@ enabled_publishers = ["moesif"] [traffic_logging] enabled = true # Header names (case-insensitive) whose values are redacted as "****". -masked_headers = ["authorization"] +masked_headers = ["authorization", "x-api-key", "x-jwt-assertion"] # Max bytes of request/response payload written to each log line (0 = no limit). max_payload_size = 2048 diff --git a/gateway/gateway-controller/pkg/utils/system_policies.go b/gateway/gateway-controller/pkg/utils/system_policies.go index 4d469fdc76..8985f53388 100644 --- a/gateway/gateway-controller/pkg/utils/system_policies.go +++ b/gateway/gateway-controller/pkg/utils/system_policies.go @@ -90,13 +90,9 @@ var defaultSystemPolicies = []systemPolicyConfig{ slog.Debug("Collector state -> ", "state", cfg.IsCollectorEnabled()) return cfg.IsCollectorEnabled() }, - // Default parameters (can be overridden via additionalProps) - Parameters: map[string]interface{}{ - "send_request_body": false, - "send_response_body": false, - "send_request_headers": false, - "send_response_headers": false, - }, + // No static defaults — all four capture flags are set at injection time + // from cfg.Collector (see InjectSystemPolicies below). + Parameters: nil, ExecutionCondition: nil, }, } diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go index c75ba9bf35..1f6c63672e 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go @@ -23,6 +23,7 @@ import ( "log/slog" "maps" "strconv" + "sync" "time" v3 "github.com/envoyproxy/go-control-plane/envoy/data/accesslog/v3" @@ -59,8 +60,6 @@ const ( DefaultAnalyticsPublisher = "default" // MoesifAnalyticsPublisher represents the Moesif analytics publisher. MoesifAnalyticsPublisher = "moesif" - // LogAnalyticsPublisher represents the stdout/log analytics publisher. - LogAnalyticsPublisher = "log" // HeaderKeys represents the header keys. RequestHeadersKey = "request_headers" @@ -96,6 +95,10 @@ type Analytics struct { cfg *config.Config // publishers represents the publishers. publishers []analytics_publisher.Publisher + // directiveCache caches parsed TrafficLogDirectives keyed by raw JSON string. + // The directive is static per API deployment, so this eliminates per-request + // allocations after the first parse. + directiveCache sync.Map } // NewAnalytics creates a new instance of Analytics. Publishers are assembled from @@ -115,8 +118,6 @@ func NewAnalytics(cfg *config.Config) *Analytics { publishers = append(publishers, publisher) slog.Info("Moesif publisher added") } - case LogAnalyticsPublisher: - slog.Warn("\"log\" in analytics.enabled_publishers is no longer supported; enable stdout traffic logging via [traffic_logging] instead") default: slog.Warn("Unknown publisher type", "type", publisherName) } @@ -225,13 +226,7 @@ func (c *Analytics) prepareAnalyticEvent(logEntry *v3.HTTPAccessLogEntry) *dto.E // publishers (e.g. Moesif). A malformed marker still opts the API in with // default (empty) presentation, since attachment alone signals intent. if raw, exists := keyValuePairsFromMetadata[TrafficLogMetadataKey]; exists { - dir := &dto.TrafficLogDirective{} - if raw != "" { - if err := json.Unmarshal([]byte(raw), dir); err != nil { - slog.Warn("Failed to parse traffic_log directive; opting in with defaults", "error", err) - } - } - event.TrafficLog = dir + event.TrafficLog = c.parseTrafficLogDirective(raw) } // Prepare extended API extendedAPI := dto.ExtendedAPI{} @@ -478,22 +473,22 @@ func (c *Analytics) prepareAnalyticEvent(logEntry *v3.HTTPAccessLogEntry) *dto.E //Adding request and response headers for the analytics event if requestHeaders, exists := keyValuePairsFromMetadata[RequestHeadersKey]; exists { - event.Properties["requestHeaders"] = requestHeaders + event.Properties[dto.PropKeyRequestHeaders] = requestHeaders } if responseHeaders, exists := keyValuePairsFromMetadata[ResponseHeadersKey]; exists { - event.Properties["responseHeaders"] = responseHeaders + event.Properties[dto.PropKeyResponseHeaders] = responseHeaders } // Optionally attach request and response payloads when enabled via the collector. if c.cfg.Collector.SendRequestBody { - if requestPayload, ok := keyValuePairsFromMetadata["request_payload"]; ok && requestPayload != "" { - event.Properties["request_payload"] = requestPayload + if requestPayload, ok := keyValuePairsFromMetadata[dto.PropKeyRequestPayload]; ok && requestPayload != "" { + event.Properties[dto.PropKeyRequestPayload] = requestPayload slog.Debug("Analytics request payload captured", "size_bytes", len(requestPayload)) } } if c.cfg.Collector.SendResponseBody { - if responsePayload, ok := keyValuePairsFromMetadata["response_payload"]; ok && responsePayload != "" { - event.Properties["response_payload"] = responsePayload + if responsePayload, ok := keyValuePairsFromMetadata[dto.PropKeyResponsePayload]; ok && responsePayload != "" { + event.Properties[dto.PropKeyResponsePayload] = responsePayload slog.Debug("Analytics response payload captured", "size_bytes", len(responsePayload)) } } @@ -544,6 +539,24 @@ func (c *Analytics) prepareAnalyticEvent(logEntry *v3.HTTPAccessLogEntry) *dto.E return event } +// parseTrafficLogDirective returns the parsed TrafficLogDirective for the given +// raw JSON string, using a cache to avoid re-parsing the same static per-API +// directive on every request. Two concurrent first-parses of the same string are +// benign — the winner's value is stored and both callers get an equivalent result. +func (c *Analytics) parseTrafficLogDirective(raw string) *dto.TrafficLogDirective { + if cached, ok := c.directiveCache.Load(raw); ok { + return cached.(*dto.TrafficLogDirective) + } + dir := &dto.TrafficLogDirective{} + if raw != "" { + if err := json.Unmarshal([]byte(raw), dir); err != nil { + slog.Warn("Failed to parse traffic_log directive; opting in with defaults", "error", err) + } + } + c.directiveCache.Store(raw, dir) + return dir +} + func (c *Analytics) getAnonymousApp() *dto.Application { application := &dto.Application{} application.ApplicationID = anonymousValue diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go index 43810b57c9..0b15b3d3e3 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go @@ -150,22 +150,6 @@ func TestNewAnalytics_TrafficLoggingEnabled(t *testing.T) { assert.Len(t, analytics.publishers, 1) // traffic-logging publisher should be registered } -func TestNewAnalytics_LogInEnabledPublishersIgnored(t *testing.T) { - // "log" in analytics.enabled_publishers is no longer supported; it must not - // register a publisher (use [traffic_logging] instead). - cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ - Enabled: true, - EnabledPublishers: []string{LogAnalyticsPublisher}, - }, - } - - analytics := NewAnalytics(cfg) - - require.NotNil(t, analytics) - assert.Empty(t, analytics.publishers) -} - // ============================================================================= // isInvalid Tests // ============================================================================= @@ -473,6 +457,20 @@ func TestPrepareAnalyticEvent_MalformedTrafficLogMarker(t *testing.T) { assert.Nil(t, event.TrafficLog.Response) } +func TestPrepareAnalyticEvent_TrafficLogDirectiveCached(t *testing.T) { + cfg := &config.Config{} + a := NewAnalytics(cfg) + raw := `{"request":{"payload":false,"headers":true}}` + logEntry := createLogEntryWithMetadata(map[string]string{TrafficLogMetadataKey: raw}) + + event1 := a.prepareAnalyticEvent(logEntry) + event2 := a.prepareAnalyticEvent(logEntry) + + require.NotNil(t, event1.TrafficLog) + // Same pointer — second call must return the cached directive, not a new allocation. + assert.Same(t, event1.TrafficLog, event2.TrafficLog) +} + func TestPrepareAnalyticEvent_WithAnonymousApp(t *testing.T) { cfg := &config.Config{} analytics := NewAnalytics(cfg) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go index 4e6daf6106..2096091f7f 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go @@ -19,6 +19,16 @@ package dto import "time" +// Property keys written into Event.Properties by the analytics pipeline and read +// back by publishers (e.g. the Log publisher's masking and field-projection paths). +// Both sides must use these constants so a rename stays in one place. +const ( + PropKeyRequestHeaders = "requestHeaders" + PropKeyResponseHeaders = "responseHeaders" + PropKeyRequestPayload = "request_payload" + PropKeyResponsePayload = "response_payload" +) + // Event represents analytics event data. type Event struct { API *ExtendedAPI `json:"api,omitempty" bson:"api"` diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go index 508cef8076..5839d97bdf 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go @@ -128,8 +128,8 @@ func (l *Log) shapeEvent(event *dto.Event) *dto.Event { // masking + per-flow excludeHeaders are applied, and the projection (in Publish) // decides which fields survive. gate := dir.Fields == nil || len(dir.Fields.Names) == 0 - l.applyFlow(props, dir.Request, "requestHeaders", "request_payload", gate) - l.applyFlow(props, dir.Response, "responseHeaders", "response_payload", gate) + l.applyFlow(props, dir.Request, dto.PropKeyRequestHeaders, dto.PropKeyRequestPayload, gate) + l.applyFlow(props, dir.Response, dto.PropKeyResponseHeaders, dto.PropKeyResponsePayload, gate) // Attach the policy's resolved custom properties under a dedicated namespace, so // they never collide with reserved keys and are projectable as "properties.custom". diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config.go b/gateway/gateway-runtime/policy-engine/internal/config/config.go index 2d6690f7a7..4e26d0fd3e 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config.go @@ -705,8 +705,6 @@ func (c *Config) validateAnalyticsConfig() error { return fmt.Errorf("analytics.publishers.moesif.moesif_base_url must be a valid URL (e.g. https://api.moesif.net), got %q", moesifCfg.BaseURL) } } - case "log": - // The stdout/log publisher has no required configuration. default: return fmt.Errorf("unknown publisher type in enabled_publishers: %s", publisherName) } diff --git a/gateway/system-policies/analytics/policy-definition.yaml b/gateway/system-policies/analytics/policy-definition.yaml index eaed1b307e..c3c223bd8a 100644 --- a/gateway/system-policies/analytics/policy-definition.yaml +++ b/gateway/system-policies/analytics/policy-definition.yaml @@ -41,14 +41,6 @@ parameters: When true, the analytics system policy captures ALL response headers into analytics metadata as response_headers, which is forwarded through the analytics pipeline (including the stdout/log and Moesif publishers). - max_payload_size: - type: integer - default: 0 - description: > - Maximum number of bytes captured per request/response body when - send_request_body / send_response_body is enabled. 0 (default) means no - limit; a positive value truncates each body to that many bytes (silent, - byte-boundary truncation). allow_payloads: type: boolean default: false From 43777f1b29efdab1892e3fae9371673cc0df9dee Mon Sep 17 00:00:00 2001 From: Dineth Date: Mon, 6 Jul 2026 15:35:32 +0530 Subject: [PATCH 04/21] Refactor traffic log handling to support labels and masked headers, and improve event structure --- .../internal/analytics/analytics_test.go | 3 +- .../internal/analytics/dto/event.go | 34 ++- .../internal/analytics/publishers/log.go | 258 +++++++---------- .../internal/analytics/publishers/log_test.go | 271 ++++++++++++------ .../analytics/publishers/traffic_log_event.go | 197 +++++++++++++ 5 files changed, 501 insertions(+), 262 deletions(-) create mode 100644 gateway/gateway-runtime/policy-engine/internal/analytics/publishers/traffic_log_event.go diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go index 0b15b3d3e3..38756eaf73 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go @@ -409,7 +409,7 @@ func TestPrepareAnalyticEvent_TrafficLogMarker(t *testing.T) { logEntry := createLogEntryWithMetadata(map[string]string{ APINameKey: "TestAPI", - TrafficLogMetadataKey: `{"request":{"payload":false,"headers":true,"excludeHeaders":["x-key"]}}`, + TrafficLogMetadataKey: `{"request":{"payload":false,"headers":true}}`, }) event := analytics.prepareAnalyticEvent(logEntry) @@ -419,7 +419,6 @@ func TestPrepareAnalyticEvent_TrafficLogMarker(t *testing.T) { require.NotNil(t, event.TrafficLog.Request) assert.True(t, event.TrafficLog.Request.Headers) assert.False(t, event.TrafficLog.Request.Payload) - assert.Equal(t, []string{"x-key"}, event.TrafficLog.Request.ExcludeHeaders) assert.Nil(t, event.TrafficLog.Response, "unset flow stays nil") // The marker must never leak into serialized properties (or other publishers). diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go index 2096091f7f..fe5d7a89c4 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go @@ -62,26 +62,30 @@ type TrafficLogDirective struct { Request *TrafficLogFlow `json:"request,omitempty"` Response *TrafficLogFlow `json:"response,omitempty"` Fields *TrafficLogFields `json:"fields,omitempty"` - // Properties holds the policy's resolved customProperties (context references - // already expanded at request time). The Log publisher emits them under - // properties.custom on the log line. - Properties map[string]interface{} `json:"properties,omitempty"` + // Labels holds the policy's resolved labels (context references already + // expanded at request time). The Log publisher emits them as a top-level + // "labels" object on the log line. + Labels map[string]interface{} `json:"labels,omitempty"` + // MaskedHeaders lists lower-cased header names whose values are redacted + // in the emitted log line. Merged with the global masked_headers config. + MaskedHeaders []string `json:"maskedHeaders,omitempty"` } // TrafficLogFlow is the per-flow (request or response) presentation config. type TrafficLogFlow struct { - Payload bool `json:"payload"` - Headers bool `json:"headers"` - ExcludeHeaders []string `json:"excludeHeaders,omitempty"` + Payload bool `json:"payload"` + Headers bool `json:"headers"` } -// TrafficLogFields selects which fields appear in the emitted line. When set it is -// authoritative over field presence: the per-flow Payload/Headers booleans are -// ignored (per-flow ExcludeHeaders and global masking still apply). Names are -// top-level keys (e.g. "latencies", "target") or dotted property paths -// (e.g. "properties.requestHeaders"). Mode "exclude" drops the named keys; any -// other value (default "include") keeps only the named keys. +// TrafficLogFields selects which fields appear in the emitted line. Exactly one +// of Only or Exclude should be set. Only keeps exactly the named fields; Exclude +// drops the named fields and keeps everything else. Names are top-level keys +// (e.g. "latencies", "requestHeaders") or dotted sub-key paths within map fields +// (e.g. "requestHeaders.authorization", "labels.env"). When set, this is +// authoritative over field presence; per-flow Payload/Headers booleans are ignored +// (global header masking still applies). If both are set, Only takes precedence. type TrafficLogFields struct { - Mode string `json:"mode,omitempty"` - Names []string `json:"names,omitempty"` + Only []string `json:"only,omitempty"` + Exclude []string `json:"exclude,omitempty"` } + diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go index 5839d97bdf..0718c1aec7 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go @@ -21,7 +21,6 @@ import ( "encoding/json" "fmt" "log/slog" - "maps" "os" "strings" "sync" @@ -81,25 +80,33 @@ func (l *Log) Publish(event *dto.Event) { return } - out := l.shapeEvent(event) + dir := event.TrafficLog + tl := l.toTrafficLogEvent(event, dir) - data, err := json.Marshal(out) + data, err := json.Marshal(tl) if err != nil { - slog.Error("Failed to marshal analytics event for log publisher", "error", err) + slog.Error("Failed to marshal traffic-log event", "error", err) return } - // When an explicit field selection is configured it is authoritative over which - // fields appear: project the serialized record down to (or removing) the named - // top-level keys and properties.* paths. - if fields := event.TrafficLog.Fields; fields != nil && len(fields.Names) > 0 { - if projected, perr := applyFieldsProjection(data, fields); perr != nil { - slog.Error("Failed to project traffic-log fields; emitting unprojected line", "error", perr) + if fields := dir.Fields; fields != nil && (len(fields.Only) > 0 || len(fields.Exclude) > 0) { + var m map[string]interface{} + if err := json.Unmarshal(data, &m); err != nil { + slog.Error("Failed to unmarshal for field projection; emitting as-is", "error", err) } else { - data = projected + applyFieldsProjection(m, fields) + if projected, merr := json.Marshal(m); merr == nil { + data = projected + } else { + slog.Error("Failed to remarshal after field projection; emitting as-is", "error", merr) + } } } + l.write(data) +} + +func (l *Log) write(data []byte) { l.mu.Lock() defer l.mu.Unlock() if _, err := fmt.Fprintln(l.out, string(data)); err != nil { @@ -107,65 +114,33 @@ func (l *Log) Publish(event *dto.Event) { } } -// shapeEvent returns the event to serialize, applying the per-API traffic-log -// directive to a shallow copy with a cloned Properties map so the shared event -// observed by other publishers is left untouched. For each flow it drops the -// headers/payload the API did not request, removes per-flow excluded headers, and -// redacts globally masked headers. ALS-derived fields (latencies, status, timing) -// are always retained. -func (l *Log) shapeEvent(event *dto.Event) *dto.Event { - dir := event.TrafficLog - hasCustom := dir != nil && len(dir.Properties) > 0 - if event.Properties == nil && !hasCustom { - return event - } - - props := make(map[string]interface{}, len(event.Properties)+1) - maps.Copy(props, event.Properties) - - // When an explicit field selection is set it is authoritative over presence, so - // the per-flow headers/payload booleans are not used for gating here; only header - // masking + per-flow excludeHeaders are applied, and the projection (in Publish) - // decides which fields survive. - gate := dir.Fields == nil || len(dir.Fields.Names) == 0 - l.applyFlow(props, dir.Request, dto.PropKeyRequestHeaders, dto.PropKeyRequestPayload, gate) - l.applyFlow(props, dir.Response, dto.PropKeyResponseHeaders, dto.PropKeyResponsePayload, gate) - - // Attach the policy's resolved custom properties under a dedicated namespace, so - // they never collide with reserved keys and are projectable as "properties.custom". - if hasCustom { - props["custom"] = dir.Properties - } - - cp := *event - cp.Properties = props - return &cp -} - -// applyFlow enforces one flow's presentation rules on the cloned properties. -// Header content, when present, is always cleaned (per-flow excludeHeaders + global -// masking). When gate is true (no authoritative field selection), a nil flow or a -// disabled headers/payload field also drops the corresponding property entirely. -func (l *Log) applyFlow(props map[string]interface{}, flow *dto.TrafficLogFlow, headersKey, payloadKey string, gate bool) { - var exclude []string - if flow != nil { - exclude = flow.ExcludeHeaders - } - if raw, ok := props[headersKey].(string); ok { - props[headersKey] = l.filterHeaders(raw, exclude) - } - if raw, ok := props[payloadKey].(string); ok { - props[payloadKey] = l.truncatePayload(raw) - } - - if gate { - if flow == nil || !flow.Headers { - delete(props, headersKey) - } - if flow == nil || !flow.Payload { - delete(props, payloadKey) +// parseHeadersFromString converts the JSON-encoded header value stored in +// event.Properties (a map[string]string or map[string][]string serialized by the +// ext_proc layer) into a map[string]string so it embeds as a plain JSON object +// in the log line. Other publishers (e.g. Moesif) read the raw string directly; +// the Log publisher calls this only on the local TrafficLogEvent it builds, so +// the shared event is never modified. Multi-value headers are flattened to their +// first value. Returns nil on empty input or parse failure. +func parseHeadersFromString(raw string) map[string]string { + if raw == "" { + return nil + } + var single map[string]string + if err := json.Unmarshal([]byte(raw), &single); err == nil { + return single + } + // Fallback: multi-value wire format — flatten to first value. + var multi map[string][]string + if err := json.Unmarshal([]byte(raw), &multi); err == nil { + out := make(map[string]string, len(multi)) + for k, vs := range multi { + if len(vs) > 0 { + out[k] = vs[0] + } } + return out } + return nil } // truncatePayload returns up to maxPayloadSize bytes of the payload (0 = no @@ -178,106 +153,73 @@ func (l *Log) truncatePayload(s string) string { return s[:l.maxPayloadSize] } -// applyFieldsProjection restricts the serialized event JSON to the configured -// fields. Names are top-level keys (e.g. "latencies") or dotted property paths -// (e.g. "properties.requestHeaders"). Mode "exclude" drops the named keys; any -// other value (default "include") keeps only the named keys. Naming the whole -// "properties" key keeps all of its subkeys. -func applyFieldsProjection(data []byte, fields *dto.TrafficLogFields) ([]byte, error) { - var m map[string]interface{} - if err := json.Unmarshal(data, &m); err != nil { - return nil, err - } - - topNames := make(map[string]bool) - propNames := make(map[string]bool) - propsReferenced := false - for _, name := range fields.Names { - if sub, ok := strings.CutPrefix(name, "properties."); ok { - if sub != "" { - propNames[sub] = true - propsReferenced = true +// applyFieldsProjection mutates m in place to restrict it to the configured +// fields. Names are top-level keys (e.g. "latencies", "requestHeaders") or +// dotted sub-key paths within map fields (e.g. "requestHeaders.authorization", +// "labels.env"). Only keeps exactly the named fields; Exclude drops the named +// fields and keeps everything else. If both are set, Only takes precedence. +func applyFieldsProjection(m map[string]interface{}, fields *dto.TrafficLogFields) { + if len(fields.Only) > 0 { + directKeys := make(map[string]bool) + subKeys := make(map[string][]string) // topKey → sub-keys to keep + for _, name := range fields.Only { + if top, sub, found := strings.Cut(name, "."); found { + subKeys[top] = append(subKeys[top], sub) + } else { + directKeys[name] = true } - continue - } - if name == "properties" { - propsReferenced = true } - if name != "" { - topNames[name] = true - } - } - - props, _ := m["properties"].(map[string]interface{}) - - if strings.EqualFold(fields.Mode, "exclude") { - for name := range topNames { - delete(m, name) - } - for sub := range propNames { - delete(props, sub) - } - if props != nil && len(props) == 0 { - delete(m, "properties") + for key := range m { + if !directKeys[key] && subKeys[key] == nil { + delete(m, key) + } } - return json.Marshal(m) - } - - // include (default): keep only referenced top-level keys. - for key := range m { - keep := topNames[key] || (key == "properties" && propsReferenced) - if !keep { - delete(m, key) + for top, subs := range subKeys { + if directKeys[top] { + continue // whole key kept; don't filter sub-keys + } + if nested, ok := m[top].(map[string]interface{}); ok { + keep := make(map[string]bool, len(subs)) + for _, s := range subs { + keep[s] = true + } + for k := range nested { + if !keep[k] { + delete(nested, k) + } + } + if len(nested) == 0 { + delete(m, top) + } + } } + return } - // When specific property subkeys were named (and not the whole "properties"), - // keep only those subkeys. - if props != nil && len(propNames) > 0 && !topNames["properties"] { - for sub := range props { - if !propNames[sub] { - delete(props, sub) + for _, name := range fields.Exclude { + if top, sub, found := strings.Cut(name, "."); found { + if nested, ok := m[top].(map[string]interface{}); ok { + delete(nested, sub) + if len(nested) == 0 { + delete(m, top) + } } - } - if len(props) == 0 { - delete(m, "properties") + } else { + delete(m, name) } } - return json.Marshal(m) } -// filterHeaders parses a JSON header map, drops any per-flow excluded headers, -// and redacts the values of globally masked headers (both case-insensitive). The -// raw string is returned unchanged if it is empty or not valid JSON. -func (l *Log) filterHeaders(raw string, excludeHeaders []string) string { - if raw == "" { - return raw - } - var headers map[string]interface{} - if err := json.Unmarshal([]byte(raw), &headers); err != nil { - return raw - } - - excluded := make(map[string]bool, len(excludeHeaders)) - for _, h := range excludeHeaders { - if h = strings.ToLower(strings.TrimSpace(h)); h != "" { - excluded[h] = true - } - } - - for name := range headers { - lower := strings.ToLower(name) - if excluded[lower] { - delete(headers, name) - continue - } - if l.maskedHeaders[lower] { - headers[name] = maskedHeaderValue +// maskHeaders redacts header values whose names appear in mask (case-insensitive). +// Returns a new map; the input is not modified. Per-header exclusion is handled +// downstream by applyFieldsProjection via dotted paths (e.g. "requestHeaders.authorization"). +func (l *Log) maskHeaders(headers map[string]string, mask map[string]bool) map[string]string { + result := make(map[string]string, len(headers)) + for name, value := range headers { + if mask[strings.ToLower(name)] { + result[name] = maskedHeaderValue + } else { + result[name] = value } } - - out, err := json.Marshal(headers) - if err != nil { - return raw - } - return string(out) + return result } diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go index a912257c18..fa4fb8c5e0 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go @@ -58,13 +58,21 @@ func bothFlows() *dto.TrafficLogDirective { } } -// decodeProps runs the single JSON line and returns the decoded event + its properties. -func decodeLine(t *testing.T, out string) (map[string]interface{}, map[string]interface{}) { +// decodeLine parses the single JSON line emitted by the Log publisher. +func decodeLine(t *testing.T, out string) map[string]interface{} { t.Helper() var decoded map[string]interface{} require.NoError(t, json.Unmarshal([]byte(out), &decoded)) - props, _ := decoded["properties"].(map[string]interface{}) - return decoded, props + return decoded +} + +// headerMap returns the JSON-decoded header object as map[string]interface{} for +// easy key/value assertions. Fails the test if v is not the expected type. +func headerMap(t *testing.T, v interface{}) map[string]interface{} { + t.Helper() + m, ok := v.(map[string]interface{}) + require.True(t, ok, "expected header object (map[string]interface{}), got %T", v) + return m } func TestNewLog_NilConfig(t *testing.T) { @@ -96,10 +104,11 @@ func TestLog_Publish_WritesJSONLineWithLatencies(t *testing.T) { // Single line (one trailing newline). assert.Equal(t, 1, strings.Count(strings.TrimRight(out, "\n"), "\n")+1) - decoded, props := decodeLine(t, out) + decoded := decodeLine(t, out) api := decoded["api"].(map[string]interface{}) - assert.Equal(t, "test-api", api["apiName"]) - assert.Equal(t, `{"x-foo":"bar"}`, props["requestHeaders"]) + assert.Equal(t, "test-api", api["name"]) + reqH := headerMap(t, decoded["requestHeaders"]) + assert.Equal(t, "bar", reqH["x-foo"]) // ALS-derived latencies are always present in the line — the key improvement // over the inline log-message policy, which could never see them. @@ -107,13 +116,13 @@ func TestLog_Publish_WritesJSONLineWithLatencies(t *testing.T) { assert.Equal(t, float64(100), latencies["responseLatency"]) } -// Custom properties from the directive are emitted under properties.custom. -func TestLog_Publish_CustomPropertiesUnderCustom(t *testing.T) { +// Labels from the directive are emitted as a top-level "labels" object. +func TestLog_Publish_LabelsTopLevel(t *testing.T) { l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) event := createBaseEvent() event.TrafficLog = &dto.TrafficLogDirective{ Request: &dto.TrafficLogFlow{Headers: true}, - Properties: map[string]interface{}{ + Labels: map[string]interface{}{ "who": "alice", "authType": "jwt", "retryCount": float64(3), @@ -123,48 +132,47 @@ func TestLog_Publish_CustomPropertiesUnderCustom(t *testing.T) { l.Publish(event) - _, props := decodeLine(t, read()) - custom, ok := props["custom"].(map[string]interface{}) - require.True(t, ok, "expected properties.custom object, got %v", props["custom"]) - assert.Equal(t, "alice", custom["who"]) - assert.Equal(t, "jwt", custom["authType"]) - assert.Equal(t, float64(3), custom["retryCount"]) - // Reserved keys are untouched by the custom namespace. - assert.Equal(t, `{"x-foo":"bar"}`, props["requestHeaders"]) + decoded := decodeLine(t, read()) + labels, ok := decoded["labels"].(map[string]interface{}) + require.True(t, ok, "expected top-level labels object, got %T", decoded["labels"]) + assert.Equal(t, "alice", labels["who"]) + assert.Equal(t, "jwt", labels["authType"]) + assert.Equal(t, float64(3), labels["retryCount"]) + reqH := headerMap(t, decoded["requestHeaders"]) + assert.Equal(t, "bar", reqH["x-foo"]) } -// A directive with no Properties emits no custom key. -func TestLog_Publish_NoCustomWhenAbsent(t *testing.T) { +// A directive with no labels emits no "labels" key. +func TestLog_Publish_NoLabelsWhenAbsent(t *testing.T) { l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) event := createBaseEvent() event.TrafficLog = bothFlows() l.Publish(event) - _, props := decodeLine(t, read()) - _, present := props["custom"] - assert.False(t, present, "no custom key expected when directive has no Properties") + decoded := decodeLine(t, read()) + _, present := decoded["labels"] + assert.False(t, present, "no labels key expected when directive has no labels") } -// The fields projection can select properties.custom like any other property path. -func TestLog_Publish_CustomProjectableViaFields(t *testing.T) { +// The fields projection can select "labels" like any other top-level key. +func TestLog_Publish_LabelsProjectableViaFields(t *testing.T) { l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) event := createBaseEvent() event.TrafficLog = &dto.TrafficLogDirective{ - Properties: map[string]interface{}{"who": "alice"}, - Fields: &dto.TrafficLogFields{Mode: "include", Names: []string{"properties.custom"}}, + Labels: map[string]interface{}{"who": "alice"}, + Fields: &dto.TrafficLogFields{Only: []string{"labels"}}, } event.Properties["requestHeaders"] = `{"x-foo":"bar"}` l.Publish(event) - _, props := decodeLine(t, read()) - custom, ok := props["custom"].(map[string]interface{}) - require.True(t, ok, "expected properties.custom retained by include projection") - assert.Equal(t, "alice", custom["who"]) - // Non-selected property dropped by the include projection. - _, hasHeaders := props["requestHeaders"] - assert.False(t, hasHeaders, "requestHeaders should be dropped by include projection") + decoded := decodeLine(t, read()) + labels, ok := decoded["labels"].(map[string]interface{}) + require.True(t, ok, "expected labels retained by include projection") + assert.Equal(t, "alice", labels["who"]) + _, hasHeaders := decoded["requestHeaders"] + assert.False(t, hasHeaders, "requestHeaders not in Only list -> dropped") } func TestLog_Publish_MasksHeaders(t *testing.T) { @@ -176,32 +184,67 @@ func TestLog_Publish_MasksHeaders(t *testing.T) { l.Publish(event) - _, props := decodeLine(t, read()) - - var reqH map[string]interface{} - require.NoError(t, json.Unmarshal([]byte(props["requestHeaders"].(string)), &reqH)) + decoded := decodeLine(t, read()) + reqH := headerMap(t, decoded["requestHeaders"]) assert.Equal(t, "****", reqH["Authorization"]) // masked assert.Equal(t, "bar", reqH["x-foo"]) // untouched - var resH map[string]interface{} - require.NoError(t, json.Unmarshal([]byte(props["responseHeaders"].(string)), &resH)) + resH := headerMap(t, decoded["responseHeaders"]) assert.Equal(t, "****", resH["authorization"]) // case-insensitive match } -// Per-API excludeHeaders drops the header entirely (vs global masking which redacts). +// Per-API maskedHeaders are merged with the global config mask; either source alone redacts. +func TestLog_Publish_PerAPIMaskedHeadersMergedWithGlobal(t *testing.T) { + // Global config masks "authorization"; per-API directive adds "x-secret". + l, read := newLogToFile(t, &config.TrafficLoggingConfig{MaskedHeaders: []string{"authorization"}}) + event := createBaseEvent() + event.TrafficLog = &dto.TrafficLogDirective{ + Request: &dto.TrafficLogFlow{Headers: true}, + MaskedHeaders: []string{"x-secret"}, + } + event.Properties["requestHeaders"] = `{"Authorization":"Bearer s","X-Secret":"top","x-foo":"bar"}` + + l.Publish(event) + + decoded := decodeLine(t, read()) + reqH := headerMap(t, decoded["requestHeaders"]) + assert.Equal(t, "****", reqH["Authorization"], "global masked header still redacted") + assert.Equal(t, "****", reqH["X-Secret"], "per-API masked header redacted") + assert.Equal(t, "bar", reqH["x-foo"], "unmasked header unchanged") +} + +// Per-API maskedHeaders work even when no global headers are configured. +func TestLog_Publish_PerAPIMaskedHeadersNoGlobal(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.TrafficLog = &dto.TrafficLogDirective{ + Request: &dto.TrafficLogFlow{Headers: true}, + MaskedHeaders: []string{"X-Token"}, + } + event.Properties["requestHeaders"] = `{"X-Token":"secret","x-foo":"bar"}` + + l.Publish(event) + + decoded := decodeLine(t, read()) + reqH := headerMap(t, decoded["requestHeaders"]) + assert.Equal(t, "****", reqH["X-Token"]) + assert.Equal(t, "bar", reqH["x-foo"]) +} + +// fields.exclude with a dotted path drops a specific header entirely (vs global masking which redacts). func TestLog_Publish_ExcludeHeadersDrops(t *testing.T) { l, read := newLogToFile(t, &config.TrafficLoggingConfig{MaskedHeaders: []string{"authorization"}}) event := createBaseEvent() event.TrafficLog = &dto.TrafficLogDirective{ - Request: &dto.TrafficLogFlow{Headers: true, ExcludeHeaders: []string{"X-Secret"}}, + Request: &dto.TrafficLogFlow{Headers: true}, + Fields: &dto.TrafficLogFields{Exclude: []string{"requestHeaders.X-Secret"}}, } event.Properties["requestHeaders"] = `{"Authorization":"Bearer s","X-Secret":"top","x-foo":"bar"}` l.Publish(event) - _, props := decodeLine(t, read()) - var reqH map[string]interface{} - require.NoError(t, json.Unmarshal([]byte(props["requestHeaders"].(string)), &reqH)) + decoded := decodeLine(t, read()) + reqH := headerMap(t, decoded["requestHeaders"]) _, hasSecret := reqH["X-Secret"] assert.False(t, hasSecret, "excluded header must be dropped") @@ -224,13 +267,13 @@ func TestLog_Publish_DisabledFieldsOmitted(t *testing.T) { l.Publish(event) - _, props := decodeLine(t, read()) - _, hasReqHeaders := props["requestHeaders"] + decoded := decodeLine(t, read()) + _, hasReqHeaders := decoded["requestHeaders"] assert.False(t, hasReqHeaders, "request headers disabled -> omitted") - assert.Equal(t, "req-body", props["request_payload"], "request payload enabled -> kept") + assert.Equal(t, "req-body", decoded["requestBody"], "request payload enabled -> kept") - _, hasRespHeaders := props["responseHeaders"] - _, hasRespPayload := props["response_payload"] + _, hasRespHeaders := decoded["responseHeaders"] + _, hasRespPayload := decoded["responseBody"] assert.False(t, hasRespHeaders, "nil response flow -> response headers omitted") assert.False(t, hasRespPayload, "nil response flow -> response payload omitted") } @@ -254,39 +297,33 @@ func TestLog_Publish_NilEvent(t *testing.T) { assert.Empty(t, read()) } -// Field selection (include): only named top-level keys and properties.* survive, -// and it is authoritative over presence (request.headers boolean is ignored, but -// excludeHeaders + masking still apply). +// Field selection (include): only named top-level keys survive; fields.only is +// authoritative over presence (request.headers boolean is ignored, masking still applies). func TestLog_Publish_FieldsInclude(t *testing.T) { l, read := newLogToFile(t, &config.TrafficLoggingConfig{MaskedHeaders: []string{"authorization"}}) event := createBaseEvent() - event.Properties["requestHeaders"] = `{"Authorization":"Bearer s","X-Drop":"d","X-Keep":"k"}` + event.Properties["requestHeaders"] = `{"Authorization":"Bearer s","X-Keep":"k"}` event.Properties["responseHeaders"] = `{"x-bar":"baz"}` event.TrafficLog = &dto.TrafficLogDirective{ - Request: &dto.TrafficLogFlow{Headers: false, ExcludeHeaders: []string{"X-Drop"}}, // boolean ignored - Fields: &dto.TrafficLogFields{Mode: "include", Names: []string{"latencies", "properties.requestHeaders"}}, + Request: &dto.TrafficLogFlow{Headers: false}, // boolean ignored when fields set + Fields: &dto.TrafficLogFields{Only: []string{"latencies", "requestHeaders"}}, } l.Publish(event) - decoded, props := decodeLine(t, read()) + decoded := decodeLine(t, read()) _, hasAPI := decoded["api"] _, hasOp := decoded["operation"] assert.False(t, hasAPI, "api not listed -> dropped") assert.False(t, hasOp, "operation not listed -> dropped") assert.Contains(t, decoded, "latencies", "latencies listed -> kept") - require.NotNil(t, props, "properties kept (a properties.* path was listed)") - _, hasResp := props["responseHeaders"] + _, hasResp := decoded["responseHeaders"] assert.False(t, hasResp, "responseHeaders not listed -> dropped") - reqRaw, ok := props["requestHeaders"].(string) - require.True(t, ok, "requestHeaders present (fields authoritative, boolean ignored)") - var reqH map[string]interface{} - require.NoError(t, json.Unmarshal([]byte(reqRaw), &reqH)) + require.NotNil(t, decoded["requestHeaders"], "requestHeaders present (fields authoritative, boolean ignored)") + reqH := headerMap(t, decoded["requestHeaders"]) assert.Equal(t, "****", reqH["Authorization"], "masking still applies") - _, hasDrop := reqH["X-Drop"] - assert.False(t, hasDrop, "excludeHeaders still applies") assert.Equal(t, "k", reqH["X-Keep"]) } @@ -297,40 +334,44 @@ func TestLog_Publish_FieldsExclude(t *testing.T) { event.Properties["requestHeaders"] = `{"x-foo":"bar"}` event.Properties["request_payload"] = "secret-body" event.TrafficLog = &dto.TrafficLogDirective{ - Fields: &dto.TrafficLogFields{Mode: "exclude", Names: []string{"operation", "properties.request_payload"}}, + Fields: &dto.TrafficLogFields{Exclude: []string{"operation", "requestBody"}}, } l.Publish(event) - decoded, props := decodeLine(t, read()) + decoded := decodeLine(t, read()) _, hasOp := decoded["operation"] assert.False(t, hasOp, "operation excluded") assert.Contains(t, decoded, "api", "api kept") assert.Contains(t, decoded, "latencies", "latencies kept") - require.NotNil(t, props) - _, hasPayload := props["request_payload"] - assert.False(t, hasPayload, "request_payload excluded") - assert.Contains(t, props, "requestHeaders", "requestHeaders kept (not excluded)") + _, hasPayload := decoded["requestBody"] + assert.False(t, hasPayload, "requestBody excluded") + assert.Contains(t, decoded, "requestHeaders", "requestHeaders kept (not excluded)") } -// Naming the whole "properties" key keeps all of its subkeys. -func TestLog_Publish_FieldsIncludeWholeProperties(t *testing.T) { +// requestBody and labels are top-level keys like any other and can be selected +// explicitly via fields.only. +func TestLog_Publish_FieldsIncludeRequestBodyAndLabels(t *testing.T) { l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) event := createBaseEvent() event.Properties["requestHeaders"] = `{"x-foo":"bar"}` - event.Properties["responseHeaders"] = `{"x-bar":"baz"}` + event.Properties["request_payload"] = "body-data" event.TrafficLog = &dto.TrafficLogDirective{ - Fields: &dto.TrafficLogFields{Mode: "include", Names: []string{"properties"}}, + Labels: map[string]interface{}{"env": "prod"}, + Fields: &dto.TrafficLogFields{Only: []string{"requestBody", "labels"}}, } l.Publish(event) - decoded, props := decodeLine(t, read()) + decoded := decodeLine(t, read()) _, hasAPI := decoded["api"] assert.False(t, hasAPI, "api not listed -> dropped") - require.NotNil(t, props) - assert.Contains(t, props, "requestHeaders") - assert.Contains(t, props, "responseHeaders") + _, hasHeaders := decoded["requestHeaders"] + assert.False(t, hasHeaders, "requestHeaders not in Only list -> dropped") + assert.Equal(t, "body-data", decoded["requestBody"]) + labels, ok := decoded["labels"].(map[string]interface{}) + require.True(t, ok) + assert.Equal(t, "prod", labels["env"]) } // Output-side payload truncation (0 = no limit). @@ -342,9 +383,9 @@ func TestLog_Publish_TruncatesPayload(t *testing.T) { event.Properties["response_payload"] = "goodbye world" l.Publish(event) - _, props := decodeLine(t, read()) - assert.Equal(t, "hello", props["request_payload"]) - assert.Equal(t, "goodb", props["response_payload"]) + decoded := decodeLine(t, read()) + assert.Equal(t, "hello", decoded["requestBody"]) + assert.Equal(t, "goodb", decoded["responseBody"]) } func TestLog_Publish_NoTruncationWhenZero(t *testing.T) { @@ -354,11 +395,11 @@ func TestLog_Publish_NoTruncationWhenZero(t *testing.T) { event.Properties["request_payload"] = "hello world" l.Publish(event) - _, props := decodeLine(t, read()) - assert.Equal(t, "hello world", props["request_payload"]) + decoded := decodeLine(t, read()) + assert.Equal(t, "hello world", decoded["requestBody"]) } -func TestLog_Publish_InvalidHeaderJSONLeftAsIs(t *testing.T) { +func TestLog_Publish_UnparseableHeadersDropped(t *testing.T) { l, read := newLogToFile(t, &config.TrafficLoggingConfig{MaskedHeaders: []string{"authorization"}}) event := createBaseEvent() event.TrafficLog = bothFlows() @@ -366,6 +407,62 @@ func TestLog_Publish_InvalidHeaderJSONLeftAsIs(t *testing.T) { l.Publish(event) - _, props := decodeLine(t, read()) - assert.Equal(t, "not-json", props["requestHeaders"]) + decoded := decodeLine(t, read()) + _, hasHeaders := decoded["requestHeaders"] + assert.False(t, hasHeaders, "unparseable header value must be silently dropped") +} + +// Application is omitted entirely for unauthenticated requests (all fields empty). +func TestLog_Publish_UnauthenticatedRequestOmitsApplication(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.Application = &dto.Application{} // all fields are "" + event.TrafficLog = bothFlows() + + l.Publish(event) + + decoded := decodeLine(t, read()) + _, hasApp := decoded["application"] + assert.False(t, hasApp, "application with all-empty fields must be absent") +} + +// TrafficLogAPI uses clean field names (id/name/kind) not the Moesif apiId/apiName/apiType. +func TestLog_Publish_APIFieldNames(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.TrafficLog = bothFlows() + + l.Publish(event) + + decoded := decodeLine(t, read()) + api, ok := decoded["api"].(map[string]interface{}) + require.True(t, ok, "api object must be present") + assert.Equal(t, "api-123", api["id"]) + assert.Equal(t, "test-api", api["name"]) + assert.Equal(t, "v1.0", api["version"]) + assert.Equal(t, "Rest", api["kind"]) + assert.Equal(t, "/test", api["context"]) + assert.Equal(t, "project-123", api["projectId"]) + // Moesif-specific keys must not bleed into traffic logs. + for _, moesifKey := range []string{"apiId", "apiName", "apiCreator", "apiCreatorTenantDomain", "organizationId"} { + _, present := api[moesifKey] + assert.False(t, present, "Moesif field %q must not appear in traffic log", moesifKey) + } +} + +// Top-level fields — status, correlationId, client — are always present when set. +func TestLog_Publish_TopLevelFieldsPresent(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.TrafficLog = bothFlows() + + l.Publish(event) + + decoded := decodeLine(t, read()) + assert.Equal(t, float64(200), decoded["status"], "proxy response code must appear as status") + assert.Equal(t, "corr-123", decoded["correlationId"]) + client, ok := decoded["client"].(map[string]interface{}) + require.True(t, ok, "client object must be present") + assert.Equal(t, "192.168.1.1", client["ip"]) + assert.Equal(t, "test-agent", client["userAgent"]) } diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/traffic_log_event.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/traffic_log_event.go new file mode 100644 index 0000000000..da6b251a70 --- /dev/null +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/traffic_log_event.go @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package publishers + +import ( + "strings" + + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/analytics/dto" +) + +// trafficLogTimestampFormat is RFC 3339 with millisecond precision. +const trafficLogTimestampFormat = "2006-01-02T15:04:05.000Z07:00" + +// TrafficLogEvent is the JSON shape written to stdout by the Log publisher. +// It is intentionally separate from dto.Event (shaped for Moesif) so its field +// names, schema, and presence rules can evolve independently. All string fields +// carry omitempty so absent or unknown values produce no key rather than "". +type TrafficLogEvent struct { + Timestamp string `json:"timestamp,omitempty"` + CorrelationID string `json:"correlationId,omitempty"` + Status int `json:"status,omitempty"` + API *TrafficLogAPI `json:"api,omitempty"` + Operation *TrafficLogOperation `json:"operation,omitempty"` + Target *TrafficLogTarget `json:"target,omitempty"` + Application *TrafficLogApplication `json:"application,omitempty"` + Client *TrafficLogClient `json:"client,omitempty"` + Latencies *dto.Latencies `json:"latencies,omitempty"` + RequestHeaders map[string]string `json:"requestHeaders,omitempty"` + ResponseHeaders map[string]string `json:"responseHeaders,omitempty"` + RequestBody string `json:"requestBody,omitempty"` + ResponseBody string `json:"responseBody,omitempty"` + Labels map[string]interface{} `json:"labels,omitempty"` +} + +// TrafficLogAPI identifies the API that processed the request. +type TrafficLogAPI struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Version string `json:"version,omitempty"` + Context string `json:"context,omitempty"` + Kind string `json:"kind,omitempty"` + ProjectID string `json:"projectId,omitempty"` +} + +// TrafficLogOperation describes the matched operation within the API. +type TrafficLogOperation struct { + Method string `json:"method,omitempty"` + Path string `json:"path,omitempty"` +} + +// TrafficLogTarget holds upstream response information. +type TrafficLogTarget struct { + StatusCode int `json:"statusCode,omitempty"` + Destination string `json:"destination,omitempty"` +} + +// TrafficLogApplication is present only for authenticated requests. +type TrafficLogApplication struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Owner string `json:"owner,omitempty"` + KeyType string `json:"keyType,omitempty"` +} + +// TrafficLogClient holds downstream caller information. +type TrafficLogClient struct { + IP string `json:"ip,omitempty"` + UserAgent string `json:"userAgent,omitempty"` +} + +// toTrafficLogEvent translates a dto.Event and its traffic-log directive into the +// traffic-log-specific output shape, applying per-flow header filtering, global +// header masking, and payload truncation. +func (l *Log) toTrafficLogEvent(event *dto.Event, dir *dto.TrafficLogDirective) *TrafficLogEvent { + tl := &TrafficLogEvent{ + Status: event.ProxyResponseCode, + Latencies: event.Latencies, + } + + if !event.RequestTimestamp.IsZero() { + tl.Timestamp = event.RequestTimestamp.UTC().Format(trafficLogTimestampFormat) + } + + if event.MetaInfo != nil { + tl.CorrelationID = event.MetaInfo.CorrelationID + } + + if event.API != nil { + tl.API = &TrafficLogAPI{ + ID: event.API.APIID, + Name: event.API.APIName, + Version: event.API.APIVersion, + Context: event.API.APIContext, + Kind: event.API.APIType, + ProjectID: event.API.ProjectID, + } + } + + if event.Operation != nil { + tl.Operation = &TrafficLogOperation{ + Method: event.Operation.APIMethod, + Path: event.Operation.APIResourceTemplate, + } + } + + if event.Target != nil { + tl.Target = &TrafficLogTarget{ + StatusCode: event.Target.TargetResponseCode, + Destination: event.Target.Destination, + } + } + + // Application is only meaningful for authenticated requests. + if a := event.Application; a != nil && (a.ApplicationID != "" || a.ApplicationName != "") { + tl.Application = &TrafficLogApplication{ + ID: a.ApplicationID, + Name: a.ApplicationName, + Owner: a.ApplicationOwner, + KeyType: a.KeyType, + } + } + + if event.UserIP != "" || event.UserAgentHeader != "" { + tl.Client = &TrafficLogClient{ + IP: event.UserIP, + UserAgent: event.UserAgentHeader, + } + } + + // When a fields selection is configured it is authoritative over presence: + // the per-flow Headers/Payload booleans are ignored and applyFieldsProjection + // in Publish decides what survives. Global masking always applies; per-API + // maskedHeaders are merged in here and passed down. + hasFieldsSelection := dir.Fields != nil && (len(dir.Fields.Only) > 0 || len(dir.Fields.Exclude) > 0) + + // Effective header mask: global config merged with any per-API additions. + mask := l.maskedHeaders + if len(dir.MaskedHeaders) > 0 { + merged := make(map[string]bool, len(l.maskedHeaders)+len(dir.MaskedHeaders)) + for k, v := range l.maskedHeaders { + merged[k] = v + } + for _, h := range dir.MaskedHeaders { + merged[strings.ToLower(h)] = true + } + mask = merged + } + + // Request flow + if raw, ok := event.Properties[dto.PropKeyRequestHeaders].(string); ok { + if hasFieldsSelection || (dir.Request != nil && dir.Request.Headers) { + if headers := parseHeadersFromString(raw); headers != nil { + tl.RequestHeaders = l.maskHeaders(headers, mask) + } + } + } + if p, ok := event.Properties[dto.PropKeyRequestPayload].(string); ok && p != "" { + if hasFieldsSelection || (dir.Request != nil && dir.Request.Payload) { + tl.RequestBody = l.truncatePayload(p) + } + } + + // Response flow + if raw, ok := event.Properties[dto.PropKeyResponseHeaders].(string); ok { + if hasFieldsSelection || (dir.Response != nil && dir.Response.Headers) { + if headers := parseHeadersFromString(raw); headers != nil { + tl.ResponseHeaders = l.maskHeaders(headers, mask) + } + } + } + if p, ok := event.Properties[dto.PropKeyResponsePayload].(string); ok && p != "" { + if hasFieldsSelection || (dir.Response != nil && dir.Response.Payload) { + tl.ResponseBody = l.truncatePayload(p) + } + } + + if len(dir.Labels) > 0 { + tl.Labels = dir.Labels + } + + return tl +} From b3450a0eb22eaf688c3bf3d77f1401cf257fe43a Mon Sep 17 00:00:00 2001 From: Dineth Date: Tue, 7 Jul 2026 00:54:03 +0530 Subject: [PATCH 05/21] Disable request and response header capture by default; update related tests and migration logic --- gateway/configs/config-template.toml | 9 ++++---- gateway/configs/config.toml | 6 ++++-- .../gateway-controller/pkg/config/config.go | 17 +++++++++++---- .../pkg/config/config_test.go | 17 +++++++++++++++ .../pkg/utils/system_policies_test.go | 21 ++++++++++++++++++- .../policy-engine/internal/config/config.go | 13 ++++++++++-- .../internal/config/config_test.go | 17 +++++++++++++++ 7 files changed, 87 insertions(+), 13 deletions(-) diff --git a/gateway/configs/config-template.toml b/gateway/configs/config-template.toml index ae83145e3a..936475b881 100644 --- a/gateway/configs/config-template.toml +++ b/gateway/configs/config-template.toml @@ -297,10 +297,11 @@ send_request_body = false send_response_body = false # Capture ALL request/response headers into the collected event (request_headers/ # response_headers). Applies to every API via the collector system policy, so no -# per-API header policy is needed. Sensitive values are redacted on output by each -# consumer (e.g. traffic_logging.masked_headers). On by default. -send_request_headers = true -send_response_headers = true +# per-API header policy is needed. Sensitive values are redacted on output by +# consumers that support it (e.g. traffic_logging.masked_headers) — the Moesif +# publisher does not mask headers. Off by default; enable deliberately per consumer. +send_request_headers = false +send_response_headers = false # ALS transport tuning (advanced; defaults are sensible). One section read by BOTH # ends of the Envoy → policy-engine access-log stream: diff --git a/gateway/configs/config.toml b/gateway/configs/config.toml index 83107adfa6..fbabb75d6d 100644 --- a/gateway/configs/config.toml +++ b/gateway/configs/config.toml @@ -8,8 +8,10 @@ send_request_body = false send_response_body = false # Capture ALL request/response headers (redacted on output by per-consumer masking). -send_request_headers = true -send_response_headers = true +# Off by default; the Moesif publisher does not mask headers, so only enable this +# for consumers you know redact sensitive values (e.g. traffic_logging.masked_headers). +send_request_headers = false +send_response_headers = false # Analytics consumer (e.g. Moesif). Enabling it activates the collector automatically. [analytics] diff --git a/gateway/gateway-controller/pkg/config/config.go b/gateway/gateway-controller/pkg/config/config.go index 2530e073d0..bc4bb7cfd0 100644 --- a/gateway/gateway-controller/pkg/config/config.go +++ b/gateway/gateway-controller/pkg/config/config.go @@ -976,8 +976,8 @@ func defaultConfig() *Config { Collector: CollectorConfig{ SendRequestBody: false, SendResponseBody: false, - SendRequestHeaders: true, - SendResponseHeaders: true, + SendRequestHeaders: false, + SendResponseHeaders: false, GRPCEventServerCfg: defaultGRPCEventServerConfig(), }, TracingConfig: TracingConfig{ @@ -1781,9 +1781,11 @@ func (c *Config) IsCollectorEnabled() bool { func (c *Config) migrateDeprecatedAnalyticsTransport() { def := defaultGRPCEventServerConfig() if c.Analytics.GRPCEventServerCfg != def { - slog.Warn("analytics.grpc_event_server is deprecated; use collector.als instead") if c.Collector.GRPCEventServerCfg == def { + slog.Warn("analytics.grpc_event_server is deprecated; migrating it to collector.als") c.Collector.GRPCEventServerCfg = c.Analytics.GRPCEventServerCfg + } else { + slog.Warn("analytics.grpc_event_server is deprecated and collector.als is already configured; ignoring the analytics.grpc_event_server override") } } } @@ -1791,8 +1793,15 @@ func (c *Config) migrateDeprecatedAnalyticsTransport() { // migrateDeprecatedAnalyticsCapture maps the deprecated analytics.allow_payloads / // analytics.send_request_body / analytics.send_response_body onto the collector's // body-capture flags (when the collector flag is not already set), so existing -// configs keep working after capture settings moved under [collector]. +// configs keep working after capture settings moved under [collector]. These are +// analytics's own deprecated flags, so they are only honored while analytics is +// enabled — otherwise a stale value left over from a disabled analytics setup +// could silently turn on body capture for an unrelated consumer (e.g. +// traffic_logging) enabled later. func (c *Config) migrateDeprecatedAnalyticsCapture() { + if !c.Analytics.Enabled { + return + } // Directional aliases take precedence over allow_payloads. if c.Analytics.SendRequestBody && !c.Collector.SendRequestBody { slog.Warn("analytics.send_request_body is deprecated; use collector.send_request_body instead") diff --git a/gateway/gateway-controller/pkg/config/config_test.go b/gateway/gateway-controller/pkg/config/config_test.go index 4b237ac386..7c4b117be9 100644 --- a/gateway/gateway-controller/pkg/config/config_test.go +++ b/gateway/gateway-controller/pkg/config/config_test.go @@ -1413,6 +1413,23 @@ func TestConfig_ValidateAnalyticsPayloadMigration(t *testing.T) { } } +// TestConfig_ValidateAnalyticsPayloadMigration_SkippedWhenAnalyticsDisabled guards +// against a stale analytics.allow_payloads left over from a disabled analytics +// setup silently turning on body capture for an unrelated consumer (traffic +// logging) enabled later. The deprecated capture aliases belong to analytics, so +// they must only be honored while analytics itself is enabled. +func TestConfig_ValidateAnalyticsPayloadMigration_SkippedWhenAnalyticsDisabled(t *testing.T) { + cfg := validConfig() + cfg.Analytics.Enabled = false + cfg.TrafficLogging.Enabled = true // an unrelated consumer activates the collector + cfg.Analytics.AllowPayloads = true + + err := cfg.Validate() + require.NoError(t, err) + assert.False(t, cfg.Collector.SendRequestBody) + assert.False(t, cfg.Collector.SendResponseBody) +} + func TestConfig_ValidateAuthConfig(t *testing.T) { tests := []struct { name string diff --git a/gateway/gateway-controller/pkg/utils/system_policies_test.go b/gateway/gateway-controller/pkg/utils/system_policies_test.go index 5f38555594..8ed2e284cb 100644 --- a/gateway/gateway-controller/pkg/utils/system_policies_test.go +++ b/gateway/gateway-controller/pkg/utils/system_policies_test.go @@ -155,6 +155,25 @@ func TestInjectSystemPolicies_CollectorEnabled(t *testing.T) { assert.Equal(t, "existing", result[1].Name) } +func TestInjectSystemPolicies_TrafficLoggingOnlyEnablesCollector(t *testing.T) { + // Traffic logging alone (no analytics) -> collector implicitly on -> system + // policy injected. Guards against a regression that gates injection on + // cfg.Analytics.Enabled alone instead of cfg.IsCollectorEnabled(). + cfg := &config.Config{ + TrafficLogging: config.TrafficLoggingConfig{Enabled: true}, + } + policies := []policyenginev1.PolicyInstance{ + {Name: "existing", Version: "v1.0.0"}, + } + + result := InjectSystemPolicies(policies, cfg, nil) + assert.Len(t, result, 2) + assert.Equal(t, constants.ANALYTICS_SYSTEM_POLICY_NAME, result[0].Name) + assert.Equal(t, constants.ANALYTICS_SYSTEM_POLICY_VERSION, result[0].Version) + assert.True(t, result[0].Enabled) + assert.Equal(t, "existing", result[1].Name) +} + func TestInjectSystemPolicies_BodyFlagsPropagated(t *testing.T) { cfg := &config.Config{ Analytics: config.AnalyticsConfig{Enabled: true}, @@ -204,7 +223,7 @@ func TestInjectSystemPolicies_HeaderFlagsPropagated(t *testing.T) { func TestInjectSystemPolicies_HeaderFlagsDefaultFalse(t *testing.T) { // Zero-value collector (headers unset) with a consumer on: propagation passes the - // struct's false through. (Production defaults these true; see defaultConfig.) + // struct's false through, matching the production default (see defaultConfig). cfg := &config.Config{ Analytics: config.AnalyticsConfig{Enabled: true}, } diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config.go b/gateway/gateway-runtime/policy-engine/internal/config/config.go index 4e26d0fd3e..9580c733a2 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config.go @@ -627,9 +627,11 @@ func (c *Config) IsCollectorEnabled() bool { func (c *Config) migrateDeprecatedAnalyticsTransport() { def := defaultAccessLogsServiceConfig() if c.Analytics.AccessLogsServiceCfg != def { - slog.Warn("analytics.access_logs_service is deprecated; use collector.als instead") if c.Collector.AccessLogsServiceCfg == def { + slog.Warn("analytics.access_logs_service is deprecated; migrating it to collector.als") c.Collector.AccessLogsServiceCfg = c.Analytics.AccessLogsServiceCfg + } else { + slog.Warn("analytics.access_logs_service is deprecated and collector.als is already configured; ignoring the analytics.access_logs_service override") } } } @@ -664,8 +666,15 @@ func validateAccessLogsServiceConfig(als AccessLogsServiceConfig) error { // migrateDeprecatedAnalyticsCapture maps the deprecated analytics.allow_payloads / // analytics.send_request_body / analytics.send_response_body onto the collector's // body-capture flags (when the collector flag is not already set), so existing -// configs keep working after capture settings moved under [collector]. +// configs keep working after capture settings moved under [collector]. These are +// analytics's own deprecated flags, so they are only honored while analytics is +// enabled — otherwise a stale value left over from a disabled analytics setup +// could silently turn on body capture for an unrelated consumer (e.g. +// traffic_logging) enabled later. func (c *Config) migrateDeprecatedAnalyticsCapture() { + if !c.Analytics.Enabled { + return + } // Directional aliases take precedence over allow_payloads. if c.Analytics.SendRequestBody && !c.Collector.SendRequestBody { slog.Warn("analytics.send_request_body is deprecated; use collector.send_request_body instead") diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config_test.go b/gateway/gateway-runtime/policy-engine/internal/config/config_test.go index b9371d6990..914b6853b3 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config_test.go @@ -1174,6 +1174,23 @@ func TestValidate_AnalyticsPayloadMigration(t *testing.T) { } } +// TestValidate_AnalyticsPayloadMigration_SkippedWhenAnalyticsDisabled guards +// against a stale analytics.allow_payloads left over from a disabled analytics +// setup silently turning on body capture for an unrelated consumer (traffic +// logging) enabled later. The deprecated capture aliases belong to analytics, so +// they must only be honored while analytics itself is enabled. +func TestValidate_AnalyticsPayloadMigration_SkippedWhenAnalyticsDisabled(t *testing.T) { + cfg := validConfig() + cfg.Analytics.Enabled = false + cfg.TrafficLogging.Enabled = true // an unrelated consumer activates the collector + cfg.Analytics.AllowPayloads = true + + err := cfg.Validate() + require.NoError(t, err) + assert.False(t, cfg.Collector.SendRequestBody) + assert.False(t, cfg.Collector.SendResponseBody) +} + // TestValidate_CollectorPrerequisite verifies that enabling a consumer (analytics, // traffic logging) without the collector auto-enables the collector (a // backward-compat soft prerequisite) rather than failing. From 9cde5947ac9a3479eab0bc2138d414eaa3067499 Mon Sep 17 00:00:00 2001 From: Dineth Date: Tue, 7 Jul 2026 11:10:08 +0530 Subject: [PATCH 06/21] Fix analytics transport migration handling when analytics is disabled to prevent stale configurations affecting unrelated consumers --- .../gateway-controller/pkg/config/config.go | 7 +++++++ .../pkg/config/config_test.go | 20 ++++++++++++++++++ .../policy-engine/internal/config/config.go | 7 +++++++ .../internal/config/config_test.go | 21 +++++++++++++++++++ 4 files changed, 55 insertions(+) diff --git a/gateway/gateway-controller/pkg/config/config.go b/gateway/gateway-controller/pkg/config/config.go index bc4bb7cfd0..b93e371d26 100644 --- a/gateway/gateway-controller/pkg/config/config.go +++ b/gateway/gateway-controller/pkg/config/config.go @@ -1778,7 +1778,14 @@ func (c *Config) IsCollectorEnabled() bool { // migrateDeprecatedAnalyticsTransport maps a deprecated [analytics].grpc_event_server // override onto the collector when the collector's transport tuning is still at its // default, so existing configs keep working after the transport moved to [collector]. +// This is analytics's own deprecated field, so it is only honored while analytics is +// enabled — otherwise a stale value left over from a disabled analytics setup could +// silently reconfigure the transport for an unrelated consumer (e.g. traffic_logging) +// enabled later. func (c *Config) migrateDeprecatedAnalyticsTransport() { + if !c.Analytics.Enabled { + return + } def := defaultGRPCEventServerConfig() if c.Analytics.GRPCEventServerCfg != def { if c.Collector.GRPCEventServerCfg == def { diff --git a/gateway/gateway-controller/pkg/config/config_test.go b/gateway/gateway-controller/pkg/config/config_test.go index 7c4b117be9..1368a09287 100644 --- a/gateway/gateway-controller/pkg/config/config_test.go +++ b/gateway/gateway-controller/pkg/config/config_test.go @@ -1430,6 +1430,26 @@ func TestConfig_ValidateAnalyticsPayloadMigration_SkippedWhenAnalyticsDisabled(t assert.False(t, cfg.Collector.SendResponseBody) } +// TestConfig_ValidateAnalyticsTransportMigration_SkippedWhenAnalyticsDisabled guards +// against a stale analytics.grpc_event_server override left over from a disabled +// analytics setup silently reconfiguring the transport for an unrelated consumer +// (traffic logging) enabled later. The deprecated transport alias belongs to +// analytics, so it must only be honored while analytics itself is enabled. +func TestConfig_ValidateAnalyticsTransportMigration_SkippedWhenAnalyticsDisabled(t *testing.T) { + cfg := validConfig() + cfg.Analytics.Enabled = false + cfg.TrafficLogging.Enabled = true // an unrelated consumer activates the collector + cfg.Analytics.GRPCEventServerCfg.Mode = "uds" + cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 1000 + cfg.Analytics.GRPCEventServerCfg.BufferSizeBytes = 16384 + cfg.Analytics.GRPCEventServerCfg.GRPCRequestTimeout = 5000 + cfg.Analytics.GRPCEventServerCfg.ServerPort = 18090 + + err := cfg.Validate() + require.NoError(t, err) + assert.Equal(t, defaultGRPCEventServerConfig(), cfg.Collector.GRPCEventServerCfg) +} + func TestConfig_ValidateAuthConfig(t *testing.T) { tests := []struct { name string diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config.go b/gateway/gateway-runtime/policy-engine/internal/config/config.go index 9580c733a2..0ee95ac8e1 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config.go @@ -624,7 +624,14 @@ func (c *Config) IsCollectorEnabled() bool { // migrateDeprecatedAnalyticsTransport maps a deprecated [analytics].access_logs_service // override onto the collector when the collector's receiver tuning is still at its // default, so existing configs keep working after the transport moved to [collector]. +// This is analytics's own deprecated field, so it is only honored while analytics is +// enabled — otherwise a stale value left over from a disabled analytics setup could +// silently reconfigure the transport for an unrelated consumer (e.g. traffic_logging) +// enabled later. func (c *Config) migrateDeprecatedAnalyticsTransport() { + if !c.Analytics.Enabled { + return + } def := defaultAccessLogsServiceConfig() if c.Analytics.AccessLogsServiceCfg != def { if c.Collector.AccessLogsServiceCfg == def { diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config_test.go b/gateway/gateway-runtime/policy-engine/internal/config/config_test.go index 914b6853b3..f26847823e 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config_test.go @@ -1191,6 +1191,27 @@ func TestValidate_AnalyticsPayloadMigration_SkippedWhenAnalyticsDisabled(t *test assert.False(t, cfg.Collector.SendResponseBody) } +// TestValidate_AnalyticsTransportMigration_SkippedWhenAnalyticsDisabled guards +// against a stale analytics.access_logs_service override left over from a disabled +// analytics setup silently reconfiguring the transport for an unrelated consumer +// (traffic logging) enabled later. The deprecated transport alias belongs to +// analytics, so it must only be honored while analytics itself is enabled. +func TestValidate_AnalyticsTransportMigration_SkippedWhenAnalyticsDisabled(t *testing.T) { + cfg := validConfig() + cfg.Analytics.Enabled = false + cfg.TrafficLogging.Enabled = true // an unrelated consumer activates the collector + cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + Mode: "uds", + ShutdownTimeout: 600 * time.Second, + ExtProcMaxMessageSize: 1000000, + ExtProcMaxHeaderLimit: 8192, + } + + err := cfg.Validate() + require.NoError(t, err) + assert.Equal(t, defaultAccessLogsServiceConfig(), cfg.Collector.AccessLogsServiceCfg) +} + // TestValidate_CollectorPrerequisite verifies that enabling a consumer (analytics, // traffic logging) without the collector auto-enables the collector (a // backward-compat soft prerequisite) rather than failing. From 9c4654a23086a16f3ed0ec5b8451ea4b08770f6d Mon Sep 17 00:00:00 2001 From: Dineth Date: Tue, 7 Jul 2026 11:22:20 +0530 Subject: [PATCH 07/21] Refactor analytics code to remove directive caching and update field projection handling for improved performance and clarity --- .../internal/analytics/analytics.go | 15 +---- .../internal/analytics/analytics_test.go | 14 ----- .../internal/analytics/publishers/log.go | 61 ++++++++++++------- 3 files changed, 42 insertions(+), 48 deletions(-) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go index 1f6c63672e..46177907bd 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go @@ -23,7 +23,6 @@ import ( "log/slog" "maps" "strconv" - "sync" "time" v3 "github.com/envoyproxy/go-control-plane/envoy/data/accesslog/v3" @@ -95,10 +94,6 @@ type Analytics struct { cfg *config.Config // publishers represents the publishers. publishers []analytics_publisher.Publisher - // directiveCache caches parsed TrafficLogDirectives keyed by raw JSON string. - // The directive is static per API deployment, so this eliminates per-request - // allocations after the first parse. - directiveCache sync.Map } // NewAnalytics creates a new instance of Analytics. Publishers are assembled from @@ -539,21 +534,15 @@ func (c *Analytics) prepareAnalyticEvent(logEntry *v3.HTTPAccessLogEntry) *dto.E return event } -// parseTrafficLogDirective returns the parsed TrafficLogDirective for the given -// raw JSON string, using a cache to avoid re-parsing the same static per-API -// directive on every request. Two concurrent first-parses of the same string are -// benign — the winner's value is stored and both callers get an equivalent result. +// parseTrafficLogDirective parses the raw traffic_log directive JSON for the +// current request. func (c *Analytics) parseTrafficLogDirective(raw string) *dto.TrafficLogDirective { - if cached, ok := c.directiveCache.Load(raw); ok { - return cached.(*dto.TrafficLogDirective) - } dir := &dto.TrafficLogDirective{} if raw != "" { if err := json.Unmarshal([]byte(raw), dir); err != nil { slog.Warn("Failed to parse traffic_log directive; opting in with defaults", "error", err) } } - c.directiveCache.Store(raw, dir) return dir } diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go index 38756eaf73..e6e5dc48e3 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go @@ -456,20 +456,6 @@ func TestPrepareAnalyticEvent_MalformedTrafficLogMarker(t *testing.T) { assert.Nil(t, event.TrafficLog.Response) } -func TestPrepareAnalyticEvent_TrafficLogDirectiveCached(t *testing.T) { - cfg := &config.Config{} - a := NewAnalytics(cfg) - raw := `{"request":{"payload":false,"headers":true}}` - logEntry := createLogEntryWithMetadata(map[string]string{TrafficLogMetadataKey: raw}) - - event1 := a.prepareAnalyticEvent(logEntry) - event2 := a.prepareAnalyticEvent(logEntry) - - require.NotNil(t, event1.TrafficLog) - // Same pointer — second call must return the cached directive, not a new allocation. - assert.Same(t, event1.TrafficLog, event2.TrafficLog) -} - func TestPrepareAnalyticEvent_WithAnonymousApp(t *testing.T) { cfg := &config.Config{} analytics := NewAnalytics(cfg) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go index 0718c1aec7..be06232e13 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go @@ -90,7 +90,9 @@ func (l *Log) Publish(event *dto.Event) { } if fields := dir.Fields; fields != nil && (len(fields.Only) > 0 || len(fields.Exclude) > 0) { - var m map[string]interface{} + // Shallow-decode only the top level; untouched fields stay as raw JSON + // bytes and are never deep-decoded or re-encoded. + var m map[string]json.RawMessage if err := json.Unmarshal(data, &m); err != nil { slog.Error("Failed to unmarshal for field projection; emitting as-is", "error", err) } else { @@ -158,7 +160,9 @@ func (l *Log) truncatePayload(s string) string { // dotted sub-key paths within map fields (e.g. "requestHeaders.authorization", // "labels.env"). Only keeps exactly the named fields; Exclude drops the named // fields and keeps everything else. If both are set, Only takes precedence. -func applyFieldsProjection(m map[string]interface{}, fields *dto.TrafficLogFields) { +// Top-level values are kept as raw JSON bytes; only the specific nested +// objects referenced by a dotted path are decoded and re-encoded. +func applyFieldsProjection(m map[string]json.RawMessage, fields *dto.TrafficLogFields) { if len(fields.Only) > 0 { directKeys := make(map[string]bool) subKeys := make(map[string][]string) // topKey → sub-keys to keep @@ -178,37 +182,52 @@ func applyFieldsProjection(m map[string]interface{}, fields *dto.TrafficLogField if directKeys[top] { continue // whole key kept; don't filter sub-keys } - if nested, ok := m[top].(map[string]interface{}); ok { - keep := make(map[string]bool, len(subs)) - for _, s := range subs { - keep[s] = true - } - for k := range nested { - if !keep[k] { - delete(nested, k) - } - } - if len(nested) == 0 { - delete(m, top) - } + keep := make(map[string]bool, len(subs)) + for _, s := range subs { + keep[s] = true } + filterNestedKeys(m, top, func(k string) bool { return keep[k] }) } return } for _, name := range fields.Exclude { if top, sub, found := strings.Cut(name, "."); found { - if nested, ok := m[top].(map[string]interface{}); ok { - delete(nested, sub) - if len(nested) == 0 { - delete(m, top) - } - } + filterNestedKeys(m, top, func(k string) bool { return k != sub }) } else { delete(m, name) } } } +// filterNestedKeys decodes the JSON object stored at m[top], keeps only the +// sub-keys for which keep returns true, and re-encodes the result back into +// m[top]. Deletes m[top] entirely if no sub-keys survive, or if m[top] is +// absent or not a JSON object. +func filterNestedKeys(m map[string]json.RawMessage, top string, keep func(string) bool) { + raw, ok := m[top] + if !ok { + return + } + var nested map[string]json.RawMessage + if err := json.Unmarshal(raw, &nested); err != nil { + return + } + for k := range nested { + if !keep(k) { + delete(nested, k) + } + } + if len(nested) == 0 { + delete(m, top) + return + } + filtered, err := json.Marshal(nested) + if err != nil { + return + } + m[top] = filtered +} + // maskHeaders redacts header values whose names appear in mask (case-insensitive). // Returns a new map; the input is not modified. Per-header exclusion is handled // downstream by applyFieldsProjection via dotted paths (e.g. "requestHeaders.authorization"). From 57452e9cb8bfaa3d7ef63703e8946f3df26b8ac7 Mon Sep 17 00:00:00 2001 From: Dineth Date: Tue, 7 Jul 2026 12:09:13 +0530 Subject: [PATCH 08/21] Implement collector package for shared data capture logic and migrate deprecated analytics configurations --- common/collector/collector.go | 104 +++++++++++++++++ common/collector/collector_test.go | 105 ++++++++++++++++++ .../gateway-controller/pkg/config/config.go | 69 +++++------- .../policy-engine/internal/config/config.go | 69 +++++------- 4 files changed, 259 insertions(+), 88 deletions(-) create mode 100644 common/collector/collector.go create mode 100644 common/collector/collector_test.go diff --git a/common/collector/collector.go b/common/collector/collector.go new file mode 100644 index 0000000000..307288301e --- /dev/null +++ b/common/collector/collector.go @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Package collector holds logic shared by the gateway-controller and +// policy-engine's [collector] configuration: the collector is the shared +// data-capture pipeline (system policy + ALS transport) that gathers +// request/response headers and bodies for every consumer (analytics, stdout +// traffic logging). It has no on/off flag of its own — it is implicit, +// derived from whether a consumer is enabled — and both modules migrate the +// same set of deprecated [analytics] aliases onto it at load time. The two +// modules keep their own Config/CollectorConfig struct shapes (their transport +// types differ: GRPCEventServerConfig vs AccessLogsServiceConfig), so this +// package holds only the derivation/migration logic, not the structs. +package collector + +import "log/slog" + +// IsEnabled reports whether the collector should run: implicitly active +// whenever any consumer of the collected data (analytics or stdout traffic +// logging) is enabled, and off otherwise. +func IsEnabled(analyticsEnabled, trafficLoggingEnabled bool) bool { + return analyticsEnabled || trafficLoggingEnabled +} + +// CaptureFlags holds the deprecated [analytics] body-capture aliases being +// migrated onto [collector]. +type CaptureFlags struct { + SendRequestBody bool + SendResponseBody bool + AllowPayloads bool +} + +// MigrateDeprecatedCapture maps the deprecated analytics.allow_payloads / +// analytics.send_request_body / analytics.send_response_body onto the +// collector's body-capture flags (when the collector flag is not already +// set), so existing configs keep working after capture settings moved under +// [collector]. These are analytics's own deprecated flags, so they are only +// honored while analyticsEnabled is true — otherwise a stale value left over +// from a disabled analytics setup could silently turn on body capture for an +// unrelated consumer (e.g. traffic_logging) enabled later. Directional +// aliases (send_request_body/send_response_body) take precedence over +// allow_payloads, which only fills in when neither directional flag is +// already set on the collector. +func MigrateDeprecatedCapture(analyticsEnabled bool, deprecated CaptureFlags, collectorSendRequestBody, collectorSendResponseBody *bool) { + if !analyticsEnabled { + return + } + if deprecated.SendRequestBody && !*collectorSendRequestBody { + slog.Warn("analytics.send_request_body is deprecated; use collector.send_request_body instead") + *collectorSendRequestBody = true + } + if deprecated.SendResponseBody && !*collectorSendResponseBody { + slog.Warn("analytics.send_response_body is deprecated; use collector.send_response_body instead") + *collectorSendResponseBody = true + } + if deprecated.AllowPayloads { + slog.Warn("analytics.allow_payloads is deprecated; use collector.send_request_body and collector.send_response_body instead") + if !*collectorSendRequestBody && !*collectorSendResponseBody { + *collectorSendRequestBody = true + *collectorSendResponseBody = true + } + } +} + +// MigrateDeprecatedTransport maps a deprecated [analytics] transport-tuning +// alias (e.g. analytics.grpc_event_server on the controller, +// analytics.access_logs_service on the policy-engine) onto the collector's +// transport config when the collector's copy is still at its default, so +// existing configs keep working after transport tuning moved to +// [collector.als]. It is generic over the two modules' differing transport +// struct shapes (T is only ever compared and assigned, never inspected). This +// is analytics's own deprecated field, so it is only honored while +// analyticsEnabled is true — otherwise a stale value left over from a +// disabled analytics setup could silently reconfigure the transport for an +// unrelated consumer (e.g. traffic_logging) enabled later. +// +// deprecatedKey names the deprecated config key for the warning message +// (e.g. "analytics.grpc_event_server"). +func MigrateDeprecatedTransport[T comparable](analyticsEnabled bool, deprecated T, collectorCfg *T, def T, deprecatedKey string) { + if !analyticsEnabled || deprecated == def { + return + } + if *collectorCfg == def { + slog.Warn(deprecatedKey + " is deprecated; migrating it to collector.als") + *collectorCfg = deprecated + } else { + slog.Warn(deprecatedKey + " is deprecated and collector.als is already configured; ignoring the " + deprecatedKey + " override") + } +} diff --git a/common/collector/collector_test.go b/common/collector/collector_test.go new file mode 100644 index 0000000000..d878f2267d --- /dev/null +++ b/common/collector/collector_test.go @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package collector + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsEnabled(t *testing.T) { + assert.False(t, IsEnabled(false, false)) + assert.True(t, IsEnabled(true, false)) + assert.True(t, IsEnabled(false, true)) + assert.True(t, IsEnabled(true, true)) +} + +func TestMigrateDeprecatedCapture_SkippedWhenAnalyticsDisabled(t *testing.T) { + collectorReq, collectorResp := false, false + MigrateDeprecatedCapture(false, CaptureFlags{SendRequestBody: true, SendResponseBody: true, AllowPayloads: true}, &collectorReq, &collectorResp) + assert.False(t, collectorReq) + assert.False(t, collectorResp) +} + +func TestMigrateDeprecatedCapture_DirectionalFlags(t *testing.T) { + collectorReq, collectorResp := false, false + MigrateDeprecatedCapture(true, CaptureFlags{SendRequestBody: true, SendResponseBody: false}, &collectorReq, &collectorResp) + assert.True(t, collectorReq) + assert.False(t, collectorResp) +} + +func TestMigrateDeprecatedCapture_AllowPayloadsFillsInWhenNoDirectionalFlags(t *testing.T) { + collectorReq, collectorResp := false, false + MigrateDeprecatedCapture(true, CaptureFlags{AllowPayloads: true}, &collectorReq, &collectorResp) + assert.True(t, collectorReq) + assert.True(t, collectorResp) +} + +func TestMigrateDeprecatedCapture_DirectionalFlagsWinOverAllowPayloads(t *testing.T) { + collectorReq, collectorResp := false, true + MigrateDeprecatedCapture(true, CaptureFlags{SendRequestBody: true, AllowPayloads: true}, &collectorReq, &collectorResp) + assert.True(t, collectorReq) + assert.True(t, collectorResp, "already true; allow_payloads must not be re-applied since a directional flag is set") +} + +type testTransportConfig struct { + Port int + Mode string +} + +func TestMigrateDeprecatedTransport_SkippedWhenAnalyticsDisabled(t *testing.T) { + def := testTransportConfig{Port: 18090, Mode: "uds"} + collectorCfg := def + deprecated := testTransportConfig{Port: 9999, Mode: "tcp"} + + MigrateDeprecatedTransport(false, deprecated, &collectorCfg, def, "analytics.grpc_event_server") + + assert.Equal(t, def, collectorCfg, "collector transport must stay at default when analytics is disabled") +} + +func TestMigrateDeprecatedTransport_SkippedWhenDeprecatedAtDefault(t *testing.T) { + def := testTransportConfig{Port: 18090, Mode: "uds"} + collectorCfg := def + + MigrateDeprecatedTransport(true, def, &collectorCfg, def, "analytics.grpc_event_server") + + assert.Equal(t, def, collectorCfg) +} + +func TestMigrateDeprecatedTransport_MigratesWhenCollectorAtDefault(t *testing.T) { + def := testTransportConfig{Port: 18090, Mode: "uds"} + collectorCfg := def + deprecated := testTransportConfig{Port: 9999, Mode: "tcp"} + + MigrateDeprecatedTransport(true, deprecated, &collectorCfg, def, "analytics.grpc_event_server") + + assert.Equal(t, deprecated, collectorCfg) +} + +func TestMigrateDeprecatedTransport_DoesNotClobberExplicitCollectorConfig(t *testing.T) { + def := testTransportConfig{Port: 18090, Mode: "uds"} + explicit := testTransportConfig{Port: 7000, Mode: "tcp"} + collectorCfg := explicit + deprecated := testTransportConfig{Port: 9999, Mode: "tcp"} + + MigrateDeprecatedTransport(true, deprecated, &collectorCfg, def, "analytics.grpc_event_server") + + assert.Equal(t, explicit, collectorCfg, "an explicit [collector.als] override must never be clobbered by the deprecated alias") +} diff --git a/gateway/gateway-controller/pkg/config/config.go b/gateway/gateway-controller/pkg/config/config.go index b93e371d26..f16bc38bb2 100644 --- a/gateway/gateway-controller/pkg/config/config.go +++ b/gateway/gateway-controller/pkg/config/config.go @@ -20,7 +20,6 @@ package config import ( "fmt" - "log/slog" "net/url" "strconv" "strings" @@ -31,6 +30,7 @@ import ( "github.com/knadh/koanf/providers/env" "github.com/knadh/koanf/providers/file" "github.com/knadh/koanf/v2" + "github.com/wso2/api-platform/common/collector" commonconstants "github.com/wso2/api-platform/common/constants" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/constants" ) @@ -1772,60 +1772,41 @@ func (c *Config) validateCollectorConfig() error { // (analytics or stdout traffic logging), and off otherwise. When active, the // controller injects the analytics system policy and configures Envoy's ALS sink. func (c *Config) IsCollectorEnabled() bool { - return c.Analytics.Enabled || c.TrafficLogging.Enabled + return collector.IsEnabled(c.Analytics.Enabled, c.TrafficLogging.Enabled) } // migrateDeprecatedAnalyticsTransport maps a deprecated [analytics].grpc_event_server // override onto the collector when the collector's transport tuning is still at its // default, so existing configs keep working after the transport moved to [collector]. -// This is analytics's own deprecated field, so it is only honored while analytics is -// enabled — otherwise a stale value left over from a disabled analytics setup could -// silently reconfigure the transport for an unrelated consumer (e.g. traffic_logging) -// enabled later. +// See collector.MigrateDeprecatedTransport for the shared (with the policy-engine) +// migration logic and its guarding-while-analytics-enabled rationale. func (c *Config) migrateDeprecatedAnalyticsTransport() { - if !c.Analytics.Enabled { - return - } - def := defaultGRPCEventServerConfig() - if c.Analytics.GRPCEventServerCfg != def { - if c.Collector.GRPCEventServerCfg == def { - slog.Warn("analytics.grpc_event_server is deprecated; migrating it to collector.als") - c.Collector.GRPCEventServerCfg = c.Analytics.GRPCEventServerCfg - } else { - slog.Warn("analytics.grpc_event_server is deprecated and collector.als is already configured; ignoring the analytics.grpc_event_server override") - } - } + collector.MigrateDeprecatedTransport( + c.Analytics.Enabled, + c.Analytics.GRPCEventServerCfg, + &c.Collector.GRPCEventServerCfg, + defaultGRPCEventServerConfig(), + "analytics.grpc_event_server", + ) } // migrateDeprecatedAnalyticsCapture maps the deprecated analytics.allow_payloads / // analytics.send_request_body / analytics.send_response_body onto the collector's -// body-capture flags (when the collector flag is not already set), so existing -// configs keep working after capture settings moved under [collector]. These are -// analytics's own deprecated flags, so they are only honored while analytics is -// enabled — otherwise a stale value left over from a disabled analytics setup -// could silently turn on body capture for an unrelated consumer (e.g. -// traffic_logging) enabled later. +// body-capture flags, so existing configs keep working after capture settings +// moved under [collector]. See collector.MigrateDeprecatedCapture for the shared +// (with the policy-engine) migration logic and its guarding-while-analytics- +// enabled rationale. func (c *Config) migrateDeprecatedAnalyticsCapture() { - if !c.Analytics.Enabled { - return - } - // Directional aliases take precedence over allow_payloads. - if c.Analytics.SendRequestBody && !c.Collector.SendRequestBody { - slog.Warn("analytics.send_request_body is deprecated; use collector.send_request_body instead") - c.Collector.SendRequestBody = true - } - if c.Analytics.SendResponseBody && !c.Collector.SendResponseBody { - slog.Warn("analytics.send_response_body is deprecated; use collector.send_response_body instead") - c.Collector.SendResponseBody = true - } - // allow_payloads only fills in when no directional body capture is configured. - if c.Analytics.AllowPayloads { - slog.Warn("analytics.allow_payloads is deprecated; use collector.send_request_body and collector.send_response_body instead") - if !c.Collector.SendRequestBody && !c.Collector.SendResponseBody { - c.Collector.SendRequestBody = true - c.Collector.SendResponseBody = true - } - } + collector.MigrateDeprecatedCapture( + c.Analytics.Enabled, + collector.CaptureFlags{ + SendRequestBody: c.Analytics.SendRequestBody, + SendResponseBody: c.Analytics.SendResponseBody, + AllowPayloads: c.Analytics.AllowPayloads, + }, + &c.Collector.SendRequestBody, + &c.Collector.SendResponseBody, + ) } // validateGRPCEventServerConfig validates the Envoy→policy-engine ALS transport tuning. diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config.go b/gateway/gateway-runtime/policy-engine/internal/config/config.go index 0ee95ac8e1..d3fee0eb03 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config.go @@ -20,7 +20,6 @@ package config import ( "fmt" - "log/slog" "math" "net/url" "strings" @@ -31,6 +30,7 @@ import ( "github.com/knadh/koanf/providers/env" "github.com/knadh/koanf/providers/file" "github.com/knadh/koanf/v2" + "github.com/wso2/api-platform/common/collector" ) const ( @@ -618,29 +618,22 @@ func (c *Config) validateCollectorConfig() error { // implicit: it is active whenever any consumer of the collected data is enabled // (analytics or stdout traffic logging), and off otherwise. func (c *Config) IsCollectorEnabled() bool { - return c.Analytics.Enabled || c.TrafficLogging.Enabled + return collector.IsEnabled(c.Analytics.Enabled, c.TrafficLogging.Enabled) } // migrateDeprecatedAnalyticsTransport maps a deprecated [analytics].access_logs_service // override onto the collector when the collector's receiver tuning is still at its // default, so existing configs keep working after the transport moved to [collector]. -// This is analytics's own deprecated field, so it is only honored while analytics is -// enabled — otherwise a stale value left over from a disabled analytics setup could -// silently reconfigure the transport for an unrelated consumer (e.g. traffic_logging) -// enabled later. +// See collector.MigrateDeprecatedTransport for the shared (with the gateway-controller) +// migration logic and its guarding-while-analytics-enabled rationale. func (c *Config) migrateDeprecatedAnalyticsTransport() { - if !c.Analytics.Enabled { - return - } - def := defaultAccessLogsServiceConfig() - if c.Analytics.AccessLogsServiceCfg != def { - if c.Collector.AccessLogsServiceCfg == def { - slog.Warn("analytics.access_logs_service is deprecated; migrating it to collector.als") - c.Collector.AccessLogsServiceCfg = c.Analytics.AccessLogsServiceCfg - } else { - slog.Warn("analytics.access_logs_service is deprecated and collector.als is already configured; ignoring the analytics.access_logs_service override") - } - } + collector.MigrateDeprecatedTransport( + c.Analytics.Enabled, + c.Analytics.AccessLogsServiceCfg, + &c.Collector.AccessLogsServiceCfg, + defaultAccessLogsServiceConfig(), + "analytics.access_logs_service", + ) } // validateAccessLogsServiceConfig validates the policy-engine ALS receiver tuning. @@ -672,33 +665,21 @@ func validateAccessLogsServiceConfig(als AccessLogsServiceConfig) error { // migrateDeprecatedAnalyticsCapture maps the deprecated analytics.allow_payloads / // analytics.send_request_body / analytics.send_response_body onto the collector's -// body-capture flags (when the collector flag is not already set), so existing -// configs keep working after capture settings moved under [collector]. These are -// analytics's own deprecated flags, so they are only honored while analytics is -// enabled — otherwise a stale value left over from a disabled analytics setup -// could silently turn on body capture for an unrelated consumer (e.g. -// traffic_logging) enabled later. +// body-capture flags, so existing configs keep working after capture settings +// moved under [collector]. See collector.MigrateDeprecatedCapture for the shared +// (with the gateway-controller) migration logic and its guarding-while-analytics- +// enabled rationale. func (c *Config) migrateDeprecatedAnalyticsCapture() { - if !c.Analytics.Enabled { - return - } - // Directional aliases take precedence over allow_payloads. - if c.Analytics.SendRequestBody && !c.Collector.SendRequestBody { - slog.Warn("analytics.send_request_body is deprecated; use collector.send_request_body instead") - c.Collector.SendRequestBody = true - } - if c.Analytics.SendResponseBody && !c.Collector.SendResponseBody { - slog.Warn("analytics.send_response_body is deprecated; use collector.send_response_body instead") - c.Collector.SendResponseBody = true - } - // allow_payloads only fills in when no directional body capture is configured. - if c.Analytics.AllowPayloads { - slog.Warn("analytics.allow_payloads is deprecated; use collector.send_request_body and collector.send_response_body instead") - if !c.Collector.SendRequestBody && !c.Collector.SendResponseBody { - c.Collector.SendRequestBody = true - c.Collector.SendResponseBody = true - } - } + collector.MigrateDeprecatedCapture( + c.Analytics.Enabled, + collector.CaptureFlags{ + SendRequestBody: c.Analytics.SendRequestBody, + SendResponseBody: c.Analytics.SendResponseBody, + AllowPayloads: c.Analytics.AllowPayloads, + }, + &c.Collector.SendRequestBody, + &c.Collector.SendResponseBody, + ) } // validateAnalyticsConfig validates the analytics consumer configuration (publishers). From 92f1729afc7846f743d57d01835d4b6e95a8794e Mon Sep 17 00:00:00 2001 From: Dineth Date: Tue, 7 Jul 2026 13:06:23 +0530 Subject: [PATCH 09/21] change configs --- gateway/configs/config-template.toml | 24 +++++++++++++----------- gateway/configs/config.toml | 6 +++--- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/gateway/configs/config-template.toml b/gateway/configs/config-template.toml index 936475b881..48acff469c 100644 --- a/gateway/configs/config-template.toml +++ b/gateway/configs/config-template.toml @@ -298,8 +298,8 @@ send_response_body = false # Capture ALL request/response headers into the collected event (request_headers/ # response_headers). Applies to every API via the collector system policy, so no # per-API header policy is needed. Sensitive values are redacted on output by -# consumers that support it (e.g. traffic_logging.masked_headers) — the Moesif -# publisher does not mask headers. Off by default; enable deliberately per consumer. +# consumers that support it (e.g. traffic_logging.masked_headers) +# Off by default; enable deliberately per consumer. send_request_headers = false send_response_headers = false @@ -328,15 +328,17 @@ max_header_limit = 8192 # ============================================================================= [analytics] enabled = false -# Deprecated: allow_payloads / send_request_body / send_response_body are preserved -# for backward compatibility and are mapped onto the [collector] body-capture flags -# during validation (with a warning). Prefer setting [collector] directly. +# Deprecated: allow_payloads is preserved for backward compatibility. When true +# and both send_request_body and send_response_body are false, both directions +# are enabled (bool fields cannot distinguish "unset" from explicitly false). +# To disable payloads entirely, set allow_payloads = false or remove it; do not +# rely on send_*_body = false while allow_payloads remains true. allow_payloads = false -# send_request_body = false -# send_response_body = false -# Publishers that receive analytics events. Supported: "moesif" (sends to the -# Moesif SaaS). For stdout JSON logging use [traffic_logging] instead of a "log" -# publisher here. +# Deprecated: send_request_body / send_response_body are preserved +# for backward compatibility and are mapped onto the [collector] body-capture flags +# during validation (with a warning). Prefer setting [collector] directly.\ +send_request_body = false +send_response_body = false enabled_publishers = ["moesif"] [analytics.publishers.moesif] @@ -371,7 +373,7 @@ enabled = false masked_headers = ["authorization", "x-api-key", "x-jwt-assertion"] # Max bytes of request/response payload written to each log line (0 = no limit). # Applied output-side, so the collector still captures full bodies and other -# consumers (e.g. Moesif) are unaffected. +# consumers are unaffected. max_payload_size = 0 # ============================================================================= diff --git a/gateway/configs/config.toml b/gateway/configs/config.toml index fbabb75d6d..6d46487587 100644 --- a/gateway/configs/config.toml +++ b/gateway/configs/config.toml @@ -15,11 +15,11 @@ send_response_headers = false # Analytics consumer (e.g. Moesif). Enabling it activates the collector automatically. [analytics] -enabled = false +enabled = true enabled_publishers = ["moesif"] -# [analytics.publishers.moesif] -# application_id = "" +[analytics.publishers.moesif] +application_id = "" # Traffic-logging consumer: writes each collected event to stdout as a JSON line # (no external service needed). Enabling it activates the collector automatically. From e009fe4c3d037e43f3449a51201148614baaea32 Mon Sep 17 00:00:00 2001 From: Dineth Date: Tue, 7 Jul 2026 14:41:18 +0530 Subject: [PATCH 10/21] Add microsecond-precision traffic log latencies to analytics events - Introduced TrafficLogLatencies struct to hold microsecond timings. - Updated prepareAnalyticEvent to compute and assign traffic log latencies. - Modified tests to validate new traffic log latency calculations. - Adjusted existing Latencies struct to maintain separation from traffic log data. --- .../internal/analytics/analytics.go | 30 +++++++++---- .../internal/analytics/analytics_test.go | 9 +++- .../internal/analytics/dto/dto_test.go | 8 ---- .../internal/analytics/dto/event.go | 7 ++++ .../internal/analytics/dto/latencies.go | 42 ++++++++++++------- .../internal/analytics/publishers/log_test.go | 5 ++- .../analytics/publishers/moesif_test.go | 6 +++ .../analytics/publishers/traffic_log_event.go | 6 +-- 8 files changed, 75 insertions(+), 38 deletions(-) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go index 46177907bd..eb6085e2bc 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go @@ -276,34 +276,50 @@ func (c *Analytics) prepareAnalyticEvent(logEntry *v3.HTTPAccessLogEntry) *dto.E toMs := func(secs int64, nanos int32) int64 { return (secs * 1000) + int64(nanos)/1_000_000 } + toUs := func(secs int64, nanos int32) int64 { + return (secs * 1_000_000) + int64(nanos)/1000 + } + // Moesif-oriented latencies (milliseconds). lastRx := toMs(properties.TimeToLastRxByte.Seconds, properties.TimeToLastRxByte.Nanos) firstUpTx := toMs(properties.TimeToFirstUpstreamTxByte.Seconds, properties.TimeToFirstUpstreamTxByte.Nanos) firstUpRx := toMs(properties.TimeToFirstUpstreamRxByte.Seconds, properties.TimeToFirstUpstreamRxByte.Nanos) lastUpRx := toMs(properties.TimeToLastUpstreamRxByte.Seconds, properties.TimeToLastUpstreamRxByte.Nanos) lastDownTx := toMs(properties.TimeToLastDownstreamTxByte.Seconds, properties.TimeToLastDownstreamTxByte.Nanos) - latencies := dto.Latencies{ - Duration: lastDownTx, + event.Latencies = &dto.Latencies{ BackendLatency: lastUpRx - firstUpTx, RequestMediationLatency: firstUpTx - lastRx, ResponseLatency: lastDownTx - firstUpRx, ResponseMediationLatency: lastDownTx - lastUpRx, } + // Traffic-log latencies (microseconds), derived from the same timepoints + // at full precision. Kept separate from the millisecond Latencies above so + // Moesif's units are unaffected. + lastRxUs := toUs(properties.TimeToLastRxByte.Seconds, properties.TimeToLastRxByte.Nanos) + firstUpTxUs := toUs(properties.TimeToFirstUpstreamTxByte.Seconds, properties.TimeToFirstUpstreamTxByte.Nanos) + firstUpRxUs := toUs(properties.TimeToFirstUpstreamRxByte.Seconds, properties.TimeToFirstUpstreamRxByte.Nanos) + lastDownTxUs := toUs(properties.TimeToLastDownstreamTxByte.Seconds, properties.TimeToLastDownstreamTxByte.Nanos) + + trafficLatencies := dto.TrafficLogLatencies{ + DurationUs: lastDownTxUs, // DS_RX_BEG → DS_TX_END + RequestMediationLatencyUs: firstUpTxUs - lastRxUs, // DS_RX_END → US_TX_BEG + } + // US_TX_END → US_RX_BEG: time the backend spent before sending the first response byte (TTFB). if properties.TimeToLastUpstreamTxByte != nil { - lastUpTx := toMs(properties.TimeToLastUpstreamTxByte.Seconds, properties.TimeToLastUpstreamTxByte.Nanos) - latencies.BackendProcDuration = firstUpRx - lastUpTx + lastUpTxUs := toUs(properties.TimeToLastUpstreamTxByte.Seconds, properties.TimeToLastUpstreamTxByte.Nanos) + trafficLatencies.BackendLatencyUs = firstUpRxUs - lastUpTxUs } // US_RX_BEG → DS_TX_BEG: gateway overhead processing the first response byte before writing downstream. if properties.TimeToFirstDownstreamTxByte != nil { - firstDownTx := toMs(properties.TimeToFirstDownstreamTxByte.Seconds, properties.TimeToFirstDownstreamTxByte.Nanos) - latencies.ResponseProcDuration = firstDownTx - firstUpRx + firstDownTxUs := toUs(properties.TimeToFirstDownstreamTxByte.Seconds, properties.TimeToFirstDownstreamTxByte.Nanos) + trafficLatencies.ResponseMediationLatencyUs = firstDownTxUs - firstUpRxUs } - event.Latencies = &latencies + event.TrafficLogLatencies = &trafficLatencies } // prepare metaInfo diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go index e6e5dc48e3..49b0a585ab 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go @@ -529,6 +529,11 @@ func TestPrepareAnalyticEvent_WithLatencies(t *testing.T) { require.NotNil(t, event) require.NotNil(t, event.Latencies) assert.True(t, event.Latencies.BackendLatency >= 0) + + // Traffic-log latencies are computed in microseconds from the same timepoints. + require.NotNil(t, event.TrafficLogLatencies) + assert.Equal(t, int64(250000), event.TrafficLogLatencies.DurationUs) // DS_RX_BEG → DS_TX_END = 250ms + assert.Equal(t, int64(50000), event.TrafficLogLatencies.RequestMediationLatencyUs) // 100ms - 50ms } func TestPrepareAnalyticEvent_WithUserID(t *testing.T) { @@ -552,8 +557,8 @@ func TestPrepareAnalyticEvent_WithLLMCost(t *testing.T) { analytics := NewAnalytics(cfg) logEntry := createLogEntryWithMetadata(map[string]string{ - AIProviderNameMetadataKey: "openai", - ModelIDMetadataKey: "gpt-4", + AIProviderNameMetadataKey: "openai", + ModelIDMetadataKey: "gpt-4", constants.LLMCostMetadataKey: "0.0000423100", }) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/dto_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/dto_test.go index c544ddc0e5..b771083d1d 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/dto_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/dto_test.go @@ -147,14 +147,6 @@ func TestLatencies_GetSetResponseMediationLatency(t *testing.T) { assert.Equal(t, int64(30), lat.GetResponseMediationLatency()) } -func TestLatencies_GetSetDuration(t *testing.T) { - lat := &Latencies{} - assert.Equal(t, int64(0), lat.GetDuration()) - - lat.SetDuration(200) - assert.Equal(t, int64(200), lat.GetDuration()) -} - // ============================================================================= // AIMetadata Tests // ============================================================================= diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go index fe5d7a89c4..d6d2cd0094 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go @@ -47,6 +47,13 @@ type Event struct { ErrorType string `json:"errorType,omitempty" bson:"error_type"` Properties map[string]interface{} `json:"properties,omitempty" bson:"properties"` + // TrafficLogLatencies carries microsecond-precision gateway/backend timings + // for the stdout traffic-logging publisher. It is computed from the same ALS + // CommonProperties timepoints as Latencies but at full precision, and is kept + // separate so Moesif's millisecond units are unaffected. Never serialized + // (json:"-") and not sent to other publishers. + TrafficLogLatencies *TrafficLogLatencies `json:"-" bson:"-"` + // TrafficLog carries the per-API stdout traffic-logging opt-in marker stamped // by the log-message policy (access-log mode). When nil, the API has not opted // in and the stdout traffic-logging publisher skips the event. It is diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/latencies.go b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/latencies.go index e9e4818f0e..384d31706e 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/latencies.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/latencies.go @@ -14,21 +14,39 @@ * limitations under the License. * */ - + package dto -// Latencies represents latency attributes in an analytics event. +// Latencies represents latency attributes in an analytics event. All values are +// in milliseconds: this shape feeds Moesif, which treats them as milliseconds. +// Finer-grained gateway/backend timings for the traffic log live in the separate +// TrafficLogLatencies (microseconds) so the units Moesif depends on never change. type Latencies struct { ResponseLatency int64 `json:"responseLatency"` BackendLatency int64 `json:"backendLatency"` RequestMediationLatency int64 `json:"requestMediationLatency"` ResponseMediationLatency int64 `json:"responseMediationLatency"` - // Duration is the total request duration: downstream request received → downstream response sent (ms). - Duration int64 `json:"duration"` - // BackendProcDuration is the backend TTFB: upstream request fully sent → first upstream response byte (ms). - BackendProcDuration int64 `json:"backendProcDuration"` - // ResponseProcDuration is the gateway response overhead: first upstream response byte → first downstream response byte (ms). - ResponseProcDuration int64 `json:"responseProcDuration"` +} + +// TrafficLogLatencies holds gateway/backend timings for a traffic-log event, in +// microseconds. It is intentionally separate from Latencies (which is shaped for +// Moesif and expressed in milliseconds) so the traffic log can carry +// finer-grained timings without changing the units Moesif depends on. The +// timepoint labels (DS_RX_BEG, US_TX_BEG, …) refer to Envoy's request/response +// timeline as reported through the ALS CommonProperties. +type TrafficLogLatencies struct { + // DurationUs is the total request duration: downstream request received → + // downstream response sent (DS_RX_BEG → DS_TX_END). + DurationUs int64 `json:"durationUs"` + // RequestMediationLatencyUs is the gateway request overhead: downstream request + // fully received → first byte sent upstream (DS_RX_END → US_TX_BEG). + RequestMediationLatencyUs int64 `json:"requestMediationLatencyUs"` + // ResponseMediationLatencyUs is the gateway response overhead: first upstream + // response byte → first downstream response byte (US_RX_BEG → DS_TX_BEG). + ResponseMediationLatencyUs int64 `json:"responseMediationLatencyUs"` + // BackendLatencyUs is the backend TTFB: upstream request fully sent → + // first upstream response byte (US_TX_END → US_RX_BEG). + BackendLatencyUs int64 `json:"backendLatencyUs"` } // GetResponseLatency returns the response latency. @@ -70,11 +88,3 @@ func (l *Latencies) GetResponseMediationLatency() int64 { func (l *Latencies) SetResponseMediationLatency(responseMediationLatency int64) { l.ResponseMediationLatency = responseMediationLatency } - -func (l *Latencies) GetDuration() int64 { - return l.Duration -} - -func (l *Latencies) SetDuration(duration int64) { - l.Duration = duration -} \ No newline at end of file diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go index fa4fb8c5e0..fbb4b5b667 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go @@ -111,9 +111,10 @@ func TestLog_Publish_WritesJSONLineWithLatencies(t *testing.T) { assert.Equal(t, "bar", reqH["x-foo"]) // ALS-derived latencies are always present in the line — the key improvement - // over the inline log-message policy, which could never see them. + // over the inline log-message policy, which could never see them. The traffic + // log carries microsecond-precision timings, separate from Moesif's ms fields. latencies := decoded["latencies"].(map[string]interface{}) - assert.Equal(t, float64(100), latencies["responseLatency"]) + assert.Equal(t, float64(250000), latencies["durationUs"]) } // Labels from the directive are emitted as a top-level "labels" object. diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/moesif_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/moesif_test.go index 63941d4615..1a5afa8197 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/moesif_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/moesif_test.go @@ -85,6 +85,12 @@ func createBaseEvent() *dto.Event { Latencies: &dto.Latencies{ ResponseLatency: 100, }, + TrafficLogLatencies: &dto.TrafficLogLatencies{ + DurationUs: 250000, + RequestMediationLatencyUs: 50000, + ResponseMediationLatencyUs: 30000, + BackendLatencyUs: 40000, + }, } } diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/traffic_log_event.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/traffic_log_event.go index da6b251a70..837c561f71 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/traffic_log_event.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/traffic_log_event.go @@ -38,8 +38,8 @@ type TrafficLogEvent struct { Operation *TrafficLogOperation `json:"operation,omitempty"` Target *TrafficLogTarget `json:"target,omitempty"` Application *TrafficLogApplication `json:"application,omitempty"` - Client *TrafficLogClient `json:"client,omitempty"` - Latencies *dto.Latencies `json:"latencies,omitempty"` + Client *TrafficLogClient `json:"client,omitempty"` + Latencies *dto.TrafficLogLatencies `json:"latencies,omitempty"` RequestHeaders map[string]string `json:"requestHeaders,omitempty"` ResponseHeaders map[string]string `json:"responseHeaders,omitempty"` RequestBody string `json:"requestBody,omitempty"` @@ -89,7 +89,7 @@ type TrafficLogClient struct { func (l *Log) toTrafficLogEvent(event *dto.Event, dir *dto.TrafficLogDirective) *TrafficLogEvent { tl := &TrafficLogEvent{ Status: event.ProxyResponseCode, - Latencies: event.Latencies, + Latencies: event.TrafficLogLatencies, } if !event.RequestTimestamp.IsZero() { From d51436caf224b06ec26fce84b91577198a12136b Mon Sep 17 00:00:00 2001 From: Dineth Date: Tue, 7 Jul 2026 19:42:19 +0530 Subject: [PATCH 11/21] Rename 'Labels' to 'Properties' in TrafficLogDirective and related tests for consistency --- .../internal/analytics/dto/event.go | 8 +-- .../internal/analytics/publishers/log_test.go | 50 +++++++++---------- .../analytics/publishers/traffic_log_event.go | 6 +-- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go index d6d2cd0094..b605ade6e3 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go @@ -69,10 +69,10 @@ type TrafficLogDirective struct { Request *TrafficLogFlow `json:"request,omitempty"` Response *TrafficLogFlow `json:"response,omitempty"` Fields *TrafficLogFields `json:"fields,omitempty"` - // Labels holds the policy's resolved labels (context references already + // Properties holds the policy's resolved properties (context references already // expanded at request time). The Log publisher emits them as a top-level - // "labels" object on the log line. - Labels map[string]interface{} `json:"labels,omitempty"` + // "properties" object on the log line. + Properties map[string]interface{} `json:"properties,omitempty"` // MaskedHeaders lists lower-cased header names whose values are redacted // in the emitted log line. Merged with the global masked_headers config. MaskedHeaders []string `json:"maskedHeaders,omitempty"` @@ -88,7 +88,7 @@ type TrafficLogFlow struct { // of Only or Exclude should be set. Only keeps exactly the named fields; Exclude // drops the named fields and keeps everything else. Names are top-level keys // (e.g. "latencies", "requestHeaders") or dotted sub-key paths within map fields -// (e.g. "requestHeaders.authorization", "labels.env"). When set, this is +// (e.g. "requestHeaders.authorization", "properties.env"). When set, this is // authoritative over field presence; per-flow Payload/Headers booleans are ignored // (global header masking still applies). If both are set, Only takes precedence. type TrafficLogFields struct { diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go index fbb4b5b667..08249f48ac 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go @@ -117,13 +117,13 @@ func TestLog_Publish_WritesJSONLineWithLatencies(t *testing.T) { assert.Equal(t, float64(250000), latencies["durationUs"]) } -// Labels from the directive are emitted as a top-level "labels" object. -func TestLog_Publish_LabelsTopLevel(t *testing.T) { +// Properties from the directive are emitted as a top-level "properties" object. +func TestLog_Publish_PropertiesTopLevel(t *testing.T) { l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) event := createBaseEvent() event.TrafficLog = &dto.TrafficLogDirective{ Request: &dto.TrafficLogFlow{Headers: true}, - Labels: map[string]interface{}{ + Properties: map[string]interface{}{ "who": "alice", "authType": "jwt", "retryCount": float64(3), @@ -134,17 +134,17 @@ func TestLog_Publish_LabelsTopLevel(t *testing.T) { l.Publish(event) decoded := decodeLine(t, read()) - labels, ok := decoded["labels"].(map[string]interface{}) - require.True(t, ok, "expected top-level labels object, got %T", decoded["labels"]) - assert.Equal(t, "alice", labels["who"]) - assert.Equal(t, "jwt", labels["authType"]) - assert.Equal(t, float64(3), labels["retryCount"]) + props, ok := decoded["properties"].(map[string]interface{}) + require.True(t, ok, "expected top-level properties object, got %T", decoded["properties"]) + assert.Equal(t, "alice", props["who"]) + assert.Equal(t, "jwt", props["authType"]) + assert.Equal(t, float64(3), props["retryCount"]) reqH := headerMap(t, decoded["requestHeaders"]) assert.Equal(t, "bar", reqH["x-foo"]) } -// A directive with no labels emits no "labels" key. -func TestLog_Publish_NoLabelsWhenAbsent(t *testing.T) { +// A directive with no properties emits no "properties" key. +func TestLog_Publish_NoPropertiesWhenAbsent(t *testing.T) { l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) event := createBaseEvent() event.TrafficLog = bothFlows() @@ -152,26 +152,26 @@ func TestLog_Publish_NoLabelsWhenAbsent(t *testing.T) { l.Publish(event) decoded := decodeLine(t, read()) - _, present := decoded["labels"] - assert.False(t, present, "no labels key expected when directive has no labels") + _, present := decoded["properties"] + assert.False(t, present, "no properties key expected when directive has no properties") } -// The fields projection can select "labels" like any other top-level key. -func TestLog_Publish_LabelsProjectableViaFields(t *testing.T) { +// The fields projection can select "properties" like any other top-level key. +func TestLog_Publish_PropertiesProjectableViaFields(t *testing.T) { l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) event := createBaseEvent() event.TrafficLog = &dto.TrafficLogDirective{ - Labels: map[string]interface{}{"who": "alice"}, - Fields: &dto.TrafficLogFields{Only: []string{"labels"}}, + Properties: map[string]interface{}{"who": "alice"}, + Fields: &dto.TrafficLogFields{Only: []string{"properties"}}, } event.Properties["requestHeaders"] = `{"x-foo":"bar"}` l.Publish(event) decoded := decodeLine(t, read()) - labels, ok := decoded["labels"].(map[string]interface{}) - require.True(t, ok, "expected labels retained by include projection") - assert.Equal(t, "alice", labels["who"]) + props, ok := decoded["properties"].(map[string]interface{}) + require.True(t, ok, "expected properties retained by include projection") + assert.Equal(t, "alice", props["who"]) _, hasHeaders := decoded["requestHeaders"] assert.False(t, hasHeaders, "requestHeaders not in Only list -> dropped") } @@ -350,16 +350,16 @@ func TestLog_Publish_FieldsExclude(t *testing.T) { assert.Contains(t, decoded, "requestHeaders", "requestHeaders kept (not excluded)") } -// requestBody and labels are top-level keys like any other and can be selected +// requestBody and properties are top-level keys like any other and can be selected // explicitly via fields.only. -func TestLog_Publish_FieldsIncludeRequestBodyAndLabels(t *testing.T) { +func TestLog_Publish_FieldsIncludeRequestBodyAndProperties(t *testing.T) { l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) event := createBaseEvent() event.Properties["requestHeaders"] = `{"x-foo":"bar"}` event.Properties["request_payload"] = "body-data" event.TrafficLog = &dto.TrafficLogDirective{ - Labels: map[string]interface{}{"env": "prod"}, - Fields: &dto.TrafficLogFields{Only: []string{"requestBody", "labels"}}, + Properties: map[string]interface{}{"env": "prod"}, + Fields: &dto.TrafficLogFields{Only: []string{"requestBody", "properties"}}, } l.Publish(event) @@ -370,9 +370,9 @@ func TestLog_Publish_FieldsIncludeRequestBodyAndLabels(t *testing.T) { _, hasHeaders := decoded["requestHeaders"] assert.False(t, hasHeaders, "requestHeaders not in Only list -> dropped") assert.Equal(t, "body-data", decoded["requestBody"]) - labels, ok := decoded["labels"].(map[string]interface{}) + props, ok := decoded["properties"].(map[string]interface{}) require.True(t, ok) - assert.Equal(t, "prod", labels["env"]) + assert.Equal(t, "prod", props["env"]) } // Output-side payload truncation (0 = no limit). diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/traffic_log_event.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/traffic_log_event.go index 837c561f71..3e1e203091 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/traffic_log_event.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/traffic_log_event.go @@ -44,7 +44,7 @@ type TrafficLogEvent struct { ResponseHeaders map[string]string `json:"responseHeaders,omitempty"` RequestBody string `json:"requestBody,omitempty"` ResponseBody string `json:"responseBody,omitempty"` - Labels map[string]interface{} `json:"labels,omitempty"` + Properties map[string]interface{} `json:"properties,omitempty"` } // TrafficLogAPI identifies the API that processed the request. @@ -189,8 +189,8 @@ func (l *Log) toTrafficLogEvent(event *dto.Event, dir *dto.TrafficLogDirective) } } - if len(dir.Labels) > 0 { - tl.Labels = dir.Labels + if len(dir.Properties) > 0 { + tl.Properties = dir.Properties } return tl From a8e42e97d9e368e83de5a90f10e39d2165c322d9 Mon Sep 17 00:00:00 2001 From: Dineth Date: Tue, 7 Jul 2026 19:42:52 +0530 Subject: [PATCH 12/21] Disable traffic logging by setting 'enabled' to false in config.toml --- gateway/configs/config.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gateway/configs/config.toml b/gateway/configs/config.toml index 6d46487587..0361cefb36 100644 --- a/gateway/configs/config.toml +++ b/gateway/configs/config.toml @@ -24,7 +24,7 @@ application_id = "" # Traffic-logging consumer: writes each collected event to stdout as a JSON line # (no external service needed). Enabling it activates the collector automatically. [traffic_logging] -enabled = true +enabled = false # Header names (case-insensitive) whose values are redacted as "****". masked_headers = ["authorization", "x-api-key", "x-jwt-assertion"] # Max bytes of request/response payload written to each log line (0 = no limit). From 07f4afe5ff990eeefc5b973aa7c6303e07a5a8d6 Mon Sep 17 00:00:00 2001 From: Dineth Date: Wed, 8 Jul 2026 10:50:15 +0530 Subject: [PATCH 13/21] Remove comments --- gateway/configs/config.toml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/gateway/configs/config.toml b/gateway/configs/config.toml index 0361cefb36..4be7780ef3 100644 --- a/gateway/configs/config.toml +++ b/gateway/configs/config.toml @@ -1,19 +1,9 @@ - -# The collector is the shared data-capture pipeline. It has no on/off switch of its -# own: it turns on automatically whenever a consumer (analytics or traffic_logging) -# is enabled, and stays off otherwise. This section only tunes what is captured. [collector] -# Capture request/response payloads (bodies) into the collected event (full bodies; -# per-consumer size caps are applied on output). send_request_body = false send_response_body = false -# Capture ALL request/response headers (redacted on output by per-consumer masking). -# Off by default; the Moesif publisher does not mask headers, so only enable this -# for consumers you know redact sensitive values (e.g. traffic_logging.masked_headers). send_request_headers = false send_response_headers = false -# Analytics consumer (e.g. Moesif). Enabling it activates the collector automatically. [analytics] enabled = true enabled_publishers = ["moesif"] @@ -21,13 +11,9 @@ enabled_publishers = ["moesif"] [analytics.publishers.moesif] application_id = "" -# Traffic-logging consumer: writes each collected event to stdout as a JSON line -# (no external service needed). Enabling it activates the collector automatically. [traffic_logging] enabled = false -# Header names (case-insensitive) whose values are redacted as "****". masked_headers = ["authorization", "x-api-key", "x-jwt-assertion"] -# Max bytes of request/response payload written to each log line (0 = no limit). max_payload_size = 2048 [router] From f7cbbac4c31044039c1970b80cb2c4e7910745d3 Mon Sep 17 00:00:00 2001 From: Dineth Date: Wed, 8 Jul 2026 12:27:39 +0530 Subject: [PATCH 14/21] Add per-flow excludeHeaders support for traffic logging - Introduced ExcludeHeaders field in TrafficLogFlow to drop specified headers entirely from emitted logs. - Updated maskHeaders function to handle per-flow exclusions. - Implemented dropHeaders function to remove headers based on ExcludeHeaders directive. - Enhanced log publishing tests to verify excludeHeaders functionality for both request and response flows. --- .../ANALYTICS_LOG_PUBLISHER_DESIGN.md | 366 ++++++++++++++++++ .../internal/analytics/dto/event.go | 7 + .../internal/analytics/publishers/log.go | 25 +- .../internal/analytics/publishers/log_test.go | 51 +++ .../analytics/publishers/traffic_log_event.go | 12 +- 5 files changed, 457 insertions(+), 4 deletions(-) create mode 100644 gateway/gateway-runtime/policy-engine/ANALYTICS_LOG_PUBLISHER_DESIGN.md diff --git a/gateway/gateway-runtime/policy-engine/ANALYTICS_LOG_PUBLISHER_DESIGN.md b/gateway/gateway-runtime/policy-engine/ANALYTICS_LOG_PUBLISHER_DESIGN.md new file mode 100644 index 0000000000..7639bb2c5d --- /dev/null +++ b/gateway/gateway-runtime/policy-engine/ANALYTICS_LOG_PUBLISHER_DESIGN.md @@ -0,0 +1,366 @@ +# Design: Collector & Stdout Traffic Logging + +> **Maintenance:** This document describes the shared **collector** (data-capture pipeline) +> and its stdout **traffic-logging** consumer, both built on the analytics pipeline. **Keep +> it in sync with the code — every change to this feature (new config, new capture point, +> behavior change) must update this document in the same PR**, including the Change Log at +> the bottom. + +_Last updated: 2026-07-07_ + +--- + +## 1. Problem we are trying to solve + +Operators running a **self-hosted** API Platform gateway want to observe API traffic — +who called what, status/latency, the request/response **headers**, and optionally the +**bodies** — as structured logs on **stdout** + +Constraints / sub-problems: + +- **Bodies and arbitrary headers cannot come from the Envoy access log.** Envoy's file + access logger has no body operator, and headers must each be named explicitly — there is + no "all headers" or "mask these" capability. So enriching `[router.access_logs]` is not + enough. +- **Stdout logging must not require "enabling analytics."** Data capture and external + analytics (Moesif) used to be a single `analytics.enabled` switch, so an operator who only + wanted stdout logs had to turn on the whole analytics feature. Capture is a *shared* + concern; the consumers (analytics, traffic logging) are independent. +- **Payload size control.** Bodies can be large; operators need a cap. +- **Sensitive data.** Header values such as `Authorization` must be redactable, both + globally and per-API. +- **Per-API opt-in, not all-or-nothing.** Emitting a stdout line for *every* API is too + noisy and rarely wanted. Operators need to log only specific APIs, chosen the same way + other behavior is chosen — by **attaching a policy** to those APIs — while still getting + the access-log-derived fields (latencies, status) that inline policies cannot see. + +## 2. Proposed Solution + +Split the existing analytics pipeline into a shared **collector** (data capture + transport) +and independent **consumers** that read what the collector gathers: + +- **`[collector]`** — the capture pipeline: the auto-injected analytics system policy + (captures headers/bodies) plus the Envoy→policy-engine ALS transport. It has **no on/off + flag of its own**: it is **implicit**, active whenever a consumer is enabled and off + otherwise (see `IsCollectorEnabled`). +- **`[analytics]`** — consumer 1: external analytics publishers (e.g. Moesif). The + collector captures data for **all** publishers uniformly; masking/redaction is each + publisher's own concern — the collector does not filter on a publisher's behalf (see D4 + in prior designs; Moesif receives raw headers regardless of `traffic_logging.masked_headers`). +- **`[traffic_logging]`** — consumer 2 (this feature): serializes a collected event to + stdout as a single JSON line. No external SaaS needed. It is **per-API opt-in**: a line + is emitted only for APIs that attach the **`log-message` policy with + `enableTrafficLogging: true`**. + +Capture settings (`send_*_body`, `send_*_headers`) live under `[collector]` and are shared +by all consumers — capture of each is **off by default**; operators enable what they need. +Output/presentation settings — `masked_headers` (redaction) and `max_payload_size` (payload +byte cap) — live under `[traffic_logging]` and apply only to the stdout publisher; other +consumers (e.g. Moesif) are unaffected. + +### Per-API gating: the `log-message` policy (`enableTrafficLogging`) + +The stdout publisher does **not** log every API. An API's traffic is logged only when a +two-way AND holds: `[traffic_logging].enabled` **and** the **`log-message` policy with +`enableTrafficLogging: true`** is attached to that API/route. (The collector is implicit — +enabling `[traffic_logging]` activates it automatically.) + +`log-message` (in `gateway/dev-policies/log-message`, v1.1+) has two modes. The default +(`enableTrafficLogging: false`, or absent) logs in real time during mediation via `slog` +(including per-chunk streaming) with no dependency on the collector. Setting +`enableTrafficLogging: true` turns the policy into a lightweight **signal**: it emits no log +line itself — latencies only exist at access-log time, beyond an inline policy's reach. +Instead, in `OnRequestHeaders` it stamps a marker (`AnalyticsMetadata["traffic_log"] = +`) carrying its per-API presentation config, and `Mode()` requests only the +request-header phase (no body buffering). The kernel merges that into `analytics_data`, +Envoy carries it onto the `HTTPAccessLogEntry`, and `prepareAnalyticEvent` reads it back +into `event.TrafficLog`. `Log.Publish` emits **only** when `event.TrafficLog != nil`, and +uses the directive to shape the line. Because capture happens globally in the collector +*before* any user policy runs, the directive can **filter/mask** what was captured but +cannot enable capture the collector skipped. + +### Policy parameters (`log-message` v1.1, `enableTrafficLogging`) + +Attach the policy to an API's definition YAML with `enableTrafficLogging: true` to opt that +API into stdout traffic logging. Both `request` and `response` are optional but at least +one, or `fields`, should be set for the line to carry anything beyond the base event fields. + +```yaml +policies: + request: + - policyName: log-message + policyVersion: v1.1.0 + parameters: + enableTrafficLogging: true # required to activate stdout traffic logging + request: + payload: true # include request body in the log line + headers: true # include request headers + response: + payload: false + headers: true + # Optional: additional header names (case-insensitive) redacted for this API, + # merged with the global traffic_logging.masked_headers. + maskedHeaders: + - x-internal-token + # Optional: extra key/value pairs added under the top-level "properties" object. + # "$ctx:" values are resolved from the request context at request time; other + # values are literal. + properties: + subject: "$ctx:auth.subject" + authType: "$ctx:auth.type" + tenant: "$ctx:auth.property.tenant" + env: prod + # Optional: fine-grained field projection over the emitted JSON line. + # When set, fields is authoritative — request/response payload & header booleans + # are ignored (maskedHeaders still applies). + fields: + exclude: + - requestBody +``` + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `enableTrafficLogging` | bool | `false` | **Must be `true`** to opt in. When `false` (default), the policy logs inline in real time via `slog` during mediation — independent of the collector. | +| `request.payload` | bool | `false` | Include the request body. No-op if `[collector].send_request_body = false`. Ignored when `fields` is set. | +| `request.headers` | bool | `false` | Include request headers. No-op if `[collector].send_request_headers = false`. Ignored when `fields` is set. | +| `response.payload` | bool | `false` | Include the response body. No-op if `[collector].send_response_body = false`. Ignored when `fields` is set. | +| `response.headers` | bool | `false` | Include response headers. No-op if `[collector].send_response_headers = false`. Ignored when `fields` is set. | +| `maskedHeaders` | `[]string` | `[]` | Additional header names (case-insensitive) redacted to `****` in `requestHeaders`/`responseHeaders`, merged with the global `traffic_logging.masked_headers`. Always applied, even when `fields` is set. To drop a header entirely instead of redacting it, use a dotted `fields.exclude` path (e.g. `requestHeaders.X-Secret`). | +| `properties` | `object` | `{}` | Extra key→value pairs emitted under a top-level `properties` object. String values prefixed `$ctx:` are resolved from the request context at request time (see below); non-`$ctx` strings are literals; non-string values pass through as-is. Unresolvable `$ctx:` refs are skipped. | +| `fields.only` | `[]string` | `[]` | Keep only the listed keys (top-level, e.g. `latencies`, `requestHeaders`, `properties`, or dotted sub-paths, e.g. `requestHeaders.authorization`, `properties.env`). Authoritative over `request`/`response` payload & header booleans. If both `only` and `exclude` are set, `only` wins. | +| `fields.exclude` | `[]string` | `[]` | Drop the listed keys/dotted sub-paths and keep everything else. | + +**`properties` `$ctx:` references** (resolved in the policy's `OnRequestHeaders`, so only request-phase context is reachable; fixed names are case-insensitive, `auth.property.` is case-sensitive): + +| Reference | Resolves to | +|---|---| +| `$ctx:request.path` / `.method` / `.authority` / `.scheme` / `.vhost` | request line fields | +| `$ctx:request.id` | correlation request ID (`RequestHeaderContext.RequestID`) | +| `$ctx:request.header.` | first value of that request header (skipped if absent) | +| `$ctx:api.id` / `.name` / `.version` / `.context` / `.kind` / `.operation_path` | API metadata | +| `$ctx:project.id` | project the API belongs to | +| `$ctx:auth.subject` / `.type` / `.issuer` / `.credential_id` / `.token_id` | `AuthContext` string fields (`token_id` = jwt `jti`, skipped when empty) | +| `$ctx:auth.authenticated` / `.authorized` | `"true"` / `"false"` | +| `$ctx:auth.audience` | `AuthContext.Audience` joined by `,` | +| `$ctx:auth.scopes` | granted scopes, space-joined, sorted | +| `$ctx:auth.property.` | `AuthContext.Properties[""]` (custom claims) | + +> `auth.*` references require an authentication policy to have run **before** `log-message` in the request-header phase; when `AuthContext` is nil they resolve to nothing and the property is skipped. `properties` is only meaningful with `enableTrafficLogging: true` (ignored in inline mode). + +> **Precedence:** `fields` (when present) overrides `request.payload`, `request.headers`, +> `response.payload`, and `response.headers`. `maskedHeaders` (per-API, merged with the +> global `traffic_logging.masked_headers`) always applies regardless. +> +> **Capture limitation:** the directive can only filter/mask what the collector already +> captured. `request.payload: true` has no effect if `[collector].send_request_body = false`. + +### Data flow (gating in **bold**) + +``` +client → Envoy (ALS cluster + gRPC access log attached when COLLECTOR enabled) + │ ext_proc → policy-engine system policies + │ • collector system policy (injected when COLLECTOR enabled) captures + │ headers/body into dynamic metadata "analytics_data" + ▼ + Envoy gRPC ALS ── HTTPAccessLogEntry ──▶ policy-engine ALS server + (started when COLLECTOR enabled) + │ + ▼ + Analytics.Process(entry) + • prepareAnalyticEvent → dto.Event + • for each registered publisher: Publish(event) + │ + ┌───────────────────────┴───────────────────────┐ + ▼ ▼ + Moesif publisher (ANALYTICS enabled) Log publisher (TRAFFIC_LOGGING enabled) + (logs every event, raw headers) **if event.TrafficLog == nil → skip** + (per-API: set only when log-message + with enableTrafficLogging is attached) + → shape per directive (filter/mask, + truncate to max_payload_size) + → JSON line (incl. latencies) → stdout +``` + +The `log-message` policy (with `enableTrafficLogging: true`) runs on the ext_proc side +(above, "collector system policy" step): it stamps `analytics_data["traffic_log"]` +alongside the captured headers/bodies, so by the time the entry reaches +`prepareAnalyticEvent` the opt-in marker is present. + +## 3. Test scenarios + +| Area | Scenario | Where | +|---|---|---| +| Log publisher | Per-API gate: an event **without** `TrafficLog` is not logged | `publishers/log_test.go: TestLog_Publish_SkipsWhenNoDirective` | +| Log publisher | Emits one JSON line incl. ALS-derived `latencies`; event fields present | `TestLog_Publish_WritesJSONLineWithLatencies` | +| Log publisher | `headers:false`/nil flow omits the property; `payload:false` omits the payload | `TestLog_Publish_DisabledFieldsOmitted` | +| Directive extraction | `prepareAnalyticEvent` reads `traffic_log` marker into `event.TrafficLog` (nil when absent; opts in with defaults when malformed); never leaks into `Properties` | `analytics_test.go: TestPrepareAnalyticEvent_TrafficLogMarker`, `_NoTrafficLogMarker`, `_MalformedTrafficLogMarker` | +| log-message (traffic logging) | `enableTrafficLogging` parsing, `Mode()` (request-header only in that mode), marker JSON from params (incl. `fields`, `properties`, `maskedHeaders`), no inline log, safe degradation | `gateway/dev-policies/log-message/logmessage_test.go` | +| `properties` `$ctx:` | `resolveContextValue` resolves request/api/auth refs (incl. `auth.property.`, joined audience/scopes); literals pass through; unresolvable/nil-auth refs skipped; non-string passthrough; empty result omits the marker field | policy `logmessage_test.go: TestResolveContextValue`, `_NilAuthSkipped`, marker-build tests | +| Properties emission | resolved properties appear under a top-level `properties` object; absent when the directive has none; projectable via `fields` (`properties`/`properties.`) | `publishers/log_test.go: TestLog_Publish_PropertiesTopLevel`, `_NoPropertiesWhenAbsent`, `_PropertiesProjectableViaFields` | +| Field projection | `fields.only` keeps only named keys/dotted sub-paths; `fields.exclude` drops them; authoritative over presence (flow booleans ignored) while `maskedHeaders` still applies | `publishers/log_test.go: TestLog_Publish_FieldsInclude`, `_FieldsExclude`, `_FieldsIncludeRequestBodyAndProperties` | +| Log publisher | `masked_headers` (global) redacts values to `****`, case-insensitive, for request & response headers | `TestLog_Publish_MasksHeaders` | +| Log publisher | Per-API `maskedHeaders` merges with the global list (with and without a global list configured) | `TestLog_Publish_PerAPIMaskedHeadersMergedWithGlobal`, `_PerAPIMaskedHeadersNoGlobal` | +| Log publisher | A dotted `fields.exclude` path (e.g. `requestHeaders.X-Secret`) drops a header entirely, vs. masking which redacts it | `TestLog_Publish_ExcludeHeadersDrops` | +| Log publisher | Masking does **not** mutate the shared event seen by other publishers | `TestLog_Publish_DoesNotMutateSharedEvent` | +| Log publisher | `nil` event is a no-op (no panic, no output) | `TestLog_Publish_NilEvent` | +| Log publisher | A header value that is not valid JSON is passed through unchanged / dropped | `TestLog_Publish_UnparseableHeadersDropped` | +| Log publisher | `NewLog(nil)` returns a usable publisher with no masked headers | `TestNewLog_NilConfig` | +| Publisher wiring | `traffic_logging.enabled` registers the stdout publisher | `analytics_test.go: TestNewAnalytics_TrafficLoggingEnabled` | +| Collector (implicit) | `IsCollectorEnabled()` is false with no consumer, true when analytics or traffic logging is on; config validates in each case | engine `config_test.go: TestIsCollectorEnabled`; controller `config_test.go: TestConfig_IsCollectorEnabled` | +| Deprecated aliases | `analytics.allow_payloads`/`send_*_body` migrate onto `collector.send_*_body`; directional flags win; skipped when analytics is disabled | `TestValidate_AnalyticsPayloadMigration(_SkippedWhenAnalyticsDisabled)` (engine), `TestConfig_ValidateAnalyticsPayloadMigration(_SkippedWhenAnalyticsDisabled)` (controller) | +| Deprecated transport aliases | `analytics.access_logs_service`/`grpc_event_server` migrate onto `collector.als` only while analytics is enabled, so a traffic-logging-only deployment never re-applies a stale analytics override | `TestValidate_AnalyticsTransportMigration_SkippedWhenAnalyticsDisabled` (engine), `TestConfig_ValidateAnalyticsTransportMigration_SkippedWhenAnalyticsDisabled` (controller) | +| Max payload (publisher) | Log publisher truncates `request_payload`/`response_payload` to `traffic_logging.max_payload_size` (0 = no limit) | `publishers/log_test.go: TestLog_Publish_TruncatesPayload`, `_NoTruncationWhenZero`; validation: `config_test.go: TestValidate_TrafficLoggingMaxPayloadSize` | +| Collector gating | system policy injected only when a consumer is enabled (`IsCollectorEnabled`); capture flags (body and header, both off by default) propagate from `[collector]` | `gateway-controller .../system_policies_test.go: TestInjectSystemPolicies_Collector*`, `_BodyFlagsPropagated`, `_BodyFlagsDefaultFalse`, `_HeaderFlagsPropagated`, `_HeaderFlagsDefaultFalse`, `_TrafficLoggingOnlyEnablesCollector` | +| Header capture | `getHeaderFlags` parses bool/string forms; `serializeHeaders` emits a JSON object (lower-cased keys, multi-value joined) and "" for empty | `system-policies/analytics/analytics_headers_test.go` | + +**Manual / end-to-end (not yet automated):** with `traffic_logging.enabled = true`, +`analytics.enabled = false` (the collector activates implicitly), and +`collector.send_request_body`/`send_response_body`/`send_request_headers`/`send_response_headers` +turned on, deploy two APIs and attach the **`log-message`** policy with +**`enableTrafficLogging: true`** to only one. A request to the API with the policy produces +one JSON line on the **policy-engine** stdout containing the latencies, headers (auth +masked), and body (≤ cap), with **no** Moesif activity; a request to the API **without** the +policy produces **no** line; disabling `traffic_logging.enabled` (with analytics also off) +produces no lines even with the policy attached — the collector goes inactive. + +## 4. Implementation + +| Component | File | Responsibility | +|---|---|---| +| Log publisher | `internal/analytics/publishers/log.go` | `Log` implements `Publisher.Publish`; **per-API gate** (`event.TrafficLog == nil → skip`); `toTrafficLogEvent` (in `traffic_log_event.go`) shapes the per-flow headers/payload per the directive, merges per-API `MaskedHeaders` with the global config list, and truncates to `traffic_logging.max_payload_size` (0 = no limit) on a locally-built `TrafficLogEvent`, never mutating the shared `dto.Event`/`Properties`. When `Fields` is set, `applyFieldsProjection` shallow-decodes the marshaled line into `map[string]json.RawMessage` and projects to (`Only`) or without (`Exclude`) the named top-level keys / dotted sub-paths — decoding and re-encoding only the touched nested objects, leaving untouched top-level fields as raw bytes. `sync.Mutex` serializes writes; `NewLog(*config.TrafficLoggingConfig)` | +| Publisher selection | `internal/analytics/analytics.go` | `NewAnalytics` registers Moesif when `analytics.enabled` and the stdout `Log` publisher when `traffic_logging.enabled` (independently) | +| Event enrichment | `internal/analytics/analytics.go` | `prepareAnalyticEvent` attaches `requestHeaders`/`responseHeaders` and `request_payload`/`response_payload`; reads the `traffic_log` marker and parses it per-request via `parseTrafficLogDirective` (no caching — the marker's `Properties` are already request-specific, so caching by raw JSON would leak one API's properties into another request that happens to share the same static portion) into `event.TrafficLog` (gating state, kept off `Properties`) | +| log-message policy (`enableTrafficLogging`) | `gateway/dev-policies/log-message/logmessage.go` (v1.1+, in `build-manifest.yaml`/`build.yaml`) | `enableTrafficLogging: true` turns the policy into a per-API opt-in signal: `Mode()` = request-header phase only; `OnRequestHeaders` → `stampTrafficLogMarker` stamps `AnalyticsMetadata["traffic_log"] = ` (request/response `payload`/`headers`, `fields`, `properties`, `maskedHeaders`). `buildProperties`/`resolveContextValue` expand `$ctx:` refs in `properties` at request time and bake the resolved values into the marker. `false` (default) keeps the original real-time slog logging. Registered via build-manifest | +| Traffic-log directive | `internal/analytics/dto/event.go` | `Event.TrafficLog *TrafficLogDirective` (`json:"-"`); `TrafficLogDirective{Request,Response *TrafficLogFlow; Fields *TrafficLogFields; Properties map[string]interface{}; MaskedHeaders []string}`, `TrafficLogFlow{Payload,Headers bool}`, `TrafficLogFields{Only,Exclude []string}` | +| Engine ALS startup | `cmd/policy-engine/main.go` | ALS gRPC server started when `IsCollectorEnabled()` (a consumer is on) | +| Policy-engine config | `internal/config/config.go` | `CollectorConfig{SendRequestBody, SendResponseBody, AccessLogsServiceCfg}` (no `Enabled` — implicit; no header flags — header capture is a controller-side/capture-time concern); `IsCollectorEnabled() = Analytics.Enabled \|\| TrafficLogging.Enabled`; `TrafficLoggingConfig{Enabled, MaskedHeaders, MaxPayloadSize}`; `validateCollectorConfig` (deprecated-alias migration for body flags **and** `access_logs_service`, the latter only while `Analytics.Enabled`; validates the ALS receiver when the collector is active); `traffic_logging.max_payload_size >= 0` validated in `Validate` | +| Controller config | `gateway-controller/pkg/config/config.go` | `CollectorConfig{SendRequest/ResponseBody, SendRequest/ResponseHeaders, GRPCEventServerCfg}` (no `Enabled`; body and header capture both default **false** — off until an operator opts in); `TrafficLoggingConfig{Enabled}` (so the controller knows traffic logging activates the collector); `IsCollectorEnabled() = Analytics.Enabled \|\| TrafficLogging.Enabled`; `validateCollectorConfig` (alias migration for body flags **and** `grpc_event_server`, the latter only while `Analytics.Enabled`; validates the ALS sink via `validateGRPCEventServerConfig` when the collector is active) | +| Controller gating | `gateway-controller/pkg/xds/translator.go` | ALS cluster + gRPC access-log config attached when `IsCollectorEnabled()`, built from `Collector.GRPCEventServerCfg` | +| Engine ALS receiver | `internal/utils/access_logger_server.go` | Reads `Collector.AccessLogsServiceCfg` (mode, port, TLS, message/header limits) to start the ALS gRPC server | +| System-policy wiring | `gateway-controller/pkg/utils/system_policies.go` | The analytics (collector) system policy is injected when `IsCollectorEnabled()`; propagates `send_*_body`, `send_*_headers` from `[collector]` into policy params | +| Analytics system policy | `gateway/system-policies/analytics/analytics.go` | Captures headers (`serializeHeaders`, gated by `getHeaderFlags`, off by default) into `request_headers`/`response_headers` and request/response bodies (gated by the body flags, off by default) at all capture points (request, buffered response, streaming chunks). No size cap at capture — payload truncation is output-side in the Log publisher | +| Config docs | `gateway/configs/config-template.toml`, `config.toml` | Document/example `[collector]`, `[analytics]`, `[traffic_logging]` | + +### Configuration surface + +```toml +[collector] # implicit — shared capture pipeline + transport (no enabled flag) +send_request_body = false # capture bodies in full (controller→capture AND engine→attach); off by default +send_response_body = false +send_request_headers = false # capture ALL headers (capture side, controller only); off by default +send_response_headers = false + +# ALS transport tuning (advanced; sensible defaults). One shared section read by +# BOTH ends: the controller (Envoy's gRPC ALS sink — buffers, ports, TLS) and the +# policy-engine (the receiving ALS server — mode, port, limits). Envoy-sender-only +# keys (buffer_*, grpc_request_timeout) are ignored by the engine. +[collector.als] + +[analytics] # consumer 1 — enabling it activates the collector +enabled = false +enabled_publishers = ["moesif"] + +[traffic_logging] # consumer 2 — enabling it activates the collector +enabled = true +masked_headers = ["authorization"] # output-side redaction (log publisher only) +max_payload_size = 2048 # 0 = no limit; truncates the stdout line only +``` + +**Where each key is read (two-sided config note):** + +- `collector.send_*_body` — **both** sides: controller config drives the system policy to + *capture* the (full) payload; policy-engine config drives `prepareAnalyticEvent` to *attach* it. +- `collector.send_*_headers` — **capture side only** (controller config → system policy param). +- `traffic_logging.max_payload_size` — **policy-engine** config, applied output-side by the + Log publisher (per-consumer; other consumers unaffected). +- `collector.als` — **both** sides (one shared section): the controller reads it to tune + Envoy's gRPC ALS sink (validated in the controller's `validateCollectorConfig`), and the + policy-engine reads it to configure/validate its receiving ALS server. Shared keys + (`server_port`, TLS, `max_message_size`/`max_header_limit`) must agree on both ends; + Envoy-sender-only keys (`buffer_*`, `grpc_request_timeout`) are ignored by the engine. +- collector activation — **derived on both sides** from the consumer flags + (`IsCollectorEnabled() = analytics.enabled || traffic_logging.enabled`): the controller + injects the system policy + ALS sink and the engine starts the ALS server when it is true. + There is no `collector.enabled` key. +- `traffic_logging.enabled` / `masked_headers` — **policy-engine** config, consumed by the + log publisher. Note `enabled` only *arms* the publisher; whether a given API is logged is + decided per-API by the marker (`event.TrafficLog`) from the attached `log-message` policy + with `enableTrafficLogging: true`. +- `log-message` **policy with `enableTrafficLogging: true`** (per-API attachment) — + data-plane opt-in: its `OnRequestHeaders` stamps the `analytics_data["traffic_log"]` + marker read back in `prepareAnalyticEvent`. + +## 4a. Configuration changes & backward compatibility + +Relative to the previous release, this PR **adds** the `[collector]`, `[collector.als]`, and +`[traffic_logging]` sections and turns `[analytics]` into a consumer of an **implicit** +collector (the collector has no `enabled` flag — it activates whenever a consumer is on). +The only pre-existing (released) `[analytics]` keys affected are the payload-capture flags +and the ALS transport tuning, which are **deprecated and migrated** onto `[collector]`. + +**Which side reads each section:** `[collector]` capture flags split by side — the +controller owns header capture (`send_*_headers`), the engine owns body-attach +(`send_*_body`); `[collector.als]` is read by **both**; collector *activation* is derived on +both sides from the consumer flags; `[traffic_logging]` (including its output caps) is +**engine-only**, except `[traffic_logging].enabled`, which the controller also reads so it +can activate the collector when only traffic logging is on. + +### Added + +| Key | Read by | Purpose | +|---|---|---| +| `[collector].send_request_body` / `send_response_body` | both | Capture (controller) + attach (engine) request/response bodies (in full). Default `false`. | +| `[collector].send_request_headers` / `send_response_headers` | controller | Capture all request/response headers. Default `false`. | +| `[collector.als].*` | both | ALS transport tuning (one shared section — see §4) | +| `[traffic_logging].enabled` | both | Arm the stdout traffic-logging publisher (engine); activate the collector (controller) | +| `[traffic_logging].masked_headers` | engine | Output-side header value redaction | +| `[traffic_logging].max_payload_size` | engine | Truncate the logged payload (`0` = no limit); output-side, publisher-only | + +### Deprecated & migrated (from `[analytics]`) + +These released `[analytics]` keys still work: they are migrated onto `[collector]` at load +time with a `slog.Warn`, so existing configs keep functioning unchanged. + +| Deprecated key | Migrated onto | Notes | +|---|---|---| +| `analytics.grpc_event_server.*` (controller) | `collector.als.*` | Whole-struct migration; only applied when `[collector.als]` is still at its default **and** `analytics.enabled = true`, so an explicit `[collector.als]` is never clobbered, and a traffic-logging-only deployment never re-applies a stale analytics override | +| `analytics.access_logs_service.*` (engine) | `collector.als.*` | Same whole-struct migration, same `analytics.enabled` guard | +| `analytics.allow_payloads` | `collector.send_request_body` + `collector.send_response_body` | Directional flags win over `allow_payloads` | +| `analytics.send_request_body` / `send_response_body` | `collector.send_*_body` | — | + +### Backward-compatibility verdict + +- **Deprecated `[analytics]` keys — backward compatible.** The keys above are migrated onto + `[collector]` at load with a deprecation warning; nothing an operator set previously stops + working. +- **Implicit collector — upgrades are zero-touch, and there is no `collector.enabled` key.** + A previously-valid config that sets `analytics.enabled = true` keeps working: the collector + activates automatically because a consumer is on (`IsCollectorEnabled`). A stale + `collector.enabled = true` left in an old config is harmless — the key is no longer bound, + so koanf ignores it. Nothing an operator must edit on upgrade. + +## 5. Open items / future considerations + +- Automated end-to-end test (containerized run) is not yet in place; covered manually. +- No `max_headers` / allow-list cap on header capture (all captured headers are attached); + add if header cardinality becomes a concern. +- Cross-publisher masking is out of scope: masking/redaction is a per-publisher presentation + concern, not a collector guarantee. The collector hands every publisher the same raw + captured data; if Moesif is enabled alongside traffic logging, it receives unmasked + headers regardless of `traffic_logging.masked_headers` or a policy's `maskedHeaders`. If + cross-publisher masking is ever needed, it would have to move to capture time. + +## Change Log + +| Date | Change | +|---|---| +| 2026-07-02 | Initial version — PR "Add Access Logging support". Introduced the shared **`[collector]`** capture pipeline (request/response header & body capture, plus ALS transport tuning under **`[collector.als]`**) as a prerequisite for its consumers. `[analytics]` (Moesif) became an explicit consumer; its released capture/transport keys (`allow_payloads`, `send_*_body`, `grpc_event_server`/`access_logs_service`) are deprecated and migrated onto `[collector]` with warnings (§4a). Added **`[traffic_logging]`**, a stdout JSON consumer that is **per-API**: the `log-message` policy stamps a `traffic_log` marker that gates and shapes the emitted line — including a `fields` include/exclude projection and header masking — enriched with ALS-derived latencies. | +| 2026-07-03 | Simplified the collector config. **Removed `collector.enabled`** — the collector is now **implicit**, derived from whether a consumer is enabled (`IsCollectorEnabled() = analytics.enabled \|\| traffic_logging.enabled`); the controller gained a minimal `[traffic_logging].enabled` binding so it activates the collector when only traffic logging is on. **Renamed the `log-message` activation param** from `destination: inline\|access-log` to the boolean **`enableTrafficLogging`** (default `false`; `true` ≡ the old `access-log`). Per-API stdout logging is now a two-way AND (publisher + policy). | +| 2026-07-03 | Added **`customProperties`** to `log-message` (traffic-logging mode): user-defined key→value pairs, later renamed to **`labels`** (see 2026-07-07), resolved from `$ctx:`-prefixed request-context references and emitted on the log line. Resolved values are baked into the `traffic_log` marker. | +| 2026-07-03 | Expanded the request-context `$ctx:` surface with five more request-phase scalars: `request.vhost`, `request.id`, `api.kind`, `api.operation_path`, and `project.id`. The inter-policy `Metadata` map and all response-phase fields remain intentionally unexposed (the latter is a hard phase limit — the marker is stamped at request-header time). | +| 2026-07-03 | Audited the `$ctx:` surface against the latest SDK (`sdk/core` v0.2.16). Bumped `log-message` from v0.2.4 → v0.2.16 and exposed `$ctx:auth.token_id` (jwt `jti`, added in v0.2.15). | +| 2026-07-07 | Header capture flags (`collector.send_request_headers` / `send_response_headers`) now default to **`false`**, matching body capture — nothing is captured until an operator opts in. Renamed `customProperties` → **`labels`** and dropped the separate per-flow `excludeHeaders` param in favor of a single per-API **`maskedHeaders`** list (merged with the global `traffic_logging.masked_headers`); dropping a header entirely (rather than redacting it) is done via a dotted `fields.exclude` path. `TrafficLogDirective` gained `MaskedHeaders`. | +| 2026-07-07 | Removed the per-request `directiveCache` from `analytics.go`: the marker's resolved `Labels` are request-specific, so caching the parsed `TrafficLogDirective` by raw JSON string risked leaking one request's labels onto another that happened to produce the same cache key. `parseTrafficLogDirective` now parses the marker fresh on every request. | +| 2026-07-07 | Log publisher field projection (`applyFieldsProjection`) now shallow-decodes the marshaled line into `map[string]json.RawMessage` instead of a full `map[string]interface{}`, and only decodes/re-encodes the specific nested objects a dotted `fields` path references — avoiding a deep unmarshal/remarshal of the whole event on every projected line. | +| 2026-07-07 | Renamed the `log-message` **`labels`** param to **`properties`** (and the emitted top-level log-line key, the marker key, `TrafficLogDirective.Properties`, and the policy's `buildProperties`) — a straight rename of the concept, no behavioral change. `fields` dotted paths are now `properties.`. | +| 2026-07-07 | Gave the traffic log its own **microsecond** latencies block (`dto.TrafficLogLatencies`: `durationUs`, `requestMediationLatencyUs`, `responseMediationLatencyUs`, `backendLatencyUs`), computed from the ALS `CommonProperties` timepoints at full precision via a new `toUs` in `analytics.go`. It no longer embeds the Moesif-oriented `dto.Latencies` (milliseconds), so the Moesif-oriented `*Latency` fields no longer leak into the traffic-log line and Moesif's millisecond units are unchanged. Dropped the now-unused millisecond `Duration`/`BackendProcDuration`/`ResponseProcDuration` fields (and `Get/SetDuration`) from `dto.Latencies`. | diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go index b605ade6e3..7cc1c2f158 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go @@ -82,6 +82,13 @@ type TrafficLogDirective struct { type TrafficLogFlow struct { Payload bool `json:"payload"` Headers bool `json:"headers"` + // ExcludeHeaders lists header names (case-insensitive) dropped entirely from + // this flow's headers in the emitted line. It is orthogonal to Fields and to + // masking: it always applies when the flow's headers are present, and removes + // the header rather than redacting it (use the global/per-API masked headers to + // redact instead). The log-message policy lower-cases these names when stamping + // the marker; the publisher matches case-insensitively regardless. + ExcludeHeaders []string `json:"excludeHeaders,omitempty"` } // TrafficLogFields selects which fields appear in the emitted line. Exactly one diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go index be06232e13..c3982ce8ca 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go @@ -229,8 +229,9 @@ func filterNestedKeys(m map[string]json.RawMessage, top string, keep func(string } // maskHeaders redacts header values whose names appear in mask (case-insensitive). -// Returns a new map; the input is not modified. Per-header exclusion is handled -// downstream by applyFieldsProjection via dotted paths (e.g. "requestHeaders.authorization"). +// Returns a new map; the input is not modified. To drop a header entirely rather +// than redacting its value, use the per-flow excludeHeaders directive (see +// dropHeaders) or a dotted fields.exclude path (e.g. "requestHeaders.authorization"). func (l *Log) maskHeaders(headers map[string]string, mask map[string]bool) map[string]string { result := make(map[string]string, len(headers)) for name, value := range headers { @@ -242,3 +243,23 @@ func (l *Log) maskHeaders(headers map[string]string, mask map[string]bool) map[s } return result } + +// dropHeaders deletes, in place, any header whose name (case-insensitive) appears +// in exclude. It is a no-op when exclude is empty. Unlike masking (which redacts +// the value) this removes the key entirely, matching the log-message policy's +// per-flow excludeHeaders semantics. Callers pass the freshly-built, non-shared +// map returned by maskHeaders, so in-place mutation never affects other publishers. +func dropHeaders(headers map[string]string, exclude []string) { + if len(exclude) == 0 { + return + } + drop := make(map[string]bool, len(exclude)) + for _, h := range exclude { + drop[strings.ToLower(strings.TrimSpace(h))] = true + } + for name := range headers { + if drop[strings.ToLower(name)] { + delete(headers, name) + } + } +} diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go index 08249f48ac..992c929422 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go @@ -253,6 +253,57 @@ func TestLog_Publish_ExcludeHeadersDrops(t *testing.T) { assert.Equal(t, "bar", reqH["x-foo"]) } +// Per-flow excludeHeaders drops a header entirely (case-insensitive), in both the +// request and response flows, while masking still redacts other headers. This is +// the traffic-logging counterpart of the inline excludeHeaders param. +func TestLog_Publish_PerFlowExcludeHeadersDrops(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{MaskedHeaders: []string{"authorization"}}) + event := createBaseEvent() + event.TrafficLog = &dto.TrafficLogDirective{ + // Directive carries lower-cased names (as the policy stamps them); the + // captured header keys use mixed case to prove case-insensitive matching. + Request: &dto.TrafficLogFlow{Headers: true, ExcludeHeaders: []string{"x-secret"}}, + Response: &dto.TrafficLogFlow{Headers: true, ExcludeHeaders: []string{"set-cookie"}}, + } + event.Properties["requestHeaders"] = `{"Authorization":"Bearer s","X-Secret":"top","x-foo":"bar"}` + event.Properties["responseHeaders"] = `{"Set-Cookie":"sid=1","x-bar":"baz"}` + + l.Publish(event) + + decoded := decodeLine(t, read()) + + reqH := headerMap(t, decoded["requestHeaders"]) + _, hasSecret := reqH["X-Secret"] + assert.False(t, hasSecret, "excluded request header must be dropped entirely") + assert.Equal(t, "****", reqH["Authorization"], "masked header still redacted") + assert.Equal(t, "bar", reqH["x-foo"], "untouched header retained") + + resH := headerMap(t, decoded["responseHeaders"]) + _, hasCookie := resH["Set-Cookie"] + assert.False(t, hasCookie, "excluded response header must be dropped entirely") + assert.Equal(t, "baz", resH["x-bar"], "untouched response header retained") +} + +// Per-flow excludeHeaders must not mutate the shared event, so other publishers +// (e.g. Moesif) still see the full captured header set. +func TestLog_Publish_PerFlowExcludeHeadersDoesNotMutateSharedEvent(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.TrafficLog = &dto.TrafficLogDirective{ + Request: &dto.TrafficLogFlow{Headers: true, ExcludeHeaders: []string{"x-secret"}}, + } + const raw = `{"X-Secret":"top","x-foo":"bar"}` + event.Properties["requestHeaders"] = raw + + l.Publish(event) + + decoded := decodeLine(t, read()) + reqH := headerMap(t, decoded["requestHeaders"]) + _, hasSecret := reqH["X-Secret"] + assert.False(t, hasSecret, "header dropped from the emitted line") + assert.Equal(t, raw, event.Properties["requestHeaders"], "shared event.Properties must be untouched") +} + // headers:false omits the headers property; a nil flow omits its whole side. func TestLog_Publish_DisabledFieldsOmitted(t *testing.T) { l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/traffic_log_event.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/traffic_log_event.go index 3e1e203091..81e531e976 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/traffic_log_event.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/traffic_log_event.go @@ -165,7 +165,11 @@ func (l *Log) toTrafficLogEvent(event *dto.Event, dir *dto.TrafficLogDirective) if raw, ok := event.Properties[dto.PropKeyRequestHeaders].(string); ok { if hasFieldsSelection || (dir.Request != nil && dir.Request.Headers) { if headers := parseHeadersFromString(raw); headers != nil { - tl.RequestHeaders = l.maskHeaders(headers, mask) + masked := l.maskHeaders(headers, mask) + if dir.Request != nil { + dropHeaders(masked, dir.Request.ExcludeHeaders) + } + tl.RequestHeaders = masked } } } @@ -179,7 +183,11 @@ func (l *Log) toTrafficLogEvent(event *dto.Event, dir *dto.TrafficLogDirective) if raw, ok := event.Properties[dto.PropKeyResponseHeaders].(string); ok { if hasFieldsSelection || (dir.Response != nil && dir.Response.Headers) { if headers := parseHeadersFromString(raw); headers != nil { - tl.ResponseHeaders = l.maskHeaders(headers, mask) + masked := l.maskHeaders(headers, mask) + if dir.Response != nil { + dropHeaders(masked, dir.Response.ExcludeHeaders) + } + tl.ResponseHeaders = masked } } } From 2ba49121aca655f99b42ad0a8ee84e2b93cdc722 Mon Sep 17 00:00:00 2001 From: Dineth Date: Wed, 8 Jul 2026 14:24:58 +0530 Subject: [PATCH 15/21] revert --- .../policy-engine/internal/analytics/dto/latencies.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/latencies.go b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/latencies.go index 384d31706e..df1e98ebbd 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/latencies.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/latencies.go @@ -26,6 +26,7 @@ type Latencies struct { BackendLatency int64 `json:"backendLatency"` RequestMediationLatency int64 `json:"requestMediationLatency"` ResponseMediationLatency int64 `json:"responseMediationLatency"` + Duration int64 `json:"duration"` } // TrafficLogLatencies holds gateway/backend timings for a traffic-log event, in @@ -88,3 +89,11 @@ func (l *Latencies) GetResponseMediationLatency() int64 { func (l *Latencies) SetResponseMediationLatency(responseMediationLatency int64) { l.ResponseMediationLatency = responseMediationLatency } + +func (l *Latencies) GetDuration() int64 { + return l.Duration +} + +func (l *Latencies) SetDuration(duration int64) { + l.Duration = duration +} \ No newline at end of file From 7248cd9b740fdecf5ef6cc33cef62c440972e221 Mon Sep 17 00:00:00 2001 From: Dineth Date: Thu, 9 Jul 2026 10:30:21 +0530 Subject: [PATCH 16/21] refactor: update payload and header capture parameters in collector configuration - Renamed parameters for request and response body capture from `send_request_body` and `send_response_body` to `request_body` and `response_body` respectively in the collector configuration. - Updated related code and tests to reflect the new parameter names across various components including system policies, analytics, and configuration validation. - Adjusted logging and error messages to align with the new parameter naming conventions. - Ensured backward compatibility by migrating deprecated aliases during validation. --- common/collector/collector.go | 12 +-- common/collector/collector_test.go | 2 +- gateway/configs/config-template.toml | 12 +-- gateway/configs/config.toml | 8 +- .../gateway-controller/pkg/config/config.go | 48 +++++----- .../pkg/config/config_test.go | 88 +++++++++---------- .../pkg/utils/system_policies.go | 8 +- .../pkg/utils/system_policies_test.go | 28 +++--- .../gateway-controller/pkg/xds/translator.go | 8 +- .../pkg/xds/translator_test.go | 10 +-- .../policy-engine/cmd/policy-engine/main.go | 2 +- .../internal/analytics/analytics.go | 4 +- .../internal/analytics/analytics_test.go | 18 ++-- .../internal/analytics/publishers/log.go | 9 +- .../internal/analytics/publishers/log_test.go | 35 ++++++++ .../analytics/publishers/traffic_log_event.go | 47 ++++++++-- .../policy-engine/internal/config/config.go | 46 +++++----- .../internal/config/config_test.go | 46 +++++----- .../internal/utils/access_logger_server.go | 18 ++-- .../utils/access_logger_server_test.go | 2 +- .../system-policies/analytics/analytics.go | 10 +-- .../analytics/analytics_headers_test.go | 6 +- .../analytics/policy-definition.yaml | 14 +-- 23 files changed, 274 insertions(+), 207 deletions(-) diff --git a/common/collector/collector.go b/common/collector/collector.go index 307288301e..5d5e7d8004 100644 --- a/common/collector/collector.go +++ b/common/collector/collector.go @@ -61,15 +61,15 @@ func MigrateDeprecatedCapture(analyticsEnabled bool, deprecated CaptureFlags, co return } if deprecated.SendRequestBody && !*collectorSendRequestBody { - slog.Warn("analytics.send_request_body is deprecated; use collector.send_request_body instead") + slog.Warn("analytics.send_request_body is deprecated; use collector.request_body instead") *collectorSendRequestBody = true } if deprecated.SendResponseBody && !*collectorSendResponseBody { - slog.Warn("analytics.send_response_body is deprecated; use collector.send_response_body instead") + slog.Warn("analytics.send_response_body is deprecated; use collector.response_body instead") *collectorSendResponseBody = true } if deprecated.AllowPayloads { - slog.Warn("analytics.allow_payloads is deprecated; use collector.send_request_body and collector.send_response_body instead") + slog.Warn("analytics.allow_payloads is deprecated; use collector.request_body and collector.response_body instead") if !*collectorSendRequestBody && !*collectorSendResponseBody { *collectorSendRequestBody = true *collectorSendResponseBody = true @@ -82,7 +82,7 @@ func MigrateDeprecatedCapture(analyticsEnabled bool, deprecated CaptureFlags, co // analytics.access_logs_service on the policy-engine) onto the collector's // transport config when the collector's copy is still at its default, so // existing configs keep working after transport tuning moved to -// [collector.als]. It is generic over the two modules' differing transport +// [collector.server]. It is generic over the two modules' differing transport // struct shapes (T is only ever compared and assigned, never inspected). This // is analytics's own deprecated field, so it is only honored while // analyticsEnabled is true — otherwise a stale value left over from a @@ -96,9 +96,9 @@ func MigrateDeprecatedTransport[T comparable](analyticsEnabled bool, deprecated return } if *collectorCfg == def { - slog.Warn(deprecatedKey + " is deprecated; migrating it to collector.als") + slog.Warn(deprecatedKey + " is deprecated; migrating it to collector.server") *collectorCfg = deprecated } else { - slog.Warn(deprecatedKey + " is deprecated and collector.als is already configured; ignoring the " + deprecatedKey + " override") + slog.Warn(deprecatedKey + " is deprecated and collector.server is already configured; ignoring the " + deprecatedKey + " override") } } diff --git a/common/collector/collector_test.go b/common/collector/collector_test.go index d878f2267d..84a03ccc57 100644 --- a/common/collector/collector_test.go +++ b/common/collector/collector_test.go @@ -101,5 +101,5 @@ func TestMigrateDeprecatedTransport_DoesNotClobberExplicitCollectorConfig(t *tes MigrateDeprecatedTransport(true, deprecated, &collectorCfg, def, "analytics.grpc_event_server") - assert.Equal(t, explicit, collectorCfg, "an explicit [collector.als] override must never be clobbered by the deprecated alias") + assert.Equal(t, explicit, collectorCfg, "an explicit [collector.server] override must never be clobbered by the deprecated alias") } diff --git a/gateway/configs/config-template.toml b/gateway/configs/config-template.toml index 48acff469c..ba3e1c49a2 100644 --- a/gateway/configs/config-template.toml +++ b/gateway/configs/config-template.toml @@ -293,15 +293,15 @@ host = "localhost" # Capture request/response payloads (bodies) into the collected event. Bodies are # captured in full; per-consumer size caps are applied on output (see # traffic_logging.max_payload_size). -send_request_body = false -send_response_body = false +request_body = false +response_body = false # Capture ALL request/response headers into the collected event (request_headers/ # response_headers). Applies to every API via the collector system policy, so no # per-API header policy is needed. Sensitive values are redacted on output by -# consumers that support it (e.g. traffic_logging.masked_headers) +# consumers that support it (e.g. traffic_logging.masked_headers) # Off by default; enable deliberately per consumer. -send_request_headers = false -send_response_headers = false +request_headers = false +response_headers = false # ALS transport tuning (advanced; defaults are sensible). One section read by BOTH # ends of the Envoy → policy-engine access-log stream: @@ -310,7 +310,7 @@ send_response_headers = false # Shared keys (server_port, TLS, message/header limits) must match on both ends and # live here once. Envoy-sender-only keys (buffer_*, grpc_request_timeout) are read # only by the controller and ignored by the policy-engine. -[collector.als] +[collector.server] mode = "uds" # "uds" (default) or "tcp" server_port = 18090 # engine listens here; Envoy dials it (tcp mode) buffer_flush_interval = 1000000000 # Envoy sender only (nanoseconds) diff --git a/gateway/configs/config.toml b/gateway/configs/config.toml index 4be7780ef3..b51b959e5d 100644 --- a/gateway/configs/config.toml +++ b/gateway/configs/config.toml @@ -1,8 +1,8 @@ [collector] -send_request_body = false -send_response_body = false -send_request_headers = false -send_response_headers = false +request_body = false +response_body = false +request_headers = false +response_headers = false [analytics] enabled = true diff --git a/gateway/gateway-controller/pkg/config/config.go b/gateway/gateway-controller/pkg/config/config.go index f16bc38bb2..76e3090dd4 100644 --- a/gateway/gateway-controller/pkg/config/config.go +++ b/gateway/gateway-controller/pkg/config/config.go @@ -67,20 +67,20 @@ type Config struct { // Config.IsCollectorEnabled. This section tunes capture and transport; it has no // on/off flag of its own. type CollectorConfig struct { - // SendRequestBody / SendResponseBody capture request/response bodies into the + // RequestBody / ResponseBody capture request/response bodies into the // collected event. - SendRequestBody bool `koanf:"send_request_body"` - SendResponseBody bool `koanf:"send_response_body"` - // SendRequestHeaders / SendResponseHeaders, when true, make the collector + RequestBody bool `koanf:"request_body"` + ResponseBody bool `koanf:"response_body"` + // RequestHeaders / ResponseHeaders, when true, make the collector // capture ALL request / response headers, so every API's headers flow into // the collected event without attaching a per-API header policy. - SendRequestHeaders bool `koanf:"send_request_headers"` - SendResponseHeaders bool `koanf:"send_response_headers"` - // GRPCEventServerCfg tunes the Envoy→policy-engine gRPC access-log (ALS) + RequestHeaders bool `koanf:"request_headers"` + ResponseHeaders bool `koanf:"response_headers"` + // Server tunes the Envoy→policy-engine gRPC access-log (ALS) // transport that ships collected data. It is part of the collector and is - // configured under the shared [collector.als] section (the policy-engine reads + // configured under the shared [collector.server] section (the policy-engine reads // the same section to configure its receiving ALS server). - GRPCEventServerCfg GRPCEventServerConfig `koanf:"als"` + Server GRPCEventServerConfig `koanf:"server"` } // AnalyticsConfig holds analytics configuration @@ -89,12 +89,12 @@ type AnalyticsConfig struct { EnabledPublishers []string `koanf:"enabled_publishers"` Publishers AnalyticsPublishersConfig `koanf:"publishers"` // GRPCEventServerCfg is a deprecated alias. ALS transport tuning moved to - // [collector.als]; when set here it is migrated onto the collector during - // validation (with a warning). Prefer [collector.als]. + // [collector.server]; when set here it is migrated onto the collector during + // validation (with a warning). Prefer [collector.server]. GRPCEventServerCfg GRPCEventServerConfig `koanf:"grpc_event_server"` // AllowPayloads, SendRequestBody and SendResponseBody are deprecated aliases. // Body/header capture now lives under [collector]. When set, these are mapped - // onto collector.send_request_body / collector.send_response_body during + // onto collector.request_body / collector.response_body during // validation (with a warning) for backward compatibility. Prefer the // [collector] fields directly. AllowPayloads bool `koanf:"allow_payloads"` @@ -974,11 +974,11 @@ func defaultConfig() *Config { SendResponseBody: false, }, Collector: CollectorConfig{ - SendRequestBody: false, - SendResponseBody: false, - SendRequestHeaders: false, - SendResponseHeaders: false, - GRPCEventServerCfg: defaultGRPCEventServerConfig(), + RequestBody: false, + ResponseBody: false, + RequestHeaders: false, + ResponseHeaders: false, + Server: defaultGRPCEventServerConfig(), }, TracingConfig: TracingConfig{ Enabled: false, @@ -1760,7 +1760,7 @@ func (c *Config) validateCollectorConfig() error { c.migrateDeprecatedAnalyticsTransport() if c.IsCollectorEnabled() { - if err := validateGRPCEventServerConfig(c.Collector.GRPCEventServerCfg); err != nil { + if err := validateGRPCEventServerConfig(c.Collector.Server); err != nil { return err } } @@ -1784,7 +1784,7 @@ func (c *Config) migrateDeprecatedAnalyticsTransport() { collector.MigrateDeprecatedTransport( c.Analytics.Enabled, c.Analytics.GRPCEventServerCfg, - &c.Collector.GRPCEventServerCfg, + &c.Collector.Server, defaultGRPCEventServerConfig(), "analytics.grpc_event_server", ) @@ -1804,8 +1804,8 @@ func (c *Config) migrateDeprecatedAnalyticsCapture() { SendResponseBody: c.Analytics.SendResponseBody, AllowPayloads: c.Analytics.AllowPayloads, }, - &c.Collector.SendRequestBody, - &c.Collector.SendResponseBody, + &c.Collector.RequestBody, + &c.Collector.ResponseBody, ) } @@ -1818,10 +1818,10 @@ func validateGRPCEventServerConfig(cfg GRPCEventServerConfig) error { case "tcp": // TCP mode - validate port (host is derived from policy_engine.host) if cfg.Port <= 0 || cfg.Port > 65535 { - return fmt.Errorf("collector.als.port must be between 1 and 65535 when mode is tcp, got %d", cfg.Port) + return fmt.Errorf("collector.server.port must be between 1 and 65535 when mode is tcp, got %d", cfg.Port) } default: - return fmt.Errorf("collector.als.mode must be 'uds' or 'tcp', got: %s", cfg.Mode) + return fmt.Errorf("collector.server.mode must be 'uds' or 'tcp', got: %s", cfg.Mode) } // Validate buffer and timeout settings @@ -1836,7 +1836,7 @@ func validateGRPCEventServerConfig(cfg GRPCEventServerConfig) error { // Validate server port if cfg.ServerPort <= 0 || cfg.ServerPort > 65535 { - return fmt.Errorf("collector.als.server_port must be between 1 and 65535, got %d", cfg.ServerPort) + return fmt.Errorf("collector.server.server_port must be between 1 and 65535, got %d", cfg.ServerPort) } return nil } diff --git a/gateway/gateway-controller/pkg/config/config_test.go b/gateway/gateway-controller/pkg/config/config_test.go index 1368a09287..882e585a96 100644 --- a/gateway/gateway-controller/pkg/config/config_test.go +++ b/gateway/gateway-controller/pkg/config/config_test.go @@ -105,7 +105,7 @@ func validConfig() *Config { // Transport defaults mirror production so collector-gated grpc validation // passes and the deprecated [analytics] alias stays neutral. Collector: CollectorConfig{ - GRPCEventServerCfg: defaultGRPCEventServerConfig(), + Server: defaultGRPCEventServerConfig(), }, Analytics: AnalyticsConfig{ GRPCEventServerCfg: defaultGRPCEventServerConfig(), @@ -1213,11 +1213,11 @@ func TestConfig_ValidateAnalyticsConfig(t *testing.T) { name: "Analytics enabled with valid UDS config (default mode)", enabled: true, setupConfig: func(cfg *Config) { - cfg.Collector.GRPCEventServerCfg.Mode = "uds" - cfg.Collector.GRPCEventServerCfg.BufferFlushInterval = 1000 - cfg.Collector.GRPCEventServerCfg.BufferSizeBytes = 16384 - cfg.Collector.GRPCEventServerCfg.GRPCRequestTimeout = 5000 - cfg.Collector.GRPCEventServerCfg.ServerPort = 18090 + cfg.Collector.Server.Mode = "uds" + cfg.Collector.Server.BufferFlushInterval = 1000 + cfg.Collector.Server.BufferSizeBytes = 16384 + cfg.Collector.Server.GRPCRequestTimeout = 5000 + cfg.Collector.Server.ServerPort = 18090 }, wantErr: false, }, @@ -1225,12 +1225,12 @@ func TestConfig_ValidateAnalyticsConfig(t *testing.T) { name: "Analytics enabled with valid TCP config", enabled: true, setupConfig: func(cfg *Config) { - cfg.Collector.GRPCEventServerCfg.Mode = "tcp" - cfg.Collector.GRPCEventServerCfg.Port = 18090 - cfg.Collector.GRPCEventServerCfg.BufferFlushInterval = 1000 - cfg.Collector.GRPCEventServerCfg.BufferSizeBytes = 16384 - cfg.Collector.GRPCEventServerCfg.GRPCRequestTimeout = 5000 - cfg.Collector.GRPCEventServerCfg.ServerPort = 18090 + cfg.Collector.Server.Mode = "tcp" + cfg.Collector.Server.Port = 18090 + cfg.Collector.Server.BufferFlushInterval = 1000 + cfg.Collector.Server.BufferSizeBytes = 16384 + cfg.Collector.Server.GRPCRequestTimeout = 5000 + cfg.Collector.Server.ServerPort = 18090 }, wantErr: false, }, @@ -1238,11 +1238,11 @@ func TestConfig_ValidateAnalyticsConfig(t *testing.T) { name: "Analytics enabled with empty mode defaults to UDS", enabled: true, setupConfig: func(cfg *Config) { - cfg.Collector.GRPCEventServerCfg.Mode = "" - cfg.Collector.GRPCEventServerCfg.BufferFlushInterval = 1000 - cfg.Collector.GRPCEventServerCfg.BufferSizeBytes = 16384 - cfg.Collector.GRPCEventServerCfg.GRPCRequestTimeout = 5000 - cfg.Collector.GRPCEventServerCfg.ServerPort = 18090 + cfg.Collector.Server.Mode = "" + cfg.Collector.Server.BufferFlushInterval = 1000 + cfg.Collector.Server.BufferSizeBytes = 16384 + cfg.Collector.Server.GRPCRequestTimeout = 5000 + cfg.Collector.Server.ServerPort = 18090 }, wantErr: false, }, @@ -1250,49 +1250,49 @@ func TestConfig_ValidateAnalyticsConfig(t *testing.T) { name: "Invalid mode value", enabled: true, setupConfig: func(cfg *Config) { - cfg.Collector.GRPCEventServerCfg.Mode = "invalid" - cfg.Collector.GRPCEventServerCfg.BufferFlushInterval = 1000 - cfg.Collector.GRPCEventServerCfg.BufferSizeBytes = 16384 - cfg.Collector.GRPCEventServerCfg.GRPCRequestTimeout = 5000 - cfg.Collector.GRPCEventServerCfg.ServerPort = 18090 + cfg.Collector.Server.Mode = "invalid" + cfg.Collector.Server.BufferFlushInterval = 1000 + cfg.Collector.Server.BufferSizeBytes = 16384 + cfg.Collector.Server.GRPCRequestTimeout = 5000 + cfg.Collector.Server.ServerPort = 18090 }, wantErr: true, - errContains: "collector.als.mode must be 'uds' or 'tcp'", + errContains: "collector.server.mode must be 'uds' or 'tcp'", }, { name: "TCP mode - invalid port", enabled: true, setupConfig: func(cfg *Config) { - cfg.Collector.GRPCEventServerCfg.Mode = "tcp" - cfg.Collector.GRPCEventServerCfg.Port = 0 - cfg.Collector.GRPCEventServerCfg.BufferFlushInterval = 1000 - cfg.Collector.GRPCEventServerCfg.BufferSizeBytes = 16384 - cfg.Collector.GRPCEventServerCfg.GRPCRequestTimeout = 5000 - cfg.Collector.GRPCEventServerCfg.ServerPort = 18090 + cfg.Collector.Server.Mode = "tcp" + cfg.Collector.Server.Port = 0 + cfg.Collector.Server.BufferFlushInterval = 1000 + cfg.Collector.Server.BufferSizeBytes = 16384 + cfg.Collector.Server.GRPCRequestTimeout = 5000 + cfg.Collector.Server.ServerPort = 18090 }, wantErr: true, - errContains: "collector.als.port must be between 1 and 65535", + errContains: "collector.server.port must be between 1 and 65535", }, { name: "Invalid server port", enabled: true, setupConfig: func(cfg *Config) { - cfg.Collector.GRPCEventServerCfg.Mode = "uds" - cfg.Collector.GRPCEventServerCfg.BufferFlushInterval = 1000 - cfg.Collector.GRPCEventServerCfg.BufferSizeBytes = 16384 - cfg.Collector.GRPCEventServerCfg.GRPCRequestTimeout = 5000 - cfg.Collector.GRPCEventServerCfg.ServerPort = 0 + cfg.Collector.Server.Mode = "uds" + cfg.Collector.Server.BufferFlushInterval = 1000 + cfg.Collector.Server.BufferSizeBytes = 16384 + cfg.Collector.Server.GRPCRequestTimeout = 5000 + cfg.Collector.Server.ServerPort = 0 }, wantErr: true, - errContains: "collector.als.server_port must be between 1 and 65535", + errContains: "collector.server.server_port must be between 1 and 65535", }, { name: "Invalid buffer flush interval", enabled: true, setupConfig: func(cfg *Config) { - cfg.Collector.GRPCEventServerCfg.Mode = "uds" - cfg.Collector.GRPCEventServerCfg.BufferFlushInterval = 0 - cfg.Collector.GRPCEventServerCfg.ServerPort = 18090 + cfg.Collector.Server.Mode = "uds" + cfg.Collector.Server.BufferFlushInterval = 0 + cfg.Collector.Server.ServerPort = 18090 }, wantErr: true, errContains: "invalid gRPC event server configuration", @@ -1407,8 +1407,8 @@ func TestConfig_ValidateAnalyticsPayloadMigration(t *testing.T) { err := cfg.Validate() require.NoError(t, err) // Deprecated analytics body aliases now migrate onto the collector. - assert.Equal(t, tt.wantSendReq, cfg.Collector.SendRequestBody) - assert.Equal(t, tt.wantSendResp, cfg.Collector.SendResponseBody) + assert.Equal(t, tt.wantSendReq, cfg.Collector.RequestBody) + assert.Equal(t, tt.wantSendResp, cfg.Collector.ResponseBody) }) } } @@ -1426,8 +1426,8 @@ func TestConfig_ValidateAnalyticsPayloadMigration_SkippedWhenAnalyticsDisabled(t err := cfg.Validate() require.NoError(t, err) - assert.False(t, cfg.Collector.SendRequestBody) - assert.False(t, cfg.Collector.SendResponseBody) + assert.False(t, cfg.Collector.RequestBody) + assert.False(t, cfg.Collector.ResponseBody) } // TestConfig_ValidateAnalyticsTransportMigration_SkippedWhenAnalyticsDisabled guards @@ -1447,7 +1447,7 @@ func TestConfig_ValidateAnalyticsTransportMigration_SkippedWhenAnalyticsDisabled err := cfg.Validate() require.NoError(t, err) - assert.Equal(t, defaultGRPCEventServerConfig(), cfg.Collector.GRPCEventServerCfg) + assert.Equal(t, defaultGRPCEventServerConfig(), cfg.Collector.Server) } func TestConfig_ValidateAuthConfig(t *testing.T) { diff --git a/gateway/gateway-controller/pkg/utils/system_policies.go b/gateway/gateway-controller/pkg/utils/system_policies.go index 8985f53388..01038b5eac 100644 --- a/gateway/gateway-controller/pkg/utils/system_policies.go +++ b/gateway/gateway-controller/pkg/utils/system_policies.go @@ -201,10 +201,10 @@ func InjectSystemPolicies(policies []policyenginev1.PolicyInstance, cfg *config. // For the analytics (collector) system policy, propagate the payload and // header capture flags from the collector config. if sysPol.Name == constants.ANALYTICS_SYSTEM_POLICY_NAME { - effectiveDefaults["send_request_body"] = cfg.Collector.SendRequestBody - effectiveDefaults["send_response_body"] = cfg.Collector.SendResponseBody - effectiveDefaults["send_request_headers"] = cfg.Collector.SendRequestHeaders - effectiveDefaults["send_response_headers"] = cfg.Collector.SendResponseHeaders + effectiveDefaults["request_body"] = cfg.Collector.RequestBody + effectiveDefaults["response_body"] = cfg.Collector.ResponseBody + effectiveDefaults["request_headers"] = cfg.Collector.RequestHeaders + effectiveDefaults["response_headers"] = cfg.Collector.ResponseHeaders } // Merge parameters efficiently diff --git a/gateway/gateway-controller/pkg/utils/system_policies_test.go b/gateway/gateway-controller/pkg/utils/system_policies_test.go index 8ed2e284cb..747d02bc7e 100644 --- a/gateway/gateway-controller/pkg/utils/system_policies_test.go +++ b/gateway/gateway-controller/pkg/utils/system_policies_test.go @@ -178,47 +178,47 @@ func TestInjectSystemPolicies_BodyFlagsPropagated(t *testing.T) { cfg := &config.Config{ Analytics: config.AnalyticsConfig{Enabled: true}, Collector: config.CollectorConfig{ - SendRequestBody: true, - SendResponseBody: true, + RequestBody: true, + ResponseBody: true, }, } result := InjectSystemPolicies(nil, cfg, nil) assert.Len(t, result, 1) assert.Equal(t, constants.ANALYTICS_SYSTEM_POLICY_NAME, result[0].Name) - assert.Equal(t, true, result[0].Parameters["send_request_body"]) - assert.Equal(t, true, result[0].Parameters["send_response_body"]) + assert.Equal(t, true, result[0].Parameters["request_body"]) + assert.Equal(t, true, result[0].Parameters["response_body"]) } func TestInjectSystemPolicies_BodyFlagsDefaultFalse(t *testing.T) { cfg := &config.Config{ Analytics: config.AnalyticsConfig{Enabled: true}, Collector: config.CollectorConfig{ - SendRequestBody: false, - SendResponseBody: false, + RequestBody: false, + ResponseBody: false, }, } result := InjectSystemPolicies(nil, cfg, nil) assert.Len(t, result, 1) - assert.Equal(t, false, result[0].Parameters["send_request_body"]) - assert.Equal(t, false, result[0].Parameters["send_response_body"]) + assert.Equal(t, false, result[0].Parameters["request_body"]) + assert.Equal(t, false, result[0].Parameters["response_body"]) } func TestInjectSystemPolicies_HeaderFlagsPropagated(t *testing.T) { cfg := &config.Config{ Analytics: config.AnalyticsConfig{Enabled: true}, Collector: config.CollectorConfig{ - SendRequestHeaders: true, - SendResponseHeaders: true, + RequestHeaders: true, + ResponseHeaders: true, }, } result := InjectSystemPolicies(nil, cfg, nil) assert.Len(t, result, 1) assert.Equal(t, constants.ANALYTICS_SYSTEM_POLICY_NAME, result[0].Name) - assert.Equal(t, true, result[0].Parameters["send_request_headers"]) - assert.Equal(t, true, result[0].Parameters["send_response_headers"]) + assert.Equal(t, true, result[0].Parameters["request_headers"]) + assert.Equal(t, true, result[0].Parameters["response_headers"]) } func TestInjectSystemPolicies_HeaderFlagsDefaultFalse(t *testing.T) { @@ -230,8 +230,8 @@ func TestInjectSystemPolicies_HeaderFlagsDefaultFalse(t *testing.T) { result := InjectSystemPolicies(nil, cfg, nil) assert.Len(t, result, 1) - assert.Equal(t, false, result[0].Parameters["send_request_headers"]) - assert.Equal(t, false, result[0].Parameters["send_response_headers"]) + assert.Equal(t, false, result[0].Parameters["request_headers"]) + assert.Equal(t, false, result[0].Parameters["response_headers"]) } func TestInjectSystemPolicies_WithAdditionalProps(t *testing.T) { diff --git a/gateway/gateway-controller/pkg/xds/translator.go b/gateway/gateway-controller/pkg/xds/translator.go index f0b5708d8f..c7837005e3 100644 --- a/gateway/gateway-controller/pkg/xds/translator.go +++ b/gateway/gateway-controller/pkg/xds/translator.go @@ -562,7 +562,7 @@ func (t *Translator) TranslateConfigs( clusters = append(clusters, policyEngineCluster) // Add ALS cluster if the collector is active (it ships access logs over gRPC) - log.Debug("gRPC event server config", slog.Any("config", t.config.Collector.GRPCEventServerCfg)) + log.Debug("gRPC event server config", slog.Any("config", t.config.Collector.Server)) if t.config.IsCollectorEnabled() { log.Info("collector is enabled, creating ALS cluster") alsCluster := t.createALSCluster() @@ -2156,7 +2156,7 @@ func (t *Translator) createPolicyEngineCluster() *cluster.Cluster { // createALSCluster creates an Envoy cluster for the gRPC access log service func (t *Translator) createALSCluster() *cluster.Cluster { - grpcConfig := t.config.Collector.GRPCEventServerCfg + grpcConfig := t.config.Collector.Server // Build the endpoint address (UDS or TCP) var address *core.Address @@ -2802,8 +2802,8 @@ func (t *Translator) createAccessLogConfig() ([]*accesslog.AccessLog, error) { // createGRPCAccessLog creates a gRPC access log configuration for the gateway controller func (t *Translator) createGRPCAccessLog() (*accesslog.AccessLog, error) { - grpcConfig := t.config.Collector.GRPCEventServerCfg - bufferSizeBytes, err := checkedUInt32FromPositiveInt("collector.als.buffer_size_bytes", grpcConfig.BufferSizeBytes) + grpcConfig := t.config.Collector.Server + bufferSizeBytes, err := checkedUInt32FromPositiveInt("collector.server.buffer_size_bytes", grpcConfig.BufferSizeBytes) if err != nil { return nil, err } diff --git a/gateway/gateway-controller/pkg/xds/translator_test.go b/gateway/gateway-controller/pkg/xds/translator_test.go index 4a99e0036d..72befbb871 100644 --- a/gateway/gateway-controller/pkg/xds/translator_test.go +++ b/gateway/gateway-controller/pkg/xds/translator_test.go @@ -1330,7 +1330,7 @@ func TestTranslator_CreateALSCluster(t *testing.T) { routerCfg := testRouterConfig() cfg := testConfig() cfg.Analytics.Enabled = true - cfg.Collector.GRPCEventServerCfg = config.GRPCEventServerConfig{ + cfg.Collector.Server = config.GRPCEventServerConfig{ Mode: "uds", BufferFlushInterval: 1000000000, BufferSizeBytes: 16384, @@ -1357,7 +1357,7 @@ func TestTranslator_CreateALSCluster(t *testing.T) { routerCfg := testRouterConfig() cfg := testConfig() cfg.Analytics.Enabled = true - cfg.Collector.GRPCEventServerCfg = config.GRPCEventServerConfig{ + cfg.Collector.Server = config.GRPCEventServerConfig{ Mode: "", BufferFlushInterval: 1000000000, BufferSizeBytes: 16384, @@ -1383,7 +1383,7 @@ func TestTranslator_CreateALSCluster(t *testing.T) { routerCfg := testRouterConfig() cfg := testConfig() cfg.Analytics.Enabled = true - cfg.Collector.GRPCEventServerCfg = config.GRPCEventServerConfig{ + cfg.Collector.Server = config.GRPCEventServerConfig{ Mode: "tcp", Port: 18090, BufferFlushInterval: 1000000000, @@ -1415,7 +1415,7 @@ func TestTranslator_CreateGRPCAccessLog(t *testing.T) { logger := createTestLogger() routerCfg := testRouterConfig() cfg := testConfig() - cfg.Collector.GRPCEventServerCfg = config.GRPCEventServerConfig{ + cfg.Collector.Server = config.GRPCEventServerConfig{ Mode: "tcp", Port: 18090, BufferFlushInterval: 1000, @@ -1433,7 +1433,7 @@ func TestTranslator_CreateGRPCAccessLog_BufferSizeOverflow(t *testing.T) { logger := createTestLogger() routerCfg := testRouterConfig() cfg := testConfig() - cfg.Collector.GRPCEventServerCfg = config.GRPCEventServerConfig{ + cfg.Collector.Server = config.GRPCEventServerConfig{ Mode: "tcp", Port: 18090, BufferFlushInterval: 1000, diff --git a/gateway/gateway-runtime/policy-engine/cmd/policy-engine/main.go b/gateway/gateway-runtime/policy-engine/cmd/policy-engine/main.go index 1281a7a519..e63886b085 100644 --- a/gateway/gateway-runtime/policy-engine/cmd/policy-engine/main.go +++ b/gateway/gateway-runtime/policy-engine/cmd/policy-engine/main.go @@ -272,7 +272,7 @@ func main() { // collector is the shared transport that carries collected data to its // consumers (analytics, traffic logging). var alsServer *grpc.Server - slog.DebugContext(ctx, "Policy engine ALS server config", "config", cfg.Collector.AccessLogsServiceCfg) + slog.DebugContext(ctx, "Policy engine ALS server config", "config", cfg.Collector.Server) if cfg.IsCollectorEnabled() { // Start the access log service server slog.Info("Starting the ALS gRPC server...") diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go index eb6085e2bc..39a9896981 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go @@ -491,13 +491,13 @@ func (c *Analytics) prepareAnalyticEvent(logEntry *v3.HTTPAccessLogEntry) *dto.E } // Optionally attach request and response payloads when enabled via the collector. - if c.cfg.Collector.SendRequestBody { + if c.cfg.Collector.RequestBody { if requestPayload, ok := keyValuePairsFromMetadata[dto.PropKeyRequestPayload]; ok && requestPayload != "" { event.Properties[dto.PropKeyRequestPayload] = requestPayload slog.Debug("Analytics request payload captured", "size_bytes", len(requestPayload)) } } - if c.cfg.Collector.SendResponseBody { + if c.cfg.Collector.ResponseBody { if responsePayload, ok := keyValuePairsFromMetadata[dto.PropKeyResponsePayload]; ok && responsePayload != "" { event.Properties[dto.PropKeyResponsePayload] = responsePayload slog.Debug("Analytics response payload captured", "size_bytes", len(responsePayload)) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go index 49b0a585ab..268572ad93 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go @@ -64,7 +64,7 @@ func validAnalyticsConfigForValidation(analytics config.AnalyticsConfig) *config Analytics: analytics, } cfg.Analytics.Enabled = true // a consumer being on makes the collector implicit - cfg.Collector.AccessLogsServiceCfg = config.AccessLogsServiceConfig{ + cfg.Collector.Server = config.AccessLogsServiceConfig{ Mode: "uds", ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -614,8 +614,8 @@ func TestPrepareAnalyticEvent_WithRequestResponseHeaders(t *testing.T) { func TestPrepareAnalyticEvent_WithPayloadsEnabled(t *testing.T) { cfg := &config.Config{ Collector: config.CollectorConfig{ - SendRequestBody: true, - SendResponseBody: true, + RequestBody: true, + ResponseBody: true, }, } analytics := NewAnalytics(cfg) @@ -640,8 +640,8 @@ func TestPrepareAnalyticEvent_WithPayloadsEnabled(t *testing.T) { func TestPrepareAnalyticEvent_WithPayloadsDisabled(t *testing.T) { cfg := &config.Config{ Collector: config.CollectorConfig{ - SendRequestBody: false, - SendResponseBody: false, + RequestBody: false, + ResponseBody: false, }, } analytics := NewAnalytics(cfg) @@ -663,8 +663,8 @@ func TestPrepareAnalyticEvent_WithPayloadsDisabled(t *testing.T) { func TestPrepareAnalyticEvent_RequestPayloadOnly(t *testing.T) { cfg := &config.Config{ Collector: config.CollectorConfig{ - SendRequestBody: true, - SendResponseBody: false, + RequestBody: true, + ResponseBody: false, }, } analytics := NewAnalytics(cfg) @@ -687,8 +687,8 @@ func TestPrepareAnalyticEvent_RequestPayloadOnly(t *testing.T) { func TestPrepareAnalyticEvent_ResponsePayloadOnly(t *testing.T) { cfg := &config.Config{ Collector: config.CollectorConfig{ - SendRequestBody: false, - SendResponseBody: true, + RequestBody: false, + ResponseBody: true, }, } analytics := NewAnalytics(cfg) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go index c3982ce8ca..40e8132888 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go @@ -36,7 +36,7 @@ const maskedHeaderValue = "****" // stdout as a single JSON line. It is intended for log-scraping pipelines // (Fluent Bit, Loki, ELK, etc.) and as a lightweight alternative to a SaaS // analytics backend. The event already carries the rich metadata, headers and -// (when send_request_body/send_response_body are enabled) payloads attached by +// (when request_body/response_body are enabled) payloads attached by // the analytics engine, so this publisher only serializes it. type Log struct { // maskedHeaders holds lower-cased header names whose values are redacted in @@ -230,8 +230,11 @@ func filterNestedKeys(m map[string]json.RawMessage, top string, keep func(string // maskHeaders redacts header values whose names appear in mask (case-insensitive). // Returns a new map; the input is not modified. To drop a header entirely rather -// than redacting its value, use the per-flow excludeHeaders directive (see -// dropHeaders) or a dotted fields.exclude path (e.g. "requestHeaders.authorization"). +// than redacting its value, prefer the per-flow excludeHeaders directive (see +// dropHeaders), which matches header names case-insensitively. A dotted +// fields.exclude path (e.g. "requestHeaders.Authorization") can also drop a field, +// but it matches the emitted key case-sensitively — so it must reproduce the exact +// casing Envoy delivered, and is not a reliable way to drop a header by name. func (l *Log) maskHeaders(headers map[string]string, mask map[string]bool) map[string]string { result := make(map[string]string, len(headers)) for name, value := range headers { diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go index 992c929422..cca18ea9bb 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go @@ -401,6 +401,41 @@ func TestLog_Publish_FieldsExclude(t *testing.T) { assert.Contains(t, decoded, "requestHeaders", "requestHeaders kept (not excluded)") } +// An unrelated fields.exclude entry (dropping one header sub-key) must not defeat +// an explicitly configured flow's payload:false/headers:true booleans. Regression +// test for a bug where any fields.exclude entry made the per-flow booleans +// globally inert, silently re-enabling payload logging that request.payload:false +// was supposed to suppress. +func TestLog_Publish_FieldsExcludeDoesNotOverrideExplicitFlowBooleans(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.Properties["requestHeaders"] = `{"Cookie":"sid=1","x-foo":"bar"}` + event.Properties["request_payload"] = "secret-request-body" + event.Properties["responseHeaders"] = `{"x-bar":"baz"}` + event.Properties["response_payload"] = "secret-response-body" + event.TrafficLog = &dto.TrafficLogDirective{ + Request: &dto.TrafficLogFlow{Headers: true, Payload: false}, + Response: &dto.TrafficLogFlow{Headers: true, Payload: false}, + Fields: &dto.TrafficLogFields{Exclude: []string{"requestHeaders.Cookie"}}, + } + + l.Publish(event) + decoded := decodeLine(t, read()) + + _, hasReqBody := decoded["requestBody"] + assert.False(t, hasReqBody, "request.payload:false must still suppress the request body") + _, hasRespBody := decoded["responseBody"] + assert.False(t, hasRespBody, "response.payload:false must still suppress the response body") + + reqH := headerMap(t, decoded["requestHeaders"]) + _, hasCookie := reqH["Cookie"] + assert.False(t, hasCookie, "excluded header sub-key still dropped") + assert.Equal(t, "bar", reqH["x-foo"], "other request headers still present") + + respH := headerMap(t, decoded["responseHeaders"]) + assert.Equal(t, "baz", respH["x-bar"], "response headers still present per headers:true") +} + // requestBody and properties are top-level keys like any other and can be selected // explicitly via fields.only. func TestLog_Publish_FieldsIncludeRequestBodyAndProperties(t *testing.T) { diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/traffic_log_event.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/traffic_log_event.go index 81e531e976..669345c473 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/traffic_log_event.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/traffic_log_event.go @@ -142,11 +142,21 @@ func (l *Log) toTrafficLogEvent(event *dto.Event, dir *dto.TrafficLogDirective) } } - // When a fields selection is configured it is authoritative over presence: - // the per-flow Headers/Payload booleans are ignored and applyFieldsProjection - // in Publish decides what survives. Global masking always applies; per-API - // maskedHeaders are merged in here and passed down. - hasFieldsSelection := dir.Fields != nil && (len(dir.Fields.Only) > 0 || len(dir.Fields.Exclude) > 0) + // fields.only is a whitelist and is always authoritative over presence: every + // field gets attached here and applyFieldsProjection in Publish alone decides + // what survives, regardless of the per-flow Headers/Payload booleans. + // + // fields.exclude is different: when a flow (Request/Response) is explicitly + // configured, its Headers/Payload booleans keep governing presence as normal — + // exclude only trims already-present fields/sub-keys (e.g. one header) on top + // of that. This is what makes an unrelated `fields.exclude: + // [requestHeaders.cookie]` entry not silently defeat `request.payload: false`. + // Only when a flow isn't configured at all does exclude fall back to its + // "present unless named" convenience behavior, so `fields.exclude` alone (with + // no request/response block) still works as a blanket "log everything except + // X" shorthand. + hasOnlySelection := dir.Fields != nil && len(dir.Fields.Only) > 0 + hasExcludeSelection := dir.Fields != nil && len(dir.Fields.Only) == 0 && len(dir.Fields.Exclude) > 0 // Effective header mask: global config merged with any per-API additions. mask := l.maskedHeaders @@ -163,7 +173,8 @@ func (l *Log) toTrafficLogEvent(event *dto.Event, dir *dto.TrafficLogDirective) // Request flow if raw, ok := event.Properties[dto.PropKeyRequestHeaders].(string); ok { - if hasFieldsSelection || (dir.Request != nil && dir.Request.Headers) { + headersOn := dir.Request != nil && dir.Request.Headers + if fieldEnabled(hasOnlySelection, hasExcludeSelection, dir.Request, headersOn) { if headers := parseHeadersFromString(raw); headers != nil { masked := l.maskHeaders(headers, mask) if dir.Request != nil { @@ -174,14 +185,16 @@ func (l *Log) toTrafficLogEvent(event *dto.Event, dir *dto.TrafficLogDirective) } } if p, ok := event.Properties[dto.PropKeyRequestPayload].(string); ok && p != "" { - if hasFieldsSelection || (dir.Request != nil && dir.Request.Payload) { + payloadOn := dir.Request != nil && dir.Request.Payload + if fieldEnabled(hasOnlySelection, hasExcludeSelection, dir.Request, payloadOn) { tl.RequestBody = l.truncatePayload(p) } } // Response flow if raw, ok := event.Properties[dto.PropKeyResponseHeaders].(string); ok { - if hasFieldsSelection || (dir.Response != nil && dir.Response.Headers) { + headersOn := dir.Response != nil && dir.Response.Headers + if fieldEnabled(hasOnlySelection, hasExcludeSelection, dir.Response, headersOn) { if headers := parseHeadersFromString(raw); headers != nil { masked := l.maskHeaders(headers, mask) if dir.Response != nil { @@ -192,7 +205,8 @@ func (l *Log) toTrafficLogEvent(event *dto.Event, dir *dto.TrafficLogDirective) } } if p, ok := event.Properties[dto.PropKeyResponsePayload].(string); ok && p != "" { - if hasFieldsSelection || (dir.Response != nil && dir.Response.Payload) { + payloadOn := dir.Response != nil && dir.Response.Payload + if fieldEnabled(hasOnlySelection, hasExcludeSelection, dir.Response, payloadOn) { tl.ResponseBody = l.truncatePayload(p) } } @@ -203,3 +217,18 @@ func (l *Log) toTrafficLogEvent(event *dto.Event, dir *dto.TrafficLogDirective) return tl } + +// fieldEnabled decides whether a request/response header or payload field should +// be attached to the traffic-log event, before any Fields projection trims it +// back down. See the comment above hasOnlySelection/hasExcludeSelection in +// toTrafficLogEvent for the reasoning. +func fieldEnabled(hasOnlySelection, hasExcludeSelection bool, flow *dto.TrafficLogFlow, boolValue bool) bool { + switch { + case hasOnlySelection: + return true + case flow != nil: + return boolValue + default: + return hasExcludeSelection + } +} diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config.go b/gateway/gateway-runtime/policy-engine/internal/config/config.go index d3fee0eb03..49263040a6 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config.go @@ -55,15 +55,15 @@ type Config struct { // whenever a consumer is enabled — see Config.IsCollectorEnabled. This section // tunes capture and transport; it has no on/off flag of its own. type CollectorConfig struct { - // SendRequestBody / SendResponseBody attach captured request/response bodies + // RequestBody / ResponseBody attach captured request/response bodies // onto the collected event. - SendRequestBody bool `koanf:"send_request_body"` - SendResponseBody bool `koanf:"send_response_body"` - // AccessLogsServiceCfg tunes the policy-engine ALS receiver (the gRPC server - // that ingests collected access logs). It is part of the collector transport - // and is configured under the shared [collector.als] section (the controller + RequestBody bool `koanf:"request_body"` + ResponseBody bool `koanf:"response_body"` + // Server tunes the policy-engine ALS receiver (the gRPC server that ingests + // collected access logs). It is part of the collector transport and is + // configured under the shared [collector.server] section (the controller // reads the same section to configure Envoy's sender side). - AccessLogsServiceCfg AccessLogsServiceConfig `koanf:"als"` + Server AccessLogsServiceConfig `koanf:"server"` } // AnalyticsConfig holds analytics configuration @@ -73,12 +73,12 @@ type AnalyticsConfig struct { Publishers AnalyticsPublishersConfig `koanf:"publishers"` GRPCEventServerCfg map[string]interface{} `koanf:"grpc_event_server"` // AccessLogsServiceCfg is a deprecated alias. ALS receiver tuning moved to - // [collector.als]; when set here it is migrated onto the collector during - // validation (with a warning). Prefer [collector.als]. + // [collector.server]; when set here it is migrated onto the collector during + // validation (with a warning). Prefer [collector.server]. AccessLogsServiceCfg AccessLogsServiceConfig `koanf:"access_logs_service"` // AllowPayloads, SendRequestBody and SendResponseBody are deprecated aliases. // Body capture now lives under [collector]. When set, these are mapped onto - // collector.send_request_body / collector.send_response_body during validation + // collector.request_body / collector.response_body during validation // (with a warning). Prefer the [collector] fields directly. AllowPayloads bool `koanf:"allow_payloads"` SendRequestBody bool `koanf:"send_request_body"` @@ -403,9 +403,9 @@ func defaultConfig() *Config { TracingServiceName: "policy-engine", }, Collector: CollectorConfig{ - SendRequestBody: false, - SendResponseBody: false, - AccessLogsServiceCfg: defaultAccessLogsServiceConfig(), + RequestBody: false, + ResponseBody: false, + Server: defaultAccessLogsServiceConfig(), }, TrafficLogging: TrafficLoggingConfig{ Enabled: false, @@ -607,7 +607,7 @@ func (c *Config) validateCollectorConfig() error { c.migrateDeprecatedAnalyticsTransport() if c.IsCollectorEnabled() { - if err := validateAccessLogsServiceConfig(c.Collector.AccessLogsServiceCfg); err != nil { + if err := validateAccessLogsServiceConfig(c.Collector.Server); err != nil { return err } } @@ -630,7 +630,7 @@ func (c *Config) migrateDeprecatedAnalyticsTransport() { collector.MigrateDeprecatedTransport( c.Analytics.Enabled, c.Analytics.AccessLogsServiceCfg, - &c.Collector.AccessLogsServiceCfg, + &c.Collector.Server, defaultAccessLogsServiceConfig(), "analytics.access_logs_service", ) @@ -643,22 +643,22 @@ func validateAccessLogsServiceConfig(als AccessLogsServiceConfig) error { // UDS mode (default) - port is unused case "tcp": if als.ServerPort <= 0 || als.ServerPort > 65535 { - return fmt.Errorf("collector.als.server_port must be between 1 and 65535, got %d", als.ServerPort) + return fmt.Errorf("collector.server.server_port must be between 1 and 65535, got %d", als.ServerPort) } default: - return fmt.Errorf("collector.als.mode must be 'uds' or 'tcp', got: %s", als.Mode) + return fmt.Errorf("collector.server.mode must be 'uds' or 'tcp', got: %s", als.Mode) } if als.ShutdownTimeout <= 0 { - return fmt.Errorf("collector.als.shutdown_timeout must be positive, got %s", als.ShutdownTimeout) + return fmt.Errorf("collector.server.shutdown_timeout must be positive, got %s", als.ShutdownTimeout) } if als.ExtProcMaxMessageSize <= 0 { - return fmt.Errorf("collector.als.max_message_size must be positive, got %d", als.ExtProcMaxMessageSize) + return fmt.Errorf("collector.server.max_message_size must be positive, got %d", als.ExtProcMaxMessageSize) } if als.ExtProcMaxHeaderLimit <= 0 { - return fmt.Errorf("collector.als.max_header_limit must be positive, got %d", als.ExtProcMaxHeaderLimit) + return fmt.Errorf("collector.server.max_header_limit must be positive, got %d", als.ExtProcMaxHeaderLimit) } if als.ExtProcMaxHeaderLimit > math.MaxUint32 { - return fmt.Errorf("collector.als.max_header_limit must be <= %d, got %d", uint64(math.MaxUint32), als.ExtProcMaxHeaderLimit) + return fmt.Errorf("collector.server.max_header_limit must be <= %d, got %d", uint64(math.MaxUint32), als.ExtProcMaxHeaderLimit) } return nil } @@ -677,8 +677,8 @@ func (c *Config) migrateDeprecatedAnalyticsCapture() { SendResponseBody: c.Analytics.SendResponseBody, AllowPayloads: c.Analytics.AllowPayloads, }, - &c.Collector.SendRequestBody, - &c.Collector.SendResponseBody, + &c.Collector.RequestBody, + &c.Collector.ResponseBody, ) } diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config_test.go b/gateway/gateway-runtime/policy-engine/internal/config/config_test.go index f26847823e..2d2d224964 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config_test.go @@ -73,7 +73,7 @@ func validConfig() *Config { // receiver defaults mirror production so transport validation passes and the // deprecated alias stays neutral (no spurious migration). Collector: CollectorConfig{ - AccessLogsServiceCfg: defaultAccessLogsServiceConfig(), + Server: defaultAccessLogsServiceConfig(), }, Analytics: AnalyticsConfig{ Enabled: false, @@ -955,7 +955,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - valid UDS config (default)", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.Server = AccessLogsServiceConfig{ Mode: "uds", ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -968,7 +968,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - valid UDS config (empty mode defaults to uds)", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.Server = AccessLogsServiceConfig{ Mode: "", ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -981,7 +981,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - valid TCP config", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.Server = AccessLogsServiceConfig{ Mode: "tcp", ServerPort: 18090, ShutdownTimeout: 600 * time.Second, @@ -995,7 +995,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - invalid mode", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.Server = AccessLogsServiceConfig{ Mode: "invalid", ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -1009,7 +1009,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - TCP mode invalid ALS port", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.Server = AccessLogsServiceConfig{ Mode: "tcp", ServerPort: 0, ShutdownTimeout: 600 * time.Second, @@ -1024,7 +1024,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - UDS mode skips port validation", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.Server = AccessLogsServiceConfig{ Mode: "uds", ServerPort: 0, // Invalid port, but irrelevant in UDS mode ShutdownTimeout: 600 * time.Second, @@ -1038,7 +1038,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - invalid shutdown timeout", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.Server = AccessLogsServiceConfig{ ShutdownTimeout: 0, ExtProcMaxMessageSize: 1000000, ExtProcMaxHeaderLimit: 8192, @@ -1051,7 +1051,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - invalid max message size", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.Server = AccessLogsServiceConfig{ ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 0, ExtProcMaxHeaderLimit: 8192, @@ -1064,7 +1064,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - invalid max header limit", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.Server = AccessLogsServiceConfig{ ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, ExtProcMaxHeaderLimit: 0, @@ -1077,7 +1077,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - max header limit exceeds uint32", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.Server = AccessLogsServiceConfig{ ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, ExtProcMaxHeaderLimit: math.MaxInt, @@ -1107,7 +1107,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { func TestValidate_AnalyticsPayloadMigration(t *testing.T) { setValidAnalyticsALS := func(cfg *Config) { cfg.Analytics.Enabled = true // a consumer being on makes the collector implicit - cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.Server = AccessLogsServiceConfig{ Mode: "uds", ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -1168,8 +1168,8 @@ func TestValidate_AnalyticsPayloadMigration(t *testing.T) { err := cfg.Validate() require.NoError(t, err) // Deprecated analytics body aliases now migrate onto the collector. - assert.Equal(t, tt.wantSendReq, cfg.Collector.SendRequestBody) - assert.Equal(t, tt.wantSendResp, cfg.Collector.SendResponseBody) + assert.Equal(t, tt.wantSendReq, cfg.Collector.RequestBody) + assert.Equal(t, tt.wantSendResp, cfg.Collector.ResponseBody) }) } } @@ -1187,8 +1187,8 @@ func TestValidate_AnalyticsPayloadMigration_SkippedWhenAnalyticsDisabled(t *test err := cfg.Validate() require.NoError(t, err) - assert.False(t, cfg.Collector.SendRequestBody) - assert.False(t, cfg.Collector.SendResponseBody) + assert.False(t, cfg.Collector.RequestBody) + assert.False(t, cfg.Collector.ResponseBody) } // TestValidate_AnalyticsTransportMigration_SkippedWhenAnalyticsDisabled guards @@ -1209,7 +1209,7 @@ func TestValidate_AnalyticsTransportMigration_SkippedWhenAnalyticsDisabled(t *te err := cfg.Validate() require.NoError(t, err) - assert.Equal(t, defaultAccessLogsServiceConfig(), cfg.Collector.AccessLogsServiceCfg) + assert.Equal(t, defaultAccessLogsServiceConfig(), cfg.Collector.Server) } // TestValidate_CollectorPrerequisite verifies that enabling a consumer (analytics, @@ -1262,7 +1262,7 @@ func TestValidate_AnalyticsPublishers(t *testing.T) { name: "no publishers enabled", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.Server = AccessLogsServiceConfig{ ServerPort: 18090, ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -1276,7 +1276,7 @@ func TestValidate_AnalyticsPublishers(t *testing.T) { name: "unknown publisher type", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.Server = AccessLogsServiceConfig{ ServerPort: 18090, ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -1291,7 +1291,7 @@ func TestValidate_AnalyticsPublishers(t *testing.T) { name: "moesif publisher - missing application_id", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.Server = AccessLogsServiceConfig{ ServerPort: 18090, ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -1307,7 +1307,7 @@ func TestValidate_AnalyticsPublishers(t *testing.T) { name: "moesif publisher - invalid publish_interval", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.Server = AccessLogsServiceConfig{ ServerPort: 18090, ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -1324,7 +1324,7 @@ func TestValidate_AnalyticsPublishers(t *testing.T) { name: "moesif publisher - invalid base_url", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.Server = AccessLogsServiceConfig{ ServerPort: 18090, ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -1342,7 +1342,7 @@ func TestValidate_AnalyticsPublishers(t *testing.T) { name: "moesif publisher - valid config", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.Server = AccessLogsServiceConfig{ ServerPort: 18090, ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, diff --git a/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server.go b/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server.go index b68babbe08..067681698a 100644 --- a/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server.go +++ b/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server.go @@ -90,15 +90,15 @@ func StartAccessLogServiceServer(cfg *config.Config) *grpc.Server { Timeout: 20 * time.Second, } maxHeaderListSize, err := checkedUInt32FromPositiveInt( - "collector.als.max_header_limit", - cfg.Collector.AccessLogsServiceCfg.ExtProcMaxHeaderLimit, + "collector.server.max_header_limit", + cfg.Collector.Server.ExtProcMaxHeaderLimit, ) if err != nil { panic(err) } - server, err := CreateGRPCServer(cfg.Collector.AccessLogsServiceCfg.PublicKeyPath, - cfg.Collector.AccessLogsServiceCfg.PrivateKeyPath, cfg.Collector.AccessLogsServiceCfg.ALSPlainText, - grpc.MaxRecvMsgSize(cfg.Collector.AccessLogsServiceCfg.ExtProcMaxMessageSize), + server, err := CreateGRPCServer(cfg.Collector.Server.PublicKeyPath, + cfg.Collector.Server.PrivateKeyPath, cfg.Collector.Server.ALSPlainText, + grpc.MaxRecvMsgSize(cfg.Collector.Server.ExtProcMaxMessageSize), grpc.MaxHeaderListSize(maxHeaderListSize), grpc.KeepaliveParams(kaParams)) if err != nil { @@ -109,7 +109,7 @@ func StartAccessLogServiceServer(cfg *config.Config) *grpc.Server { // Create listener based on mode (same pattern as ext_proc in main.go) var listener net.Listener - alsMode := cfg.Collector.AccessLogsServiceCfg.Mode + alsMode := cfg.Collector.Server.Mode if alsMode == "" { alsMode = "uds" } @@ -139,14 +139,14 @@ func StartAccessLogServiceServer(cfg *config.Config) *grpc.Server { } }() case "tcp": - listener, err = net.Listen("tcp", fmt.Sprintf(":%d", cfg.Collector.AccessLogsServiceCfg.ServerPort)) + listener, err = net.Listen("tcp", fmt.Sprintf(":%d", cfg.Collector.Server.ServerPort)) if err != nil { - slog.Error("Failed to listen on ALS TCP port", "port", cfg.Collector.AccessLogsServiceCfg.ServerPort) + slog.Error("Failed to listen on ALS TCP port", "port", cfg.Collector.Server.ServerPort) panic(err) } go func() { - slog.Info("Starting to serve access log service server", "mode", "tcp", "port", cfg.Collector.AccessLogsServiceCfg.ServerPort) + slog.Info("Starting to serve access log service server", "mode", "tcp", "port", cfg.Collector.Server.ServerPort) if err := server.Serve(listener); err != nil { slog.Error("ALS server exited", "error", err) } diff --git a/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server_test.go b/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server_test.go index 3176524241..e802cf5d62 100644 --- a/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server_test.go @@ -211,7 +211,7 @@ func TestStreamAccessLogs_MultipleMessages(t *testing.T) { func TestStartAccessLogServiceServer_TCP(t *testing.T) { cfg := &config.Config{ Collector: config.CollectorConfig{ - AccessLogsServiceCfg: config.AccessLogsServiceConfig{ + Server: config.AccessLogsServiceConfig{ Mode: "tcp", ServerPort: 19001, // Use non-standard port to avoid conflicts ALSPlainText: true, diff --git a/gateway/system-policies/analytics/analytics.go b/gateway/system-policies/analytics/analytics.go index 7635f5268b..c197a991b8 100644 --- a/gateway/system-policies/analytics/analytics.go +++ b/gateway/system-policies/analytics/analytics.go @@ -905,7 +905,7 @@ func convertToInt64(value interface{}) (int64, error) { } // getPayloadFlags derives per-direction payload capture flags from policy parameters. -// New parameters send_request_body and send_response_body take precedence. When neither +// New parameters request_body and response_body take precedence. When neither // is provided, the deprecated allow_payloads flag is used as a fallback, mapping to // both directions for backward compatibility. func getPayloadFlags(params map[string]interface{}) (sendRequestBody, sendResponseBody bool) { @@ -915,11 +915,11 @@ func getPayloadFlags(params map[string]interface{}) (sendRequestBody, sendRespon hasReq, hasResp := false, false - if raw, ok := params["send_request_body"]; ok { + if raw, ok := params["request_body"]; ok { sendRequestBody = parseBoolLike(raw) hasReq = true } - if raw, ok := params["send_response_body"]; ok { + if raw, ok := params["response_body"]; ok { sendResponseBody = parseBoolLike(raw) hasResp = true } @@ -959,10 +959,10 @@ func getHeaderFlags(params map[string]interface{}) (sendRequestHeaders, sendResp if params == nil { return false, false } - if raw, ok := params["send_request_headers"]; ok { + if raw, ok := params["request_headers"]; ok { sendRequestHeaders = parseBoolLike(raw) } - if raw, ok := params["send_response_headers"]; ok { + if raw, ok := params["response_headers"]; ok { sendResponseHeaders = parseBoolLike(raw) } return sendRequestHeaders, sendResponseHeaders diff --git a/gateway/system-policies/analytics/analytics_headers_test.go b/gateway/system-policies/analytics/analytics_headers_test.go index aed92c9f6b..25a3b645cb 100644 --- a/gateway/system-policies/analytics/analytics_headers_test.go +++ b/gateway/system-policies/analytics/analytics_headers_test.go @@ -16,9 +16,9 @@ func TestGetHeaderFlags(t *testing.T) { }{ {"nil params", nil, false, false}, {"absent", map[string]interface{}{}, false, false}, - {"bool true", map[string]interface{}{"send_request_headers": true, "send_response_headers": true}, true, true}, - {"string true", map[string]interface{}{"send_request_headers": "true"}, true, false}, - {"mixed", map[string]interface{}{"send_request_headers": false, "send_response_headers": "yes"}, false, true}, + {"bool true", map[string]interface{}{"request_headers": true, "response_headers": true}, true, true}, + {"string true", map[string]interface{}{"request_headers": "true"}, true, false}, + {"mixed", map[string]interface{}{"request_headers": false, "response_headers": "yes"}, false, true}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { diff --git a/gateway/system-policies/analytics/policy-definition.yaml b/gateway/system-policies/analytics/policy-definition.yaml index c3c223bd8a..8969448c27 100644 --- a/gateway/system-policies/analytics/policy-definition.yaml +++ b/gateway/system-policies/analytics/policy-definition.yaml @@ -10,14 +10,14 @@ parameters: type: object additionalProperties: false properties: - send_request_body: + request_body: type: boolean default: false description: > When true, the analytics system policy will capture the request body into analytics metadata as request_payload, which is then forwarded through the analytics pipeline (including publishers like Moesif). - send_response_body: + response_body: type: boolean default: false description: > @@ -25,7 +25,7 @@ parameters: into analytics metadata as response_payload, for both buffered and streaming responses, which is then forwarded through the analytics pipeline (including publishers like Moesif). - send_request_headers: + request_headers: type: boolean default: false description: > @@ -34,7 +34,7 @@ parameters: the analytics pipeline (including the stdout/log and Moesif publishers). Sensitive values are not masked here; the log publisher's masked_headers setting redacts them on output. - send_response_headers: + response_headers: type: boolean default: false description: > @@ -45,10 +45,10 @@ parameters: type: boolean default: false description: > - Deprecated: use send_request_body and send_response_body instead. - When true, and when send_request_body and send_response_body are not + Deprecated: use request_body and response_body instead. + When true, and when request_body and response_body are not explicitly set, the analytics system policy behaves as if both - send_request_body and send_response_body are true, capturing request + request_body and response_body are true, capturing request and response bodies into analytics metadata as request_payload and response_payload, which are then forwarded through the analytics pipeline (including publishers like Moesif). When false, payloads are From 926c4936565d081527aba74d8cec8f1bfd40274f Mon Sep 17 00:00:00 2001 From: Dineth Date: Thu, 9 Jul 2026 11:08:36 +0530 Subject: [PATCH 17/21] feat: add support for ignoring specific path prefixes in traffic logging --- gateway/configs/config-template.toml | 12 ++ .../gateway-controller/pkg/config/config.go | 30 ++++ .../pkg/config/config_test.go | 22 +++ .../gateway-controller/pkg/xds/translator.go | 91 +++++++++- .../pkg/xds/translator_test.go | 158 ++++++++++++++++++ 5 files changed, 312 insertions(+), 1 deletion(-) diff --git a/gateway/configs/config-template.toml b/gateway/configs/config-template.toml index ba3e1c49a2..cdfa7659db 100644 --- a/gateway/configs/config-template.toml +++ b/gateway/configs/config-template.toml @@ -303,6 +303,18 @@ response_body = false request_headers = false response_headers = false +# Path prefixes for which the collector produces no analytics event and no +# traffic-log line at all (as if disabled for that one request) — e.g. health +# checks or metrics scrapes. Enforced by Envoy itself via an access-log filter, +# so the policy-engine never even receives the request. Matched case-sensitively +# against the path the client called (Envoy's x-envoy-original-path, set whenever +# a route rewrite occurred), never against the rewritten backend path. Requests +# without that header (no rewrite occurred) are never suppressed by this filter — +# matching bare :path would risk suppressing traffic whose backend path merely +# happens to share the prefix. Empty by default (no suppression). +ignore_path_prefixes = [] +# ignore_path_prefixes = ["/health", "/metrics"] + # ALS transport tuning (advanced; defaults are sensible). One section read by BOTH # ends of the Envoy → policy-engine access-log stream: # • gateway-controller — configures Envoy's gRPC access-log sink (the sender) diff --git a/gateway/gateway-controller/pkg/config/config.go b/gateway/gateway-controller/pkg/config/config.go index 76e3090dd4..58a27d32c9 100644 --- a/gateway/gateway-controller/pkg/config/config.go +++ b/gateway/gateway-controller/pkg/config/config.go @@ -76,6 +76,14 @@ type CollectorConfig struct { // the collected event without attaching a per-API header policy. RequestHeaders bool `koanf:"request_headers"` ResponseHeaders bool `koanf:"response_headers"` + // IgnorePathPrefixes lists request path prefixes for which the collector + // produces no analytics event and no traffic-log line at all, as if the + // collector were disabled for that one request (e.g. health-check or + // metrics-scrape endpoints). Enforced by attaching an Envoy AccessLogFilter + // to the ALS access log (see xds.buildIgnorePathsAccessLogFilter) — Envoy + // itself never emits the log entry for a matching path, so the policy-engine + // never receives the request at all. Matched case-sensitively by prefix. + IgnorePathPrefixes []string `koanf:"ignore_path_prefixes"` // Server tunes the Envoy→policy-engine gRPC access-log (ALS) // transport that ships collected data. It is part of the collector and is // configured under the shared [collector.server] section (the policy-engine reads @@ -1758,6 +1766,7 @@ func validateDomains(field string, domains []string) error { func (c *Config) validateCollectorConfig() error { c.migrateDeprecatedAnalyticsCapture() c.migrateDeprecatedAnalyticsTransport() + c.Collector.IgnorePathPrefixes = normalizeIgnorePathPrefixes(c.Collector.IgnorePathPrefixes) if c.IsCollectorEnabled() { if err := validateGRPCEventServerConfig(c.Collector.Server); err != nil { @@ -1767,6 +1776,27 @@ func (c *Config) validateCollectorConfig() error { return nil } +// normalizeIgnorePathPrefixes trims whitespace and drops empty entries from the +// configured ignore-path prefixes. An unfiltered empty string would match every +// path via strings.HasPrefix, silently blackholing all collector output, so +// empty entries (including ones that are empty only after trimming) are +// dropped rather than passed through. Returns nil if nothing survives. +func normalizeIgnorePathPrefixes(prefixes []string) []string { + if len(prefixes) == 0 { + return nil + } + out := make([]string, 0, len(prefixes)) + for _, p := range prefixes { + if trimmed := strings.TrimSpace(p); trimmed != "" { + out = append(out, trimmed) + } + } + if len(out) == 0 { + return nil + } + return out +} + // IsCollectorEnabled reports whether the collector should run. The collector is // implicit: it is active whenever any consumer of the collected data is enabled // (analytics or stdout traffic logging), and off otherwise. When active, the diff --git a/gateway/gateway-controller/pkg/config/config_test.go b/gateway/gateway-controller/pkg/config/config_test.go index 882e585a96..7e6e0e4285 100644 --- a/gateway/gateway-controller/pkg/config/config_test.go +++ b/gateway/gateway-controller/pkg/config/config_test.go @@ -1450,6 +1450,28 @@ func TestConfig_ValidateAnalyticsTransportMigration_SkippedWhenAnalyticsDisabled assert.Equal(t, defaultGRPCEventServerConfig(), cfg.Collector.Server) } +func TestConfig_ValidateCollectorConfig_IgnorePathPrefixesNormalized(t *testing.T) { + tests := []struct { + name string + input []string + want []string + }{ + {name: "absent -> nil", input: nil, want: nil}, + {name: "empty slice -> nil", input: []string{}, want: nil}, + {name: "all blank -> nil", input: []string{"", " "}, want: nil}, + {name: "trims whitespace and drops empties", input: []string{" /health ", "", "/metrics"}, want: []string{"/health", "/metrics"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := validConfig() + cfg.Collector.IgnorePathPrefixes = tt.input + + require.NoError(t, cfg.Validate()) + assert.Equal(t, tt.want, cfg.Collector.IgnorePathPrefixes) + }) + } +} + func TestConfig_ValidateAuthConfig(t *testing.T) { tests := []struct { name string diff --git a/gateway/gateway-controller/pkg/xds/translator.go b/gateway/gateway-controller/pkg/xds/translator.go index c7837005e3..0868125c5f 100644 --- a/gateway/gateway-controller/pkg/xds/translator.go +++ b/gateway/gateway-controller/pkg/xds/translator.go @@ -2831,13 +2831,102 @@ func (t *Translator) createGRPCAccessLog() (*accesslog.AccessLog, error) { } return &accesslog.AccessLog{ - Name: "envoy.access_loggers.http_grpc", + Name: "envoy.access_loggers.http_grpc", + Filter: buildIgnorePathsAccessLogFilter(t.config.Collector.IgnorePathPrefixes), ConfigType: &accesslog.AccessLog_TypedConfig{ TypedConfig: grpcAccessLogAny, }, }, nil } +// envoyOriginalPathHeader is the header Envoy's router sets to the pre-rewrite, +// client-facing path whenever a route applies a path rewrite (PrefixRewrite/ +// RegexRewrite) — which every proxied API route in this gateway does, for +// context-path stripping. buildIgnorePathsAccessLogFilter matches against this +// header rather than :path so that collector.ignore_path_prefixes is evaluated +// against what the client actually called, not the rewritten backend path. +const envoyOriginalPathHeader = "x-envoy-original-path" + +// buildIgnorePathsAccessLogFilter builds an Envoy AccessLogFilter that suppresses +// the ALS access-log entry entirely for requests whose client-facing path matches +// one of the configured prefixes, so the policy-engine never receives them (no +// analytics event, no traffic-log line). Returns nil (no filter, log everything) +// when no usable prefixes are configured. +func buildIgnorePathsAccessLogFilter(prefixes []string) *accesslog.AccessLogFilter { + filters := make([]*accesslog.AccessLogFilter, 0, len(prefixes)) + for _, prefix := range prefixes { + prefix = strings.TrimSpace(prefix) + if prefix == "" { + continue + } + filters = append(filters, notEffectivelyMatchesPrefix(prefix)) + } + switch len(filters) { + case 0: + return nil + case 1: + return filters[0] + default: + return &accesslog.AccessLogFilter{ + FilterSpecifier: &accesslog.AccessLogFilter_AndFilter{ + AndFilter: &accesslog.AndFilter{Filters: filters}, + }, + } + } +} + +// notEffectivelyMatchesPrefix builds the per-prefix filter: OR(origAbsent, +// origDoesNotMatch) — i.e. suppress the log entry (filter evaluates false) only +// when x-envoy-original-path is present AND matches the prefix; log (filter +// evaluates true) in every other case, including when the header is absent +// entirely. There is deliberately no fallback to matching bare :path: without +// the original-path header we cannot distinguish a route that genuinely never +// rewrites (where :path is the real client path) from an upstream/backend path +// that merely happens to share the prefix, so we always log rather than risk a +// false-positive suppression of legitimate traffic. +func notEffectivelyMatchesPrefix(prefix string) *accesslog.AccessLogFilter { + origAbsent := headerPresentFilter(envoyOriginalPathHeader, false) + origDoesNotMatch := headerPrefixFilter(envoyOriginalPathHeader, prefix, true) + return &accesslog.AccessLogFilter{ + FilterSpecifier: &accesslog.AccessLogFilter_OrFilter{ + OrFilter: &accesslog.OrFilter{ + Filters: []*accesslog.AccessLogFilter{origAbsent, origDoesNotMatch}, + }, + }, + } +} + +// headerPrefixFilter builds an AccessLogFilter matching (or, when invert is +// true, not matching) a header's value against a prefix. +func headerPrefixFilter(headerName, prefix string, invert bool) *accesslog.AccessLogFilter { + return &accesslog.AccessLogFilter{ + FilterSpecifier: &accesslog.AccessLogFilter_HeaderFilter{ + HeaderFilter: &accesslog.HeaderFilter{ + Header: &route.HeaderMatcher{ + Name: headerName, + InvertMatch: invert, + HeaderMatchSpecifier: &route.HeaderMatcher_PrefixMatch{PrefixMatch: prefix}, + }, + }, + }, + } +} + +// headerPresentFilter builds an AccessLogFilter matching whether a header is +// present (present=true) or absent (present=false). +func headerPresentFilter(headerName string, present bool) *accesslog.AccessLogFilter { + return &accesslog.AccessLogFilter{ + FilterSpecifier: &accesslog.AccessLogFilter_HeaderFilter{ + HeaderFilter: &accesslog.HeaderFilter{ + Header: &route.HeaderMatcher{ + Name: headerName, + HeaderMatchSpecifier: &route.HeaderMatcher_PresentMatch{PresentMatch: present}, + }, + }, + }, + } +} + // createTracingConfig creates tracing configuration for HCM if tracing is enabled func (t *Translator) createTracingConfig() (*hcm.HttpConnectionManager_Tracing, error) { // Return nil if tracing is not enabled diff --git a/gateway/gateway-controller/pkg/xds/translator_test.go b/gateway/gateway-controller/pkg/xds/translator_test.go index 72befbb871..952d15437a 100644 --- a/gateway/gateway-controller/pkg/xds/translator_test.go +++ b/gateway/gateway-controller/pkg/xds/translator_test.go @@ -27,6 +27,7 @@ import ( "testing" "time" + accesslog "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v3" cluster "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" core "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" listener "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" @@ -1427,6 +1428,27 @@ func TestTranslator_CreateGRPCAccessLog(t *testing.T) { accessLog, err := translator.createGRPCAccessLog() assert.NoError(t, err) assert.NotNil(t, accessLog) + assert.Nil(t, accessLog.Filter, "no ignore_path_prefixes configured -> no filter") +} + +func TestTranslator_CreateGRPCAccessLog_WithIgnorePathPrefixes(t *testing.T) { + logger := createTestLogger() + routerCfg := testRouterConfig() + cfg := testConfig() + cfg.Collector.Server = config.GRPCEventServerConfig{ + Mode: "tcp", + Port: 18090, + BufferFlushInterval: 1000, + BufferSizeBytes: 16384, + GRPCRequestTimeout: 5000, + } + cfg.Collector.IgnorePathPrefixes = []string{"/health"} + translator := NewTranslator(logger, routerCfg, nil, cfg) + + accessLog, err := translator.createGRPCAccessLog() + assert.NoError(t, err) + assert.NotNil(t, accessLog) + assert.NotNil(t, accessLog.Filter, "ignore_path_prefixes configured -> filter attached") } func TestTranslator_CreateGRPCAccessLog_BufferSizeOverflow(t *testing.T) { @@ -1449,6 +1471,142 @@ func TestTranslator_CreateGRPCAccessLog_BufferSizeOverflow(t *testing.T) { assert.Contains(t, err.Error(), "buffer_size_bytes") } +// evalAccessLogFilter walks a constructed AccessLogFilter tree and evaluates it +// against a synthetic header set, mirroring how Envoy itself would evaluate the +// filter. This proves actual matching behavior, not just proto shape. +func evalAccessLogFilter(t *testing.T, filter *accesslog.AccessLogFilter, headers map[string]string) bool { + t.Helper() + switch fs := filter.FilterSpecifier.(type) { + case *accesslog.AccessLogFilter_HeaderFilter: + return evalHeaderMatcher(t, fs.HeaderFilter.Header, headers) + case *accesslog.AccessLogFilter_AndFilter: + for _, f := range fs.AndFilter.Filters { + if !evalAccessLogFilter(t, f, headers) { + return false + } + } + return true + case *accesslog.AccessLogFilter_OrFilter: + for _, f := range fs.OrFilter.Filters { + if evalAccessLogFilter(t, f, headers) { + return true + } + } + return false + default: + t.Fatalf("evalAccessLogFilter: unsupported filter specifier %T", fs) + return false + } +} + +func evalHeaderMatcher(t *testing.T, m *route.HeaderMatcher, headers map[string]string) bool { + t.Helper() + val, present := headers[m.Name] + var result bool + switch spec := m.HeaderMatchSpecifier.(type) { + case *route.HeaderMatcher_PresentMatch: + result = present == spec.PresentMatch + case *route.HeaderMatcher_PrefixMatch: + result = present && strings.HasPrefix(val, spec.PrefixMatch) + default: + t.Fatalf("evalHeaderMatcher: unsupported header match specifier %T", spec) + } + if m.InvertMatch { + result = !result + } + return result +} + +func TestBuildIgnorePathsAccessLogFilter(t *testing.T) { + t.Run("nil prefixes -> nil filter", func(t *testing.T) { + assert.Nil(t, buildIgnorePathsAccessLogFilter(nil)) + }) + + t.Run("empty prefixes -> nil filter", func(t *testing.T) { + assert.Nil(t, buildIgnorePathsAccessLogFilter([]string{})) + }) + + t.Run("whitespace-only entries -> nil filter", func(t *testing.T) { + assert.Nil(t, buildIgnorePathsAccessLogFilter([]string{"", " "})) + }) + + t.Run("single prefix -> unwrapped per-prefix filter", func(t *testing.T) { + filter := buildIgnorePathsAccessLogFilter([]string{"/health"}) + require.NotNil(t, filter) + _, isAnd := filter.FilterSpecifier.(*accesslog.AccessLogFilter_AndFilter) + assert.False(t, isAnd, "single prefix should not be wrapped in an outer AndFilter") + + assert.False(t, evalAccessLogFilter(t, filter, map[string]string{ + "x-envoy-original-path": "/health/live", + }), "matching original path -> suppressed") + assert.True(t, evalAccessLogFilter(t, filter, map[string]string{ + "x-envoy-original-path": "/orders", + }), "non-matching original path -> logged") + assert.True(t, evalAccessLogFilter(t, filter, map[string]string{ + ":path": "/health/live", + }), "no original-path header -> logged regardless of :path") + }) + + t.Run("multiple prefixes -> outer AndFilter", func(t *testing.T) { + filter := buildIgnorePathsAccessLogFilter([]string{"/health", "/metrics", ""}) + require.NotNil(t, filter) + andFilter, isAnd := filter.FilterSpecifier.(*accesslog.AccessLogFilter_AndFilter) + require.True(t, isAnd, "multiple prefixes should be wrapped in an outer AndFilter") + assert.Len(t, andFilter.AndFilter.Filters, 2, "blank entry must be dropped") + + assert.False(t, evalAccessLogFilter(t, filter, map[string]string{ + "x-envoy-original-path": "/health/live", + }), "matches first prefix -> suppressed") + assert.False(t, evalAccessLogFilter(t, filter, map[string]string{ + "x-envoy-original-path": "/metrics/scrape", + }), "matches second prefix -> suppressed") + assert.True(t, evalAccessLogFilter(t, filter, map[string]string{ + "x-envoy-original-path": "/orders", + }), "matches neither prefix -> logged") + }) +} + +func TestNotEffectivelyMatchesPrefix(t *testing.T) { + tests := []struct { + name string + headers map[string]string + wantLog bool + }{ + { + name: "original present and has prefix -> suppress even if :path (rewritten backend path) differs", + headers: map[string]string{envoyOriginalPathHeader: "/health/live", ":path": "/some/rewritten/backend/path"}, + wantLog: false, + }, + { + name: "original present and does not have prefix -> log, original is authoritative", + headers: map[string]string{envoyOriginalPathHeader: "/orders", ":path": "/health"}, + wantLog: true, + }, + { + name: "original absent, :path happens to have prefix -> log anyway, no :path fallback", + headers: map[string]string{":path": "/health/live"}, + wantLog: true, + }, + { + name: "original absent, :path does not have prefix -> log", + headers: map[string]string{":path": "/orders"}, + wantLog: true, + }, + { + name: "no headers at all -> log", + headers: map[string]string{}, + wantLog: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + filter := notEffectivelyMatchesPrefix("/health") + assert.Equal(t, tt.wantLog, evalAccessLogFilter(t, filter, tt.headers)) + }) + } +} + func TestTranslator_CreateDynamicForwardProxyCluster(t *testing.T) { logger := createTestLogger() routerCfg := testRouterConfig() From 096285a6ad621b188a6c028d2bd1eb60eb33237a Mon Sep 17 00:00:00 2001 From: Dineth Date: Thu, 9 Jul 2026 11:55:02 +0530 Subject: [PATCH 18/21] refactor: enhance deprecation warnings for request and response body settings --- common/collector/collector.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/common/collector/collector.go b/common/collector/collector.go index 5d5e7d8004..f273c778fb 100644 --- a/common/collector/collector.go +++ b/common/collector/collector.go @@ -60,13 +60,21 @@ func MigrateDeprecatedCapture(analyticsEnabled bool, deprecated CaptureFlags, co if !analyticsEnabled { return } - if deprecated.SendRequestBody && !*collectorSendRequestBody { - slog.Warn("analytics.send_request_body is deprecated; use collector.request_body instead") - *collectorSendRequestBody = true + if deprecated.SendRequestBody { + if !*collectorSendRequestBody { + slog.Warn("analytics.send_request_body is deprecated; use collector.request_body instead") + *collectorSendRequestBody = true + } else { + slog.Warn("analytics.send_request_body is deprecated and collector.request_body is already configured; ignoring the analytics.send_request_body override") + } } - if deprecated.SendResponseBody && !*collectorSendResponseBody { - slog.Warn("analytics.send_response_body is deprecated; use collector.response_body instead") - *collectorSendResponseBody = true + if deprecated.SendResponseBody { + if !*collectorSendResponseBody { + slog.Warn("analytics.send_response_body is deprecated; use collector.response_body instead") + *collectorSendResponseBody = true + } else { + slog.Warn("analytics.send_response_body is deprecated and collector.response_body is already configured; ignoring the analytics.send_response_body override") + } } if deprecated.AllowPayloads { slog.Warn("analytics.allow_payloads is deprecated; use collector.request_body and collector.response_body instead") From 22f437c3c71801880194dd536b781328db29fe16 Mon Sep 17 00:00:00 2001 From: Dineth Date: Thu, 9 Jul 2026 14:27:28 +0530 Subject: [PATCH 19/21] Preserve prior traffic log marker during short-circuit in request header actions --- .../internal/kernel/translator.go | 36 +++++++++- .../internal/kernel/translator_test.go | 66 +++++++++++++++++++ 2 files changed, 101 insertions(+), 1 deletion(-) diff --git a/gateway/gateway-runtime/policy-engine/internal/kernel/translator.go b/gateway/gateway-runtime/policy-engine/internal/kernel/translator.go index 8cd30d85a6..5c72e5e478 100644 --- a/gateway/gateway-runtime/policy-engine/internal/kernel/translator.go +++ b/gateway/gateway-runtime/policy-engine/internal/kernel/translator.go @@ -387,7 +387,41 @@ func TranslateRequestHeaderActions(result *executor.RequestHeaderExecutionResult }, }, } - analyticsStruct, err := buildAnalyticsStruct(immResp.AnalyticsMetadata, execCtx) + // Preserve request-header-phase analytics metadata from policies that + // executed before the short-circuit (e.g. the log-message traffic-log + // marker) so an immediate response like a 401 still carries it to the + // ALS access log. Without this, a short-circuiting policy (auth) drops + // the marker of any earlier policy and the traffic-log line is never + // emitted. Mirrors translateRequestActionsCore's short-circuit path. + shortCircuitAnalyticsData := make(map[string]any) + for key, value := range execCtx.analyticsMetadata { + shortCircuitAnalyticsData[key] = value + } + for _, policyResult := range result.Results { + if policyResult.Skipped || policyResult.Action == nil { + continue + } + mods, ok := policyResult.Action.(policy.UpstreamRequestHeaderModifications) + if !ok { + continue + } + if mods.AnalyticsMetadata != nil { + for key, value := range mods.AnalyticsMetadata { + shortCircuitAnalyticsData[key] = value + } + } + dropAction := mods.AnalyticsHeaderFilter + if dropAction.Action != "" || len(dropAction.Headers) > 0 { + originalHeaders := execCtx.requestBodyCtx.Headers.GetAll() + shortCircuitAnalyticsData["request_headers"] = finalizeAnalyticsHeaders(dropAction, originalHeaders) + } + } + if immResp.AnalyticsMetadata != nil { + for key, value := range immResp.AnalyticsMetadata { + shortCircuitAnalyticsData[key] = value + } + } + analyticsStruct, err := buildAnalyticsStruct(shortCircuitAnalyticsData, execCtx) if err != nil { return nil, fmt.Errorf("failed to build analytics metadata for immediate response: %w", err) } diff --git a/gateway/gateway-runtime/policy-engine/internal/kernel/translator_test.go b/gateway/gateway-runtime/policy-engine/internal/kernel/translator_test.go index 669d4e8c18..6aad945e10 100644 --- a/gateway/gateway-runtime/policy-engine/internal/kernel/translator_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/kernel/translator_test.go @@ -608,6 +608,72 @@ func TestTranslateRequestActionsCore_ShortCircuit_PreservesPriorRequestAnalytics assert.Equal(t, "immediate-response", analyticsData.GetFields()["source"].GetStringValue()) } +// TestTranslateRequestHeaderActions_ShortCircuit_PreservesPriorTrafficLogMarker +// covers the request-header phase short-circuit path: the log-message policy +// (traffic-logging mode) stamps its marker as request-header-phase analytics +// metadata, then an auth policy rejects the request with 401 and short-circuits +// the chain. The prior policy's marker must survive onto the immediate response's +// dynamic metadata, otherwise the ALS access-log entry carries no traffic_log key +// and the stdout traffic-log line is never emitted for rejected requests. +func TestTranslateRequestHeaderActions_ShortCircuit_PreservesPriorTrafficLogMarker(t *testing.T) { + kernel := NewKernel() + chainExecutor := executor.NewChainExecutor(nil, nil, nil) + server := NewExternalProcessorServer(kernel, chainExecutor, config.TracingConfig{}, "") + + chain := ®istry.PolicyChain{} + execCtx := newPolicyExecutionContext(server, "test-route", chain) + execCtx.requestBodyCtx = &policy.RequestContext{ + Path: "/api/test", + SharedContext: &policy.SharedContext{ + APIId: "api-1", + }, + } + + result := &executor.RequestHeaderExecutionResult{ + ShortCircuited: true, + Results: []executor.RequestHeaderPolicyResult{ + { + Skipped: false, + Action: policy.UpstreamRequestHeaderModifications{ + AnalyticsMetadata: map[string]any{ + "traffic_log": `{"request":{"headers":true}}`, + }, + }, + }, + }, + FinalAction: policy.ImmediateResponse{ + StatusCode: 401, + Headers: map[string]string{ + "content-type": "application/json", + }, + Body: []byte(`{"error":"unauthorized"}`), + AnalyticsMetadata: map[string]any{ + "source": "immediate-response", + }, + }, + } + + resp, err := TranslateRequestHeaderActions(result, chain, execCtx) + + assert.NoError(t, err) + require.NotNil(t, resp) + + immediate := resp.GetImmediateResponse() + require.NotNil(t, immediate) + assert.Equal(t, uint32(401), uint32(immediate.Status.Code)) + + extProcNamespace := resp.DynamicMetadata.GetFields()[constants.ExtProcFilterName].GetStructValue() + require.NotNil(t, extProcNamespace) + + analyticsData := extProcNamespace.GetFields()["analytics_data"].GetStructValue() + require.NotNil(t, analyticsData) + + // The traffic-log marker stamped by the earlier policy survives the short-circuit... + assert.Equal(t, `{"request":{"headers":true}}`, analyticsData.GetFields()["traffic_log"].GetStringValue()) + // ...and the immediate response's own analytics metadata is still present. + assert.Equal(t, "immediate-response", analyticsData.GetFields()["source"].GetStringValue()) +} + func TestTranslateRequestActionsCore_SkippedPolicy(t *testing.T) { kernel := NewKernel() chainExecutor := executor.NewChainExecutor(nil, nil, nil) From c23f97d0633c9593a2d1db3a40892bc779285aa2 Mon Sep 17 00:00:00 2001 From: Dineth Date: Thu, 9 Jul 2026 15:00:02 +0530 Subject: [PATCH 20/21] feat: add collector and traffic logging configuration options in gateway config --- .../templates/gateway/gateway-config.yaml | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/kubernetes/helm/gateway-helm-chart/templates/gateway/gateway-config.yaml b/kubernetes/helm/gateway-helm-chart/templates/gateway/gateway-config.yaml index a90894b5cf..28827f9449 100644 --- a/kubernetes/helm/gateway-helm-chart/templates/gateway/gateway-config.yaml +++ b/kubernetes/helm/gateway-helm-chart/templates/gateway/gateway-config.yaml @@ -256,6 +256,34 @@ data: {{- end }} {{- end }} + {{- if .Values.gateway.config.collector }} + {{- if .Values.gateway.config.collector.server }} + [collector.server] + buffer_flush_interval = {{ .Values.gateway.config.collector.server.buffer_flush_interval | quote }} + buffer_size_bytes = {{ .Values.gateway.config.collector.server.buffer_size_bytes | int64 }} + grpc_request_timeout = {{ .Values.gateway.config.collector.server.grpc_request_timeout | quote }} + server_port = {{ .Values.gateway.config.collector.server.server_port | int64 }} + shutdown_timeout = {{ .Values.gateway.config.collector.server.shutdown_timeout | quote }} + public_key_path = {{ .Values.gateway.config.collector.server.public_key_path | quote }} + private_key_path = {{ .Values.gateway.config.collector.server.private_key_path | quote }} + als_plain_text = {{ .Values.gateway.config.collector.server.als_plain_text }} + max_message_size = {{ .Values.gateway.config.collector.server.max_message_size | int64 }} + max_header_limit = {{ .Values.gateway.config.collector.server.max_header_limit | int64 }} + {{- end }} + {{- end }} + + {{- if .Values.gateway.config.traffic_logging }} + {{- $trafficLoggingEnabled := and (kindIs "bool" .Values.gateway.config.traffic_logging.enabled) .Values.gateway.config.traffic_logging.enabled }} + [traffic_logging] + enabled = {{ $trafficLoggingEnabled }} + {{- if .Values.gateway.config.traffic_logging.masked_headers }} + masked_headers = [{{- range $i, $h := .Values.gateway.config.traffic_logging.masked_headers }}{{- if gt $i 0 }}, {{ end }}{{ $h | quote }}{{- end }}] + {{- end }} + {{- if .Values.gateway.config.traffic_logging.max_payload_size }} + max_payload_size = {{ .Values.gateway.config.traffic_logging.max_payload_size | int64 }} + {{- end }} + {{- end }} + {{- if .Values.gateway.config.tracing }} [tracing] enabled = {{ .Values.gateway.config.tracing.enabled }} From 9ed7707a6a83b04a795be259c0ebeb8bcb9f4e57 Mon Sep 17 00:00:00 2001 From: Dineth Date: Thu, 9 Jul 2026 15:40:09 +0530 Subject: [PATCH 21/21] Standardize collector server port configuration and deprecate overrides --- common/collector/collector.go | 9 ++++ gateway/configs/config-template.toml | 10 +++-- .../gateway-controller/pkg/config/config.go | 38 +++++++++------- .../pkg/config/config_test.go | 42 ++++++++---------- .../gateway-controller/pkg/xds/translator.go | 11 ++++- .../pkg/xds/translator_test.go | 28 +++++++++--- .../policy-engine/internal/config/config.go | 43 ++++++++++++------- .../internal/config/config_test.go | 25 +++++------ .../internal/utils/access_logger_server.go | 13 ++++-- .../utils/access_logger_server_test.go | 2 +- .../templates/gateway/gateway-config.yaml | 1 - 11 files changed, 135 insertions(+), 87 deletions(-) diff --git a/common/collector/collector.go b/common/collector/collector.go index f273c778fb..307e57dda9 100644 --- a/common/collector/collector.go +++ b/common/collector/collector.go @@ -30,6 +30,15 @@ package collector import "log/slog" +// ServerPort is the fixed TCP port for the Envoy → policy-engine ALS +// (access-log service) gRPC stream in "tcp" mode: the gateway-controller +// dials it (translator.go, Envoy cluster config) and the policy-engine +// listens on it (access_logger_server.go). It is intentionally +// non-configurable — the two sides previously had independent config fields +// that defaulted to the same value but could drift apart, silently breaking +// the connection; a single shared constant makes that impossible. +const ServerPort = 18090 + // IsEnabled reports whether the collector should run: implicitly active // whenever any consumer of the collected data (analytics or stdout traffic // logging) is enabled, and off otherwise. diff --git a/gateway/configs/config-template.toml b/gateway/configs/config-template.toml index cdfa7659db..5ec4c033e6 100644 --- a/gateway/configs/config-template.toml +++ b/gateway/configs/config-template.toml @@ -319,12 +319,14 @@ ignore_path_prefixes = [] # ends of the Envoy → policy-engine access-log stream: # • gateway-controller — configures Envoy's gRPC access-log sink (the sender) # • policy-engine — runs the receiving gRPC server -# Shared keys (server_port, TLS, message/header limits) must match on both ends and -# live here once. Envoy-sender-only keys (buffer_*, grpc_request_timeout) are read -# only by the controller and ignored by the policy-engine. +# Shared keys (TLS, message/header limits) must match on both ends and live here +# once. Envoy-sender-only keys (buffer_*, grpc_request_timeout) are read only by +# the controller and ignored by the policy-engine. +# NOTE: the TCP port (18090) is not configurable here — it is a fixed constant +# (collector.ServerPort in common/collector) shared by both ends, so they can +# never disagree on it. See ANALYTICS_LOG_PUBLISHER_DESIGN.md changelog. [collector.server] mode = "uds" # "uds" (default) or "tcp" -server_port = 18090 # engine listens here; Envoy dials it (tcp mode) buffer_flush_interval = 1000000000 # Envoy sender only (nanoseconds) buffer_size_bytes = 16384 # Envoy sender only grpc_request_timeout = 20000000000 # Envoy sender only (nanoseconds) diff --git a/gateway/gateway-controller/pkg/config/config.go b/gateway/gateway-controller/pkg/config/config.go index 58a27d32c9..62a54ab6dc 100644 --- a/gateway/gateway-controller/pkg/config/config.go +++ b/gateway/gateway-controller/pkg/config/config.go @@ -20,6 +20,7 @@ package config import ( "fmt" + "log/slog" "net/url" "strconv" "strings" @@ -153,9 +154,14 @@ type MoesifPublisherConfig struct { // GRPCEventServerConfig holds configuration for gRPC event server (combines access log service and ALS server config) type GRPCEventServerConfig struct { - Mode string `koanf:"mode"` // Connection mode: "uds" (default) or "tcp" - Port int `koanf:"port"` // ALS port for Envoy connection (TCP mode only) - ServerPort int `koanf:"server_port"` // gRPC server port for ALS server + Mode string `koanf:"mode"` // Connection mode: "uds" (default) or "tcp" + // Port overrides the fixed Envoy→policy-engine ALS dial port (collector.ServerPort, + // 18090), used only in "tcp" mode. Deprecated: no longer defaulted here or documented + // in config-template.toml/Helm charts, so new deployments have no way to discover or + // set it. Kept solely so a config that already sets it explicitly keeps working; + // leave unset (0) to use the fixed port. Must match the policy-engine's + // collector.server.server_port override, or the two sides will fail to connect. + Port int `koanf:"port"` BufferFlushInterval int `koanf:"buffer_flush_interval"` // Envoy buffer flush interval (nanoseconds) BufferSizeBytes int `koanf:"buffer_size_bytes"` // Envoy buffer size GRPCRequestTimeout int `koanf:"grpc_request_timeout"` // Envoy gRPC timeout (nanoseconds) @@ -742,8 +748,6 @@ func LoadConfig(configPath string) (*Config, error) { func defaultGRPCEventServerConfig() GRPCEventServerConfig { return GRPCEventServerConfig{ Mode: "uds", // UDS mode by default - Port: 18090, // Only used in TCP mode - ServerPort: 18090, // ALS server port BufferFlushInterval: 1000000000, // 1 second BufferSizeBytes: 16384, // 16 KiB GRPCRequestTimeout: 20000000000, // 20 seconds @@ -1840,20 +1844,26 @@ func (c *Config) migrateDeprecatedAnalyticsCapture() { } // validateGRPCEventServerConfig validates the Envoy→policy-engine ALS transport tuning. +// The transport port is normally the fixed, non-configurable collector.ServerPort +// constant (see collector.ServerPort); cfg.Port is a deprecated override honored only +// for backward compatibility with configs that already set it (see its doc comment). func validateGRPCEventServerConfig(cfg GRPCEventServerConfig) error { // Validate connection mode switch cfg.Mode { - case "uds", "": - // UDS mode (default) - port is unused for Envoy connection - case "tcp": - // TCP mode - validate port (host is derived from policy_engine.host) - if cfg.Port <= 0 || cfg.Port > 65535 { - return fmt.Errorf("collector.server.port must be between 1 and 65535 when mode is tcp, got %d", cfg.Port) - } + case "uds", "tcp", "": default: return fmt.Errorf("collector.server.mode must be 'uds' or 'tcp', got: %s", cfg.Mode) } + if cfg.Port != 0 { + slog.Warn("collector.server.port is deprecated and no longer documented; the ALS port is fixed at " + + strconv.Itoa(collector.ServerPort) + " by default. Honoring the configured override for backward " + + "compatibility — ensure the policy-engine's collector.server.server_port matches, or the two sides will fail to connect.") + if cfg.Port < 0 || cfg.Port > 65535 { + return fmt.Errorf("collector.server.port must be between 1 and 65535, got %d", cfg.Port) + } + } + // Validate buffer and timeout settings if cfg.BufferFlushInterval <= 0 || cfg.BufferSizeBytes <= 0 || cfg.GRPCRequestTimeout <= 0 { return fmt.Errorf( @@ -1864,10 +1874,6 @@ func validateGRPCEventServerConfig(cfg GRPCEventServerConfig) error { ) } - // Validate server port - if cfg.ServerPort <= 0 || cfg.ServerPort > 65535 { - return fmt.Errorf("collector.server.server_port must be between 1 and 65535, got %d", cfg.ServerPort) - } return nil } diff --git a/gateway/gateway-controller/pkg/config/config_test.go b/gateway/gateway-controller/pkg/config/config_test.go index 7e6e0e4285..c154d47649 100644 --- a/gateway/gateway-controller/pkg/config/config_test.go +++ b/gateway/gateway-controller/pkg/config/config_test.go @@ -1217,7 +1217,6 @@ func TestConfig_ValidateAnalyticsConfig(t *testing.T) { cfg.Collector.Server.BufferFlushInterval = 1000 cfg.Collector.Server.BufferSizeBytes = 16384 cfg.Collector.Server.GRPCRequestTimeout = 5000 - cfg.Collector.Server.ServerPort = 18090 }, wantErr: false, }, @@ -1226,11 +1225,9 @@ func TestConfig_ValidateAnalyticsConfig(t *testing.T) { enabled: true, setupConfig: func(cfg *Config) { cfg.Collector.Server.Mode = "tcp" - cfg.Collector.Server.Port = 18090 cfg.Collector.Server.BufferFlushInterval = 1000 cfg.Collector.Server.BufferSizeBytes = 16384 cfg.Collector.Server.GRPCRequestTimeout = 5000 - cfg.Collector.Server.ServerPort = 18090 }, wantErr: false, }, @@ -1242,7 +1239,6 @@ func TestConfig_ValidateAnalyticsConfig(t *testing.T) { cfg.Collector.Server.BufferFlushInterval = 1000 cfg.Collector.Server.BufferSizeBytes = 16384 cfg.Collector.Server.GRPCRequestTimeout = 5000 - cfg.Collector.Server.ServerPort = 18090 }, wantErr: false, }, @@ -1254,48 +1250,46 @@ func TestConfig_ValidateAnalyticsConfig(t *testing.T) { cfg.Collector.Server.BufferFlushInterval = 1000 cfg.Collector.Server.BufferSizeBytes = 16384 cfg.Collector.Server.GRPCRequestTimeout = 5000 - cfg.Collector.Server.ServerPort = 18090 }, wantErr: true, errContains: "collector.server.mode must be 'uds' or 'tcp'", }, { - name: "TCP mode - invalid port", + name: "Invalid buffer flush interval", enabled: true, setupConfig: func(cfg *Config) { - cfg.Collector.Server.Mode = "tcp" - cfg.Collector.Server.Port = 0 - cfg.Collector.Server.BufferFlushInterval = 1000 - cfg.Collector.Server.BufferSizeBytes = 16384 - cfg.Collector.Server.GRPCRequestTimeout = 5000 - cfg.Collector.Server.ServerPort = 18090 + cfg.Collector.Server.Mode = "uds" + cfg.Collector.Server.BufferFlushInterval = 0 }, wantErr: true, - errContains: "collector.server.port must be between 1 and 65535", + errContains: "invalid gRPC event server configuration", }, { - name: "Invalid server port", + // Backward compat: an existing config that already sets a custom port + // (the deprecated Port override) must keep working, not error. + name: "Deprecated port override still accepted (backward compat)", enabled: true, setupConfig: func(cfg *Config) { - cfg.Collector.Server.Mode = "uds" + cfg.Collector.Server.Mode = "tcp" + cfg.Collector.Server.Port = 9099 cfg.Collector.Server.BufferFlushInterval = 1000 cfg.Collector.Server.BufferSizeBytes = 16384 cfg.Collector.Server.GRPCRequestTimeout = 5000 - cfg.Collector.Server.ServerPort = 0 }, - wantErr: true, - errContains: "collector.server.server_port must be between 1 and 65535", + wantErr: false, }, { - name: "Invalid buffer flush interval", + name: "Deprecated port override out of range still errors", enabled: true, setupConfig: func(cfg *Config) { - cfg.Collector.Server.Mode = "uds" - cfg.Collector.Server.BufferFlushInterval = 0 - cfg.Collector.Server.ServerPort = 18090 + cfg.Collector.Server.Mode = "tcp" + cfg.Collector.Server.Port = 70000 + cfg.Collector.Server.BufferFlushInterval = 1000 + cfg.Collector.Server.BufferSizeBytes = 16384 + cfg.Collector.Server.GRPCRequestTimeout = 5000 }, wantErr: true, - errContains: "invalid gRPC event server configuration", + errContains: "collector.server.port must be between 1 and 65535", }, } @@ -1351,7 +1345,6 @@ func TestConfig_ValidateAnalyticsPayloadMigration(t *testing.T) { cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 1000 cfg.Analytics.GRPCEventServerCfg.BufferSizeBytes = 16384 cfg.Analytics.GRPCEventServerCfg.GRPCRequestTimeout = 5000 - cfg.Analytics.GRPCEventServerCfg.ServerPort = 18090 } tests := []struct { @@ -1443,7 +1436,6 @@ func TestConfig_ValidateAnalyticsTransportMigration_SkippedWhenAnalyticsDisabled cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 1000 cfg.Analytics.GRPCEventServerCfg.BufferSizeBytes = 16384 cfg.Analytics.GRPCEventServerCfg.GRPCRequestTimeout = 5000 - cfg.Analytics.GRPCEventServerCfg.ServerPort = 18090 err := cfg.Validate() require.NoError(t, err) diff --git a/gateway/gateway-controller/pkg/xds/translator.go b/gateway/gateway-controller/pkg/xds/translator.go index 0868125c5f..a6dbf76d4d 100644 --- a/gateway/gateway-controller/pkg/xds/translator.go +++ b/gateway/gateway-controller/pkg/xds/translator.go @@ -31,6 +31,7 @@ import ( "strings" "time" + "github.com/wso2/api-platform/common/collector" commonconstants "github.com/wso2/api-platform/common/constants" accesslog "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v3" @@ -2162,14 +2163,20 @@ func (t *Translator) createALSCluster() *cluster.Cluster { var address *core.Address if grpcConfig.Mode == "tcp" { - // TCP mode - use host:port + // TCP mode - use host:port. grpcConfig.Port is a deprecated override + // (see its doc comment); the fixed collector.ServerPort constant is used + // unless a config already sets it explicitly. + port := collector.ServerPort + if grpcConfig.Port != 0 { + port = grpcConfig.Port + } address = &core.Address{ Address: &core.Address_SocketAddress{ SocketAddress: &core.SocketAddress{ Protocol: core.SocketAddress_TCP, Address: t.config.Router.PolicyEngine.Host, PortSpecifier: &core.SocketAddress_PortValue{ - PortValue: uint32(grpcConfig.Port), + PortValue: uint32(port), }, }, }, diff --git a/gateway/gateway-controller/pkg/xds/translator_test.go b/gateway/gateway-controller/pkg/xds/translator_test.go index 952d15437a..13b196aec0 100644 --- a/gateway/gateway-controller/pkg/xds/translator_test.go +++ b/gateway/gateway-controller/pkg/xds/translator_test.go @@ -1386,7 +1386,6 @@ func TestTranslator_CreateALSCluster(t *testing.T) { cfg.Analytics.Enabled = true cfg.Collector.Server = config.GRPCEventServerConfig{ Mode: "tcp", - Port: 18090, BufferFlushInterval: 1000000000, BufferSizeBytes: 16384, GRPCRequestTimeout: 20000000000, @@ -1410,6 +1409,29 @@ func TestTranslator_CreateALSCluster(t *testing.T) { assert.Equal(t, "policy-engine", socketAddr.Address) assert.Equal(t, uint32(18090), socketAddr.GetPortValue()) }) + + t.Run("TCP mode honors deprecated port override (backward compat)", func(t *testing.T) { + routerCfg := testRouterConfig() + cfg := testConfig() + cfg.Analytics.Enabled = true + cfg.Collector.Server = config.GRPCEventServerConfig{ + Mode: "tcp", + Port: 9099, + BufferFlushInterval: 1000000000, + BufferSizeBytes: 16384, + GRPCRequestTimeout: 20000000000, + } + cfg.Router.PolicyEngine.Host = "policy-engine" + translator := NewTranslator(logger, routerCfg, nil, cfg) + + c := translator.createALSCluster() + assert.NotNil(t, c) + + lbEndpoint := c.LoadAssignment.Endpoints[0].LbEndpoints[0] + socketAddr := lbEndpoint.GetEndpoint().Address.GetSocketAddress() + assert.NotNil(t, socketAddr) + assert.Equal(t, uint32(9099), socketAddr.GetPortValue()) + }) } func TestTranslator_CreateGRPCAccessLog(t *testing.T) { @@ -1418,7 +1440,6 @@ func TestTranslator_CreateGRPCAccessLog(t *testing.T) { cfg := testConfig() cfg.Collector.Server = config.GRPCEventServerConfig{ Mode: "tcp", - Port: 18090, BufferFlushInterval: 1000, BufferSizeBytes: 16384, GRPCRequestTimeout: 5000, @@ -1437,7 +1458,6 @@ func TestTranslator_CreateGRPCAccessLog_WithIgnorePathPrefixes(t *testing.T) { cfg := testConfig() cfg.Collector.Server = config.GRPCEventServerConfig{ Mode: "tcp", - Port: 18090, BufferFlushInterval: 1000, BufferSizeBytes: 16384, GRPCRequestTimeout: 5000, @@ -1457,11 +1477,9 @@ func TestTranslator_CreateGRPCAccessLog_BufferSizeOverflow(t *testing.T) { cfg := testConfig() cfg.Collector.Server = config.GRPCEventServerConfig{ Mode: "tcp", - Port: 18090, BufferFlushInterval: 1000, BufferSizeBytes: math.MaxInt, GRPCRequestTimeout: 5000, - ServerPort: 18090, } translator := NewTranslator(logger, routerCfg, nil, cfg) diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config.go b/gateway/gateway-runtime/policy-engine/internal/config/config.go index 49263040a6..cc5a1ba6d2 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config.go @@ -20,8 +20,10 @@ package config import ( "fmt" + "log/slog" "math" "net/url" + "strconv" "strings" "time" @@ -118,13 +120,13 @@ type MoesifPublisherConfig struct { // Config represents the complete policy engine configuration type PolicyEngine struct { - Server ServerConfig `koanf:"server"` - Admin AdminConfig `koanf:"admin"` - Metrics MetricsConfig `koanf:"metrics"` - ConfigMode ConfigModeConfig `koanf:"config_mode"` - XDS XDSConfig `koanf:"xds"` - FileConfig FileConfigConfig `koanf:"file_config"` - Logging LoggingConfig `koanf:"logging"` + Server ServerConfig `koanf:"server"` + Admin AdminConfig `koanf:"admin"` + Metrics MetricsConfig `koanf:"metrics"` + ConfigMode ConfigModeConfig `koanf:"config_mode"` + XDS XDSConfig `koanf:"xds"` + FileConfig FileConfigConfig `koanf:"file_config"` + Logging LoggingConfig `koanf:"logging"` PythonExecutor PythonExecutorConfig `koanf:"python_executor"` // Tracing holds OpenTelemetry exporter configuration TracingServiceName string `koanf:"tracing_service_name"` @@ -271,7 +273,13 @@ type LoggingConfig struct { // AccessLogsServiceConfig holds access logs service configuration type AccessLogsServiceConfig struct { - Mode string `koanf:"mode"` // Connection mode: "uds" (default) or "tcp" + Mode string `koanf:"mode"` // Connection mode: "uds" (default) or "tcp" + // ServerPort overrides the fixed ALS listener port (collector.ServerPort, 18090), + // used only in "tcp" mode. Deprecated: no longer defaulted here or documented in + // config-template.toml/Helm charts, so new deployments have no way to discover or + // set it. Kept solely so a config that already sets it explicitly keeps working; + // leave unset (0) to use the fixed port. Must match the gateway-controller's + // collector.server.port override, or the two sides will fail to connect. ServerPort int `koanf:"server_port"` ShutdownTimeout time.Duration `koanf:"shutdown_timeout"` PublicKeyPath string `koanf:"public_key_path"` @@ -346,7 +354,6 @@ func Load(configPath string) (*Config, error) { func defaultAccessLogsServiceConfig() AccessLogsServiceConfig { return AccessLogsServiceConfig{ Mode: "", - ServerPort: 18090, ShutdownTimeout: 600 * time.Second, PublicKeyPath: "", PrivateKeyPath: "", @@ -637,17 +644,23 @@ func (c *Config) migrateDeprecatedAnalyticsTransport() { } // validateAccessLogsServiceConfig validates the policy-engine ALS receiver tuning. +// The transport port is normally the fixed, non-configurable collector.ServerPort +// constant (see collector.ServerPort); als.ServerPort is a deprecated override honored +// only for backward compatibility with configs that already set it (see its doc comment). func validateAccessLogsServiceConfig(als AccessLogsServiceConfig) error { switch als.Mode { - case "uds", "": - // UDS mode (default) - port is unused - case "tcp": - if als.ServerPort <= 0 || als.ServerPort > 65535 { - return fmt.Errorf("collector.server.server_port must be between 1 and 65535, got %d", als.ServerPort) - } + case "uds", "tcp", "": default: return fmt.Errorf("collector.server.mode must be 'uds' or 'tcp', got: %s", als.Mode) } + if als.ServerPort != 0 { + slog.Warn("collector.server.server_port is deprecated and no longer documented; the ALS port is fixed at " + + strconv.Itoa(collector.ServerPort) + " by default. Honoring the configured override for backward " + + "compatibility — ensure the gateway-controller's collector.server.port matches, or the two sides will fail to connect.") + if als.ServerPort < 0 || als.ServerPort > 65535 { + return fmt.Errorf("collector.server.server_port must be between 1 and 65535, got %d", als.ServerPort) + } + } if als.ShutdownTimeout <= 0 { return fmt.Errorf("collector.server.shutdown_timeout must be positive, got %s", als.ShutdownTimeout) } diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config_test.go b/gateway/gateway-runtime/policy-engine/internal/config/config_test.go index 2d2d224964..d582275275 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config_test.go @@ -983,7 +983,6 @@ func TestValidate_AnalyticsConfig(t *testing.T) { cfg.Analytics.Enabled = true cfg.Collector.Server = AccessLogsServiceConfig{ Mode: "tcp", - ServerPort: 18090, ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, ExtProcMaxHeaderLimit: 8192, @@ -1006,33 +1005,35 @@ func TestValidate_AnalyticsConfig(t *testing.T) { errMsg: "mode must be 'uds' or 'tcp'", }, { - name: "analytics enabled - TCP mode invalid ALS port", + // Backward compat: an existing config that already sets a custom port + // (the deprecated ServerPort override) must keep working, not error. + name: "analytics enabled - deprecated server_port override still accepted (backward compat)", setup: func(cfg *Config) { cfg.Analytics.Enabled = true cfg.Collector.Server = AccessLogsServiceConfig{ Mode: "tcp", - ServerPort: 0, + ServerPort: 9099, ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, ExtProcMaxHeaderLimit: 8192, } }, - expectErr: true, - errMsg: "server_port must be between 1 and 65535", + expectErr: false, }, { - name: "analytics enabled - UDS mode skips port validation", + name: "analytics enabled - deprecated server_port override out of range still errors", setup: func(cfg *Config) { cfg.Analytics.Enabled = true cfg.Collector.Server = AccessLogsServiceConfig{ - Mode: "uds", - ServerPort: 0, // Invalid port, but irrelevant in UDS mode + Mode: "tcp", + ServerPort: 70000, ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, ExtProcMaxHeaderLimit: 8192, } }, - expectErr: false, + expectErr: true, + errMsg: "server_port must be between 1 and 65535", }, { name: "analytics enabled - invalid shutdown timeout", @@ -1263,7 +1264,6 @@ func TestValidate_AnalyticsPublishers(t *testing.T) { setup: func(cfg *Config) { cfg.Analytics.Enabled = true cfg.Collector.Server = AccessLogsServiceConfig{ - ServerPort: 18090, ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, ExtProcMaxHeaderLimit: 8192, @@ -1277,7 +1277,6 @@ func TestValidate_AnalyticsPublishers(t *testing.T) { setup: func(cfg *Config) { cfg.Analytics.Enabled = true cfg.Collector.Server = AccessLogsServiceConfig{ - ServerPort: 18090, ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, ExtProcMaxHeaderLimit: 8192, @@ -1292,7 +1291,6 @@ func TestValidate_AnalyticsPublishers(t *testing.T) { setup: func(cfg *Config) { cfg.Analytics.Enabled = true cfg.Collector.Server = AccessLogsServiceConfig{ - ServerPort: 18090, ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, ExtProcMaxHeaderLimit: 8192, @@ -1308,7 +1306,6 @@ func TestValidate_AnalyticsPublishers(t *testing.T) { setup: func(cfg *Config) { cfg.Analytics.Enabled = true cfg.Collector.Server = AccessLogsServiceConfig{ - ServerPort: 18090, ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, ExtProcMaxHeaderLimit: 8192, @@ -1325,7 +1322,6 @@ func TestValidate_AnalyticsPublishers(t *testing.T) { setup: func(cfg *Config) { cfg.Analytics.Enabled = true cfg.Collector.Server = AccessLogsServiceConfig{ - ServerPort: 18090, ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, ExtProcMaxHeaderLimit: 8192, @@ -1343,7 +1339,6 @@ func TestValidate_AnalyticsPublishers(t *testing.T) { setup: func(cfg *Config) { cfg.Analytics.Enabled = true cfg.Collector.Server = AccessLogsServiceConfig{ - ServerPort: 18090, ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, ExtProcMaxHeaderLimit: 8192, diff --git a/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server.go b/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server.go index 067681698a..e9715f7f9c 100644 --- a/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server.go +++ b/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server.go @@ -27,6 +27,7 @@ import ( "time" v3 "github.com/envoyproxy/go-control-plane/envoy/service/accesslog/v3" + "github.com/wso2/api-platform/common/collector" "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/analytics" "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/config" "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/constants" @@ -139,14 +140,20 @@ func StartAccessLogServiceServer(cfg *config.Config) *grpc.Server { } }() case "tcp": - listener, err = net.Listen("tcp", fmt.Sprintf(":%d", cfg.Collector.Server.ServerPort)) + // cfg.Collector.Server.ServerPort is a deprecated override (see its doc comment); + // the fixed collector.ServerPort constant is used unless a config already sets it. + port := collector.ServerPort + if cfg.Collector.Server.ServerPort != 0 { + port = cfg.Collector.Server.ServerPort + } + listener, err = net.Listen("tcp", fmt.Sprintf(":%d", port)) if err != nil { - slog.Error("Failed to listen on ALS TCP port", "port", cfg.Collector.Server.ServerPort) + slog.Error("Failed to listen on ALS TCP port", "port", port) panic(err) } go func() { - slog.Info("Starting to serve access log service server", "mode", "tcp", "port", cfg.Collector.Server.ServerPort) + slog.Info("Starting to serve access log service server", "mode", "tcp", "port", port) if err := server.Serve(listener); err != nil { slog.Error("ALS server exited", "error", err) } diff --git a/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server_test.go b/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server_test.go index e802cf5d62..c0ed6a3db1 100644 --- a/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server_test.go @@ -213,7 +213,7 @@ func TestStartAccessLogServiceServer_TCP(t *testing.T) { Collector: config.CollectorConfig{ Server: config.AccessLogsServiceConfig{ Mode: "tcp", - ServerPort: 19001, // Use non-standard port to avoid conflicts + ServerPort: 19001, // Deprecated override; also exercises backward compat ALSPlainText: true, ExtProcMaxMessageSize: 1024 * 1024 * 4, ExtProcMaxHeaderLimit: 8192, diff --git a/kubernetes/helm/gateway-helm-chart/templates/gateway/gateway-config.yaml b/kubernetes/helm/gateway-helm-chart/templates/gateway/gateway-config.yaml index 28827f9449..6d42baf607 100644 --- a/kubernetes/helm/gateway-helm-chart/templates/gateway/gateway-config.yaml +++ b/kubernetes/helm/gateway-helm-chart/templates/gateway/gateway-config.yaml @@ -262,7 +262,6 @@ data: buffer_flush_interval = {{ .Values.gateway.config.collector.server.buffer_flush_interval | quote }} buffer_size_bytes = {{ .Values.gateway.config.collector.server.buffer_size_bytes | int64 }} grpc_request_timeout = {{ .Values.gateway.config.collector.server.grpc_request_timeout | quote }} - server_port = {{ .Values.gateway.config.collector.server.server_port | int64 }} shutdown_timeout = {{ .Values.gateway.config.collector.server.shutdown_timeout | quote }} public_key_path = {{ .Values.gateway.config.collector.server.public_key_path | quote }} private_key_path = {{ .Values.gateway.config.collector.server.private_key_path | quote }}