diff --git a/common/collector/collector.go b/common/collector/collector.go new file mode 100644 index 0000000000..307e57dda9 --- /dev/null +++ b/common/collector/collector.go @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Package collector holds logic shared by the gateway-controller and +// policy-engine's [collector] configuration: the collector is the shared +// data-capture pipeline (system policy + ALS transport) that gathers +// request/response headers and bodies for every consumer (analytics, stdout +// traffic logging). It has no on/off flag of its own — it is implicit, +// derived from whether a consumer is enabled — and both modules migrate the +// same set of deprecated [analytics] aliases onto it at load time. The two +// modules keep their own Config/CollectorConfig struct shapes (their transport +// types differ: GRPCEventServerConfig vs AccessLogsServiceConfig), so this +// package holds only the derivation/migration logic, not the structs. +package collector + +import "log/slog" + +// ServerPort is the fixed TCP port for the Envoy → policy-engine ALS +// (access-log service) gRPC stream in "tcp" mode: the gateway-controller +// dials it (translator.go, Envoy cluster config) and the policy-engine +// listens on it (access_logger_server.go). It is intentionally +// non-configurable — the two sides previously had independent config fields +// that defaulted to the same value but could drift apart, silently breaking +// the connection; a single shared constant makes that impossible. +const ServerPort = 18090 + +// IsEnabled reports whether the collector should run: implicitly active +// whenever any consumer of the collected data (analytics or stdout traffic +// logging) is enabled, and off otherwise. +func IsEnabled(analyticsEnabled, trafficLoggingEnabled bool) bool { + return analyticsEnabled || trafficLoggingEnabled +} + +// CaptureFlags holds the deprecated [analytics] body-capture aliases being +// migrated onto [collector]. +type CaptureFlags struct { + SendRequestBody bool + SendResponseBody bool + AllowPayloads bool +} + +// MigrateDeprecatedCapture maps the deprecated analytics.allow_payloads / +// analytics.send_request_body / analytics.send_response_body onto the +// collector's body-capture flags (when the collector flag is not already +// set), so existing configs keep working after capture settings moved under +// [collector]. These are analytics's own deprecated flags, so they are only +// honored while analyticsEnabled is true — otherwise a stale value left over +// from a disabled analytics setup could silently turn on body capture for an +// unrelated consumer (e.g. traffic_logging) enabled later. Directional +// aliases (send_request_body/send_response_body) take precedence over +// allow_payloads, which only fills in when neither directional flag is +// already set on the collector. +func MigrateDeprecatedCapture(analyticsEnabled bool, deprecated CaptureFlags, collectorSendRequestBody, collectorSendResponseBody *bool) { + if !analyticsEnabled { + return + } + if deprecated.SendRequestBody { + if !*collectorSendRequestBody { + slog.Warn("analytics.send_request_body is deprecated; use collector.request_body instead") + *collectorSendRequestBody = true + } else { + slog.Warn("analytics.send_request_body is deprecated and collector.request_body is already configured; ignoring the analytics.send_request_body override") + } + } + if deprecated.SendResponseBody { + if !*collectorSendResponseBody { + slog.Warn("analytics.send_response_body is deprecated; use collector.response_body instead") + *collectorSendResponseBody = true + } else { + slog.Warn("analytics.send_response_body is deprecated and collector.response_body is already configured; ignoring the analytics.send_response_body override") + } + } + if deprecated.AllowPayloads { + slog.Warn("analytics.allow_payloads is deprecated; use collector.request_body and collector.response_body instead") + if !*collectorSendRequestBody && !*collectorSendResponseBody { + *collectorSendRequestBody = true + *collectorSendResponseBody = true + } + } +} + +// MigrateDeprecatedTransport maps a deprecated [analytics] transport-tuning +// alias (e.g. analytics.grpc_event_server on the controller, +// analytics.access_logs_service on the policy-engine) onto the collector's +// transport config when the collector's copy is still at its default, so +// existing configs keep working after transport tuning moved to +// [collector.server]. It is generic over the two modules' differing transport +// struct shapes (T is only ever compared and assigned, never inspected). This +// is analytics's own deprecated field, so it is only honored while +// analyticsEnabled is true — otherwise a stale value left over from a +// disabled analytics setup could silently reconfigure the transport for an +// unrelated consumer (e.g. traffic_logging) enabled later. +// +// deprecatedKey names the deprecated config key for the warning message +// (e.g. "analytics.grpc_event_server"). +func MigrateDeprecatedTransport[T comparable](analyticsEnabled bool, deprecated T, collectorCfg *T, def T, deprecatedKey string) { + if !analyticsEnabled || deprecated == def { + return + } + if *collectorCfg == def { + slog.Warn(deprecatedKey + " is deprecated; migrating it to collector.server") + *collectorCfg = deprecated + } else { + slog.Warn(deprecatedKey + " is deprecated and collector.server is already configured; ignoring the " + deprecatedKey + " override") + } +} diff --git a/common/collector/collector_test.go b/common/collector/collector_test.go new file mode 100644 index 0000000000..84a03ccc57 --- /dev/null +++ b/common/collector/collector_test.go @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package collector + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsEnabled(t *testing.T) { + assert.False(t, IsEnabled(false, false)) + assert.True(t, IsEnabled(true, false)) + assert.True(t, IsEnabled(false, true)) + assert.True(t, IsEnabled(true, true)) +} + +func TestMigrateDeprecatedCapture_SkippedWhenAnalyticsDisabled(t *testing.T) { + collectorReq, collectorResp := false, false + MigrateDeprecatedCapture(false, CaptureFlags{SendRequestBody: true, SendResponseBody: true, AllowPayloads: true}, &collectorReq, &collectorResp) + assert.False(t, collectorReq) + assert.False(t, collectorResp) +} + +func TestMigrateDeprecatedCapture_DirectionalFlags(t *testing.T) { + collectorReq, collectorResp := false, false + MigrateDeprecatedCapture(true, CaptureFlags{SendRequestBody: true, SendResponseBody: false}, &collectorReq, &collectorResp) + assert.True(t, collectorReq) + assert.False(t, collectorResp) +} + +func TestMigrateDeprecatedCapture_AllowPayloadsFillsInWhenNoDirectionalFlags(t *testing.T) { + collectorReq, collectorResp := false, false + MigrateDeprecatedCapture(true, CaptureFlags{AllowPayloads: true}, &collectorReq, &collectorResp) + assert.True(t, collectorReq) + assert.True(t, collectorResp) +} + +func TestMigrateDeprecatedCapture_DirectionalFlagsWinOverAllowPayloads(t *testing.T) { + collectorReq, collectorResp := false, true + MigrateDeprecatedCapture(true, CaptureFlags{SendRequestBody: true, AllowPayloads: true}, &collectorReq, &collectorResp) + assert.True(t, collectorReq) + assert.True(t, collectorResp, "already true; allow_payloads must not be re-applied since a directional flag is set") +} + +type testTransportConfig struct { + Port int + Mode string +} + +func TestMigrateDeprecatedTransport_SkippedWhenAnalyticsDisabled(t *testing.T) { + def := testTransportConfig{Port: 18090, Mode: "uds"} + collectorCfg := def + deprecated := testTransportConfig{Port: 9999, Mode: "tcp"} + + MigrateDeprecatedTransport(false, deprecated, &collectorCfg, def, "analytics.grpc_event_server") + + assert.Equal(t, def, collectorCfg, "collector transport must stay at default when analytics is disabled") +} + +func TestMigrateDeprecatedTransport_SkippedWhenDeprecatedAtDefault(t *testing.T) { + def := testTransportConfig{Port: 18090, Mode: "uds"} + collectorCfg := def + + MigrateDeprecatedTransport(true, def, &collectorCfg, def, "analytics.grpc_event_server") + + assert.Equal(t, def, collectorCfg) +} + +func TestMigrateDeprecatedTransport_MigratesWhenCollectorAtDefault(t *testing.T) { + def := testTransportConfig{Port: 18090, Mode: "uds"} + collectorCfg := def + deprecated := testTransportConfig{Port: 9999, Mode: "tcp"} + + MigrateDeprecatedTransport(true, deprecated, &collectorCfg, def, "analytics.grpc_event_server") + + assert.Equal(t, deprecated, collectorCfg) +} + +func TestMigrateDeprecatedTransport_DoesNotClobberExplicitCollectorConfig(t *testing.T) { + def := testTransportConfig{Port: 18090, Mode: "uds"} + explicit := testTransportConfig{Port: 7000, Mode: "tcp"} + collectorCfg := explicit + deprecated := testTransportConfig{Port: 9999, Mode: "tcp"} + + MigrateDeprecatedTransport(true, deprecated, &collectorCfg, def, "analytics.grpc_event_server") + + assert.Equal(t, explicit, collectorCfg, "an explicit [collector.server] override must never be clobbered by the deprecated alias") +} diff --git a/gateway/configs/config-template.toml b/gateway/configs/config-template.toml index d11d5ebb66..5ec4c033e6 100644 --- a/gateway/configs/config-template.toml +++ b/gateway/configs/config-template.toml @@ -281,7 +281,64 @@ port = 9010 host = "localhost" # ============================================================================= -# ANALYTICS CONFIGURATION +# COLLECTOR CONFIGURATION +# ============================================================================= +# The collector is the shared data-capture pipeline: it injects the analytics +# system policy and ships request/response headers and bodies to the policy-engine +# over ALS. It has NO on/off switch of its own — it is implicit, turning on +# automatically whenever a consumer below is enabled ([analytics] or +# [traffic_logging]) and staying off otherwise. This section only tunes what the +# collector captures. +[collector] +# Capture request/response payloads (bodies) into the collected event. Bodies are +# captured in full; per-consumer size caps are applied on output (see +# traffic_logging.max_payload_size). +request_body = false +response_body = false +# Capture ALL request/response headers into the collected event (request_headers/ +# response_headers). Applies to every API via the collector system policy, so no +# per-API header policy is needed. Sensitive values are redacted on output by +# consumers that support it (e.g. traffic_logging.masked_headers) +# Off by default; enable deliberately per consumer. +request_headers = false +response_headers = false + +# Path prefixes for which the collector produces no analytics event and no +# traffic-log line at all (as if disabled for that one request) — e.g. health +# checks or metrics scrapes. Enforced by Envoy itself via an access-log filter, +# so the policy-engine never even receives the request. Matched case-sensitively +# against the path the client called (Envoy's x-envoy-original-path, set whenever +# a route rewrite occurred), never against the rewritten backend path. Requests +# without that header (no rewrite occurred) are never suppressed by this filter — +# matching bare :path would risk suppressing traffic whose backend path merely +# happens to share the prefix. Empty by default (no suppression). +ignore_path_prefixes = [] +# ignore_path_prefixes = ["/health", "/metrics"] + +# ALS transport tuning (advanced; defaults are sensible). One section read by BOTH +# ends of the Envoy → policy-engine access-log stream: +# • gateway-controller — configures Envoy's gRPC access-log sink (the sender) +# • policy-engine — runs the receiving gRPC server +# Shared keys (TLS, message/header limits) must match on both ends and live here +# once. Envoy-sender-only keys (buffer_*, grpc_request_timeout) are read only by +# the controller and ignored by the policy-engine. +# NOTE: the TCP port (18090) is not configurable here — it is a fixed constant +# (collector.ServerPort in common/collector) shared by both ends, so they can +# never disagree on it. See ANALYTICS_LOG_PUBLISHER_DESIGN.md changelog. +[collector.server] +mode = "uds" # "uds" (default) or "tcp" +buffer_flush_interval = 1000000000 # Envoy sender only (nanoseconds) +buffer_size_bytes = 16384 # Envoy sender only +grpc_request_timeout = 20000000000 # Envoy sender only (nanoseconds) +shutdown_timeout = "600s" +public_key_path = "" # TLS for the ALS connection (both ends) +private_key_path = "" +als_plain_text = true # plaintext gRPC (skip TLS) +max_message_size = 1000000000 +max_header_limit = 8192 + +# ============================================================================= +# ANALYTICS CONFIGURATION (consumer — enabling it activates the collector) # ============================================================================= [analytics] enabled = false @@ -291,6 +348,9 @@ enabled = false # To disable payloads entirely, set allow_payloads = false or remove it; do not # rely on send_*_body = false while allow_payloads remains true. allow_payloads = false +# Deprecated: send_request_body / send_response_body are preserved +# for backward compatibility and are mapped onto the [collector] body-capture flags +# during validation (with a warning). Prefer setting [collector] directly.\ send_request_body = false send_response_body = false enabled_publishers = ["moesif"] @@ -303,17 +363,32 @@ event_queue_size = 10000 batch_size = 50 timer_wakeup_seconds = 3 -[analytics.grpc_event_server] -buffer_flush_interval = 1000000000 -buffer_size_bytes = 16384 -grpc_request_timeout = 20000000000 -server_port = 18090 -shutdown_timeout = "600s" -public_key_path = "" -private_key_path = "" -als_plain_text = true -max_message_size = 1000000000 -max_header_limit = 8192 +# ============================================================================= +# TRAFFIC LOGGING (consumer — enabling it activates the collector) +# ============================================================================= +# Writes collected events (rich API/application/AI/latency metadata, plus +# request/response headers and — when the collector's send_*_body is enabled — +# payloads) to stdout as a JSON line. Useful for log-scraping pipelines such as +# Fluent Bit/Loki/ELK; no external SaaS needed. +# +# Per-API (opt-in): a line is emitted only for APIs that attach the `log-message` +# policy with `enableTrafficLogging: true` — enabling this section alone does NOT +# log every API. So an API is logged only when both hold: [traffic_logging].enabled +# = true, and the `log-message` policy (enableTrafficLogging: true) is attached to +# that API. The policy's request/response parameters (per-flow payload/headers +# toggles and excludeHeaders) shape each API's line; they filter/mask what the +# collector captured (they cannot enable capture the collector was not configured +# to perform). +[traffic_logging] +enabled = false +# Header names (case-insensitive) whose values are redacted as "****" in the +# logged requestHeaders/responseHeaders. Applies globally; the `log-message` +# policy's per-API excludeHeaders drop additional headers entirely. +masked_headers = ["authorization", "x-api-key", "x-jwt-assertion"] +# Max bytes of request/response payload written to each log line (0 = no limit). +# Applied output-side, so the collector still captures full bodies and other +# consumers are unaffected. +max_payload_size = 0 # ============================================================================= # POLICY CONFIGURATIONS diff --git a/gateway/configs/config.toml b/gateway/configs/config.toml index cd4ea4897f..b51b959e5d 100644 --- a/gateway/configs/config.toml +++ b/gateway/configs/config.toml @@ -1,17 +1,21 @@ +[collector] +request_body = false +response_body = false +request_headers = false +response_headers = false [analytics] enabled = true -# Deprecated: allow_payloads is preserved for backward compatibility. When true -# and when send_request_body/send_response_body are not explicitly set, both -# request and response bodies will be included in analytics events. -# allow_payloads = true -send_request_body = false -send_response_body = false enabled_publishers = ["moesif"] [analytics.publishers.moesif] application_id = "" +[traffic_logging] +enabled = false +masked_headers = ["authorization", "x-api-key", "x-jwt-assertion"] +max_payload_size = 2048 + [router] gateway_host = "*" diff --git a/gateway/gateway-controller/pkg/config/config.go b/gateway/gateway-controller/pkg/config/config.go index 6402427984..62a54ab6dc 100644 --- a/gateway/gateway-controller/pkg/config/config.go +++ b/gateway/gateway-controller/pkg/config/config.go @@ -31,6 +31,7 @@ import ( "github.com/knadh/koanf/providers/env" "github.com/knadh/koanf/providers/file" "github.com/knadh/koanf/v2" + "github.com/wso2/api-platform/common/collector" commonconstants "github.com/wso2/api-platform/common/constants" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/constants" ) @@ -48,7 +49,9 @@ type Config struct { Router RouterConfig `koanf:"router"` PolicyEngine map[string]interface{} `koanf:"policy_engine"` PolicyConfigurations map[string]interface{} `koanf:"policy_configurations"` + Collector CollectorConfig `koanf:"collector"` Analytics AnalyticsConfig `koanf:"analytics"` + TrafficLogging TrafficLoggingConfig `koanf:"traffic_logging"` TracingConfig TracingConfig `koanf:"tracing"` APIKey APIKeyConfig `koanf:"api_key"` // Subscriptions controls application-level subscription behaviour for APIs. @@ -57,24 +60,68 @@ type Config struct { ImmutableGateway ImmutableGatewayConfig `koanf:"immutable_gateway"` } +// CollectorConfig holds the data-collection ("collector") configuration. The +// collector is the shared capture pipeline (the analytics system policy plus the +// Envoy→policy-engine ALS transport) that gathers request/response headers and +// bodies. It underpins every consumer of that data (analytics and traffic logging) +// and is implicitly active whenever a consumer is enabled — see +// Config.IsCollectorEnabled. This section tunes capture and transport; it has no +// on/off flag of its own. +type CollectorConfig struct { + // RequestBody / ResponseBody capture request/response bodies into the + // collected event. + RequestBody bool `koanf:"request_body"` + ResponseBody bool `koanf:"response_body"` + // RequestHeaders / ResponseHeaders, when true, make the collector + // capture ALL request / response headers, so every API's headers flow into + // the collected event without attaching a per-API header policy. + RequestHeaders bool `koanf:"request_headers"` + ResponseHeaders bool `koanf:"response_headers"` + // IgnorePathPrefixes lists request path prefixes for which the collector + // produces no analytics event and no traffic-log line at all, as if the + // collector were disabled for that one request (e.g. health-check or + // metrics-scrape endpoints). Enforced by attaching an Envoy AccessLogFilter + // to the ALS access log (see xds.buildIgnorePathsAccessLogFilter) — Envoy + // itself never emits the log entry for a matching path, so the policy-engine + // never receives the request at all. Matched case-sensitively by prefix. + IgnorePathPrefixes []string `koanf:"ignore_path_prefixes"` + // Server tunes the Envoy→policy-engine gRPC access-log (ALS) + // transport that ships collected data. It is part of the collector and is + // configured under the shared [collector.server] section (the policy-engine reads + // the same section to configure its receiving ALS server). + Server GRPCEventServerConfig `koanf:"server"` +} + // AnalyticsConfig holds analytics configuration type AnalyticsConfig struct { - Enabled bool `koanf:"enabled"` - EnabledPublishers []string `koanf:"enabled_publishers"` - Publishers AnalyticsPublishersConfig `koanf:"publishers"` - GRPCEventServerCfg GRPCEventServerConfig `koanf:"grpc_event_server"` - // AllowPayloads controls whether request and response bodies are captured - // into analytics metadata and forwarded to analytics publishers. - // Deprecated: use SendRequestBody and SendResponseBody instead. - // When true, validateAnalyticsConfig maps both SendRequestBody and SendResponseBody - // to true if both are false. Because bools cannot represent "unset", this also - // applies when both new flags are explicitly false; remove allow_payloads when - // migrating and set the directional flags directly. + Enabled bool `koanf:"enabled"` + EnabledPublishers []string `koanf:"enabled_publishers"` + Publishers AnalyticsPublishersConfig `koanf:"publishers"` + // GRPCEventServerCfg is a deprecated alias. ALS transport tuning moved to + // [collector.server]; when set here it is migrated onto the collector during + // validation (with a warning). Prefer [collector.server]. + GRPCEventServerCfg GRPCEventServerConfig `koanf:"grpc_event_server"` + // AllowPayloads, SendRequestBody and SendResponseBody are deprecated aliases. + // Body/header capture now lives under [collector]. When set, these are mapped + // onto collector.request_body / collector.response_body during + // validation (with a warning) for backward compatibility. Prefer the + // [collector] fields directly. AllowPayloads bool `koanf:"allow_payloads"` SendRequestBody bool `koanf:"send_request_body"` SendResponseBody bool `koanf:"send_response_body"` } +// TrafficLoggingConfig mirrors the policy-engine's stdout traffic-logging consumer. +// The controller only needs to know whether it is enabled, so that the collector +// (system policy + ALS sink) is activated when traffic logging is on even if +// analytics is off. Presentation keys (masked_headers, max_payload_size) are +// policy-engine-only and intentionally not bound here. +type TrafficLoggingConfig struct { + // Enabled turns stdout JSON traffic logging on. Enabling it implicitly activates + // the collector (see Config.IsCollectorEnabled). + Enabled bool `koanf:"enabled"` +} + // SubscriptionsConfig holds configuration for application-level subscriptions. type SubscriptionsConfig struct { // EnableValidation toggles automatic injection of the subscriptionValidation @@ -107,9 +154,14 @@ type MoesifPublisherConfig struct { // GRPCEventServerConfig holds configuration for gRPC event server (combines access log service and ALS server config) type GRPCEventServerConfig struct { - Mode string `koanf:"mode"` // Connection mode: "uds" (default) or "tcp" - Port int `koanf:"port"` // ALS port for Envoy connection (TCP mode only) - ServerPort int `koanf:"server_port"` // gRPC server port for ALS server + Mode string `koanf:"mode"` // Connection mode: "uds" (default) or "tcp" + // Port overrides the fixed Envoy→policy-engine ALS dial port (collector.ServerPort, + // 18090), used only in "tcp" mode. Deprecated: no longer defaulted here or documented + // in config-template.toml/Helm charts, so new deployments have no way to discover or + // set it. Kept solely so a config that already sets it explicitly keeps working; + // leave unset (0) to use the fixed port. Must match the policy-engine's + // collector.server.server_port override, or the two sides will fail to connect. + Port int `koanf:"port"` BufferFlushInterval int `koanf:"buffer_flush_interval"` // Envoy buffer flush interval (nanoseconds) BufferSizeBytes int `koanf:"buffer_size_bytes"` // Envoy buffer size GRPCRequestTimeout int `koanf:"grpc_request_timeout"` // Envoy gRPC timeout (nanoseconds) @@ -690,6 +742,24 @@ func LoadConfig(configPath string) (*Config, error) { return cfg, nil } +// defaultGRPCEventServerConfig returns the default Envoy→policy-engine ALS +// transport tuning. Shared by the collector (canonical) and the deprecated +// [analytics].grpc_event_server alias so a partial alias override migrates cleanly. +func defaultGRPCEventServerConfig() GRPCEventServerConfig { + return GRPCEventServerConfig{ + Mode: "uds", // UDS mode by default + BufferFlushInterval: 1000000000, // 1 second + BufferSizeBytes: 16384, // 16 KiB + GRPCRequestTimeout: 20000000000, // 20 seconds + ShutdownTimeout: 600 * time.Second, + PublicKeyPath: "", + PrivateKeyPath: "", + ALSPlainText: true, + MaxMessageSize: 1000000000, + MaxHeaderLimit: 8192, + } +} + // defaultConfig returns a Config struct with default configuration values func defaultConfig() *Config { return &Config{ @@ -908,23 +978,19 @@ func defaultConfig() *Config { TimerWakeupSeconds: 3, }, }, - GRPCEventServerCfg: GRPCEventServerConfig{ - Mode: "uds", // UDS mode by default - Port: 18090, // Only used in TCP mode - ServerPort: 18090, // ALS server port - BufferFlushInterval: 1000000000, // 1 second - BufferSizeBytes: 16384, // 16 KiB - GRPCRequestTimeout: 20000000000, // 20 seconds - ShutdownTimeout: 600 * time.Second, - PublicKeyPath: "", - PrivateKeyPath: "", - ALSPlainText: true, - MaxMessageSize: 1000000000, - MaxHeaderLimit: 8192, - }, - AllowPayloads: false, - SendRequestBody: false, - SendResponseBody: false, + // Deprecated alias: default mirrors the collector so a partial + // [analytics.grpc_event_server] override migrates cleanly. + GRPCEventServerCfg: defaultGRPCEventServerConfig(), + AllowPayloads: false, + SendRequestBody: false, + SendResponseBody: false, + }, + Collector: CollectorConfig{ + RequestBody: false, + ResponseBody: false, + RequestHeaders: false, + ResponseHeaders: false, + Server: defaultGRPCEventServerConfig(), }, TracingConfig: TracingConfig{ Enabled: false, @@ -1291,7 +1357,7 @@ func (c *Config) Validate() error { return err } - if err := c.validateAnalyticsConfig(); err != nil { + if err := c.validateCollectorConfig(); err != nil { return err } @@ -1697,52 +1763,117 @@ func validateDomains(field string, domains []string) error { return nil } -// validateAnalyticsConfig validates the analytics configuration -func (c *Config) validateAnalyticsConfig() error { - // Validate analytics configuration - if c.Analytics.Enabled { - // Migration path for deprecated analytics.allow_payloads. - // Runs when both directional flags are false, which is indistinguishable - // from "not set" because bool fields cannot represent unset vs explicit false. - if c.Analytics.AllowPayloads { - slog.Warn("analytics.allow_payloads is deprecated; use analytics.send_request_body and analytics.send_response_body instead") - if !c.Analytics.SendRequestBody && !c.Analytics.SendResponseBody { - c.Analytics.SendRequestBody = true - c.Analytics.SendResponseBody = true - } - } +// validateCollectorConfig migrates deprecated analytics aliases onto the collector +// and validates the ALS transport tuning when the collector is active. The collector +// has no on/off flag of its own: it is implicitly active whenever a consumer is +// enabled (analytics or traffic logging) — see IsCollectorEnabled. +func (c *Config) validateCollectorConfig() error { + c.migrateDeprecatedAnalyticsCapture() + c.migrateDeprecatedAnalyticsTransport() + c.Collector.IgnorePathPrefixes = normalizeIgnorePathPrefixes(c.Collector.IgnorePathPrefixes) - // Validate gRPC event server configuration - grpcEventServerCfg := c.Analytics.GRPCEventServerCfg - - // Validate connection mode - switch grpcEventServerCfg.Mode { - case "uds", "": - // UDS mode (default) - port is unused for Envoy connection - case "tcp": - // TCP mode - validate port (host is derived from policy_engine.host) - if grpcEventServerCfg.Port <= 0 || grpcEventServerCfg.Port > 65535 { - return fmt.Errorf("analytics.grpc_event_server.port must be between 1 and 65535 when mode is tcp, got %d", grpcEventServerCfg.Port) - } - default: - return fmt.Errorf("analytics.grpc_event_server.mode must be 'uds' or 'tcp', got: %s", grpcEventServerCfg.Mode) + if c.IsCollectorEnabled() { + if err := validateGRPCEventServerConfig(c.Collector.Server); err != nil { + return err } + } + return nil +} - // Validate buffer and timeout settings - if grpcEventServerCfg.BufferFlushInterval <= 0 || grpcEventServerCfg.BufferSizeBytes <= 0 || grpcEventServerCfg.GRPCRequestTimeout <= 0 { - return fmt.Errorf( - "invalid gRPC event server configuration: bufferFlushInterval=%d, bufferSizeBytes=%d, grpcRequestTimeout=%d (all must be > 0)", - grpcEventServerCfg.BufferFlushInterval, - grpcEventServerCfg.BufferSizeBytes, - grpcEventServerCfg.GRPCRequestTimeout, - ) +// normalizeIgnorePathPrefixes trims whitespace and drops empty entries from the +// configured ignore-path prefixes. An unfiltered empty string would match every +// path via strings.HasPrefix, silently blackholing all collector output, so +// empty entries (including ones that are empty only after trimming) are +// dropped rather than passed through. Returns nil if nothing survives. +func normalizeIgnorePathPrefixes(prefixes []string) []string { + if len(prefixes) == 0 { + return nil + } + out := make([]string, 0, len(prefixes)) + for _, p := range prefixes { + if trimmed := strings.TrimSpace(p); trimmed != "" { + out = append(out, trimmed) } + } + if len(out) == 0 { + return nil + } + return out +} + +// IsCollectorEnabled reports whether the collector should run. The collector is +// implicit: it is active whenever any consumer of the collected data is enabled +// (analytics or stdout traffic logging), and off otherwise. When active, the +// controller injects the analytics system policy and configures Envoy's ALS sink. +func (c *Config) IsCollectorEnabled() bool { + return collector.IsEnabled(c.Analytics.Enabled, c.TrafficLogging.Enabled) +} + +// migrateDeprecatedAnalyticsTransport maps a deprecated [analytics].grpc_event_server +// override onto the collector when the collector's transport tuning is still at its +// default, so existing configs keep working after the transport moved to [collector]. +// See collector.MigrateDeprecatedTransport for the shared (with the policy-engine) +// migration logic and its guarding-while-analytics-enabled rationale. +func (c *Config) migrateDeprecatedAnalyticsTransport() { + collector.MigrateDeprecatedTransport( + c.Analytics.Enabled, + c.Analytics.GRPCEventServerCfg, + &c.Collector.Server, + defaultGRPCEventServerConfig(), + "analytics.grpc_event_server", + ) +} + +// migrateDeprecatedAnalyticsCapture maps the deprecated analytics.allow_payloads / +// analytics.send_request_body / analytics.send_response_body onto the collector's +// body-capture flags, so existing configs keep working after capture settings +// moved under [collector]. See collector.MigrateDeprecatedCapture for the shared +// (with the policy-engine) migration logic and its guarding-while-analytics- +// enabled rationale. +func (c *Config) migrateDeprecatedAnalyticsCapture() { + collector.MigrateDeprecatedCapture( + c.Analytics.Enabled, + collector.CaptureFlags{ + SendRequestBody: c.Analytics.SendRequestBody, + SendResponseBody: c.Analytics.SendResponseBody, + AllowPayloads: c.Analytics.AllowPayloads, + }, + &c.Collector.RequestBody, + &c.Collector.ResponseBody, + ) +} + +// validateGRPCEventServerConfig validates the Envoy→policy-engine ALS transport tuning. +// The transport port is normally the fixed, non-configurable collector.ServerPort +// constant (see collector.ServerPort); cfg.Port is a deprecated override honored only +// for backward compatibility with configs that already set it (see its doc comment). +func validateGRPCEventServerConfig(cfg GRPCEventServerConfig) error { + // Validate connection mode + switch cfg.Mode { + case "uds", "tcp", "": + default: + return fmt.Errorf("collector.server.mode must be 'uds' or 'tcp', got: %s", cfg.Mode) + } - // Validate server port - if grpcEventServerCfg.ServerPort <= 0 || grpcEventServerCfg.ServerPort > 65535 { - return fmt.Errorf("analytics.grpc_event_server.server_port must be between 1 and 65535, got %d", grpcEventServerCfg.ServerPort) + if cfg.Port != 0 { + slog.Warn("collector.server.port is deprecated and no longer documented; the ALS port is fixed at " + + strconv.Itoa(collector.ServerPort) + " by default. Honoring the configured override for backward " + + "compatibility — ensure the policy-engine's collector.server.server_port matches, or the two sides will fail to connect.") + if cfg.Port < 0 || cfg.Port > 65535 { + return fmt.Errorf("collector.server.port must be between 1 and 65535, got %d", cfg.Port) } } + + // Validate buffer and timeout settings + if cfg.BufferFlushInterval <= 0 || cfg.BufferSizeBytes <= 0 || cfg.GRPCRequestTimeout <= 0 { + return fmt.Errorf( + "invalid gRPC event server configuration: bufferFlushInterval=%d, bufferSizeBytes=%d, grpcRequestTimeout=%d (all must be > 0)", + cfg.BufferFlushInterval, + cfg.BufferSizeBytes, + cfg.GRPCRequestTimeout, + ) + } + return nil } diff --git a/gateway/gateway-controller/pkg/config/config_test.go b/gateway/gateway-controller/pkg/config/config_test.go index 91e53733a4..c154d47649 100644 --- a/gateway/gateway-controller/pkg/config/config_test.go +++ b/gateway/gateway-controller/pkg/config/config_test.go @@ -102,6 +102,14 @@ func validConfig() *Config { ServerHeaderTransformation: commonconstants.OVERWRITE, }, }, + // Transport defaults mirror production so collector-gated grpc validation + // passes and the deprecated [analytics] alias stays neutral. + Collector: CollectorConfig{ + Server: defaultGRPCEventServerConfig(), + }, + Analytics: AnalyticsConfig{ + GRPCEventServerCfg: defaultGRPCEventServerConfig(), + }, } } @@ -1205,11 +1213,10 @@ func TestConfig_ValidateAnalyticsConfig(t *testing.T) { name: "Analytics enabled with valid UDS config (default mode)", enabled: true, setupConfig: func(cfg *Config) { - cfg.Analytics.GRPCEventServerCfg.Mode = "uds" - cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 1000 - cfg.Analytics.GRPCEventServerCfg.BufferSizeBytes = 16384 - cfg.Analytics.GRPCEventServerCfg.GRPCRequestTimeout = 5000 - cfg.Analytics.GRPCEventServerCfg.ServerPort = 18090 + cfg.Collector.Server.Mode = "uds" + cfg.Collector.Server.BufferFlushInterval = 1000 + cfg.Collector.Server.BufferSizeBytes = 16384 + cfg.Collector.Server.GRPCRequestTimeout = 5000 }, wantErr: false, }, @@ -1217,12 +1224,10 @@ func TestConfig_ValidateAnalyticsConfig(t *testing.T) { name: "Analytics enabled with valid TCP config", enabled: true, setupConfig: func(cfg *Config) { - cfg.Analytics.GRPCEventServerCfg.Mode = "tcp" - cfg.Analytics.GRPCEventServerCfg.Port = 18090 - cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 1000 - cfg.Analytics.GRPCEventServerCfg.BufferSizeBytes = 16384 - cfg.Analytics.GRPCEventServerCfg.GRPCRequestTimeout = 5000 - cfg.Analytics.GRPCEventServerCfg.ServerPort = 18090 + cfg.Collector.Server.Mode = "tcp" + cfg.Collector.Server.BufferFlushInterval = 1000 + cfg.Collector.Server.BufferSizeBytes = 16384 + cfg.Collector.Server.GRPCRequestTimeout = 5000 }, wantErr: false, }, @@ -1230,11 +1235,10 @@ func TestConfig_ValidateAnalyticsConfig(t *testing.T) { name: "Analytics enabled with empty mode defaults to UDS", enabled: true, setupConfig: func(cfg *Config) { - cfg.Analytics.GRPCEventServerCfg.Mode = "" - cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 1000 - cfg.Analytics.GRPCEventServerCfg.BufferSizeBytes = 16384 - cfg.Analytics.GRPCEventServerCfg.GRPCRequestTimeout = 5000 - cfg.Analytics.GRPCEventServerCfg.ServerPort = 18090 + cfg.Collector.Server.Mode = "" + cfg.Collector.Server.BufferFlushInterval = 1000 + cfg.Collector.Server.BufferSizeBytes = 16384 + cfg.Collector.Server.GRPCRequestTimeout = 5000 }, wantErr: false, }, @@ -1242,58 +1246,58 @@ func TestConfig_ValidateAnalyticsConfig(t *testing.T) { name: "Invalid mode value", enabled: true, setupConfig: func(cfg *Config) { - cfg.Analytics.GRPCEventServerCfg.Mode = "invalid" - cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 1000 - cfg.Analytics.GRPCEventServerCfg.BufferSizeBytes = 16384 - cfg.Analytics.GRPCEventServerCfg.GRPCRequestTimeout = 5000 - cfg.Analytics.GRPCEventServerCfg.ServerPort = 18090 + cfg.Collector.Server.Mode = "invalid" + cfg.Collector.Server.BufferFlushInterval = 1000 + cfg.Collector.Server.BufferSizeBytes = 16384 + cfg.Collector.Server.GRPCRequestTimeout = 5000 }, wantErr: true, - errContains: "grpc_event_server.mode must be 'uds' or 'tcp'", + errContains: "collector.server.mode must be 'uds' or 'tcp'", }, { - name: "TCP mode - invalid port", + name: "Invalid buffer flush interval", enabled: true, setupConfig: func(cfg *Config) { - cfg.Analytics.GRPCEventServerCfg.Mode = "tcp" - cfg.Analytics.GRPCEventServerCfg.Port = 0 - cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 1000 - cfg.Analytics.GRPCEventServerCfg.BufferSizeBytes = 16384 - cfg.Analytics.GRPCEventServerCfg.GRPCRequestTimeout = 5000 - cfg.Analytics.GRPCEventServerCfg.ServerPort = 18090 + cfg.Collector.Server.Mode = "uds" + cfg.Collector.Server.BufferFlushInterval = 0 }, wantErr: true, - errContains: "grpc_event_server.port must be between 1 and 65535", + errContains: "invalid gRPC event server configuration", }, { - name: "Invalid server port", + // Backward compat: an existing config that already sets a custom port + // (the deprecated Port override) must keep working, not error. + name: "Deprecated port override still accepted (backward compat)", enabled: true, setupConfig: func(cfg *Config) { - cfg.Analytics.GRPCEventServerCfg.Mode = "uds" - cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 1000 - cfg.Analytics.GRPCEventServerCfg.BufferSizeBytes = 16384 - cfg.Analytics.GRPCEventServerCfg.GRPCRequestTimeout = 5000 - cfg.Analytics.GRPCEventServerCfg.ServerPort = 0 + cfg.Collector.Server.Mode = "tcp" + cfg.Collector.Server.Port = 9099 + cfg.Collector.Server.BufferFlushInterval = 1000 + cfg.Collector.Server.BufferSizeBytes = 16384 + cfg.Collector.Server.GRPCRequestTimeout = 5000 }, - wantErr: true, - errContains: "grpc_event_server.server_port must be between 1 and 65535", + wantErr: false, }, { - name: "Invalid buffer flush interval", + name: "Deprecated port override out of range still errors", enabled: true, setupConfig: func(cfg *Config) { - cfg.Analytics.GRPCEventServerCfg.Mode = "uds" - cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 0 - cfg.Analytics.GRPCEventServerCfg.ServerPort = 18090 + cfg.Collector.Server.Mode = "tcp" + cfg.Collector.Server.Port = 70000 + cfg.Collector.Server.BufferFlushInterval = 1000 + cfg.Collector.Server.BufferSizeBytes = 16384 + cfg.Collector.Server.GRPCRequestTimeout = 5000 }, wantErr: true, - errContains: "invalid gRPC event server configuration", + errContains: "collector.server.port must be between 1 and 65535", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { cfg := validConfig() + // Analytics is a consumer; enabling it makes the collector implicitly + // active so these tests exercise the collector transport validation. cfg.Analytics.Enabled = tt.enabled if tt.setupConfig != nil { tt.setupConfig(cfg) @@ -1309,14 +1313,38 @@ func TestConfig_ValidateAnalyticsConfig(t *testing.T) { } } +// TestConfig_IsCollectorEnabled covers the implicit collector: it is active iff a +// consumer (analytics or traffic logging) is enabled, and off otherwise. +func TestConfig_IsCollectorEnabled(t *testing.T) { + t.Run("no consumers -> off", func(t *testing.T) { + cfg := validConfig() + assert.False(t, cfg.IsCollectorEnabled()) + require.NoError(t, cfg.Validate()) + }) + + t.Run("analytics on -> collector on", func(t *testing.T) { + cfg := validConfig() + cfg.Analytics.Enabled = true + cfg.Analytics.EnabledPublishers = []string{} + assert.True(t, cfg.IsCollectorEnabled()) + require.NoError(t, cfg.Validate()) + }) + + t.Run("traffic logging on -> collector on", func(t *testing.T) { + cfg := validConfig() + cfg.TrafficLogging.Enabled = true + assert.True(t, cfg.IsCollectorEnabled()) + require.NoError(t, cfg.Validate()) + }) +} + func TestConfig_ValidateAnalyticsPayloadMigration(t *testing.T) { setValidAnalyticsGRPC := func(cfg *Config) { - cfg.Analytics.Enabled = true + cfg.Analytics.Enabled = true // a consumer being on makes the collector implicit cfg.Analytics.GRPCEventServerCfg.Mode = "uds" cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 1000 cfg.Analytics.GRPCEventServerCfg.BufferSizeBytes = 16384 cfg.Analytics.GRPCEventServerCfg.GRPCRequestTimeout = 5000 - cfg.Analytics.GRPCEventServerCfg.ServerPort = 18090 } tests := []struct { @@ -1371,8 +1399,67 @@ func TestConfig_ValidateAnalyticsPayloadMigration(t *testing.T) { err := cfg.Validate() require.NoError(t, err) - assert.Equal(t, tt.wantSendReq, cfg.Analytics.SendRequestBody) - assert.Equal(t, tt.wantSendResp, cfg.Analytics.SendResponseBody) + // Deprecated analytics body aliases now migrate onto the collector. + assert.Equal(t, tt.wantSendReq, cfg.Collector.RequestBody) + assert.Equal(t, tt.wantSendResp, cfg.Collector.ResponseBody) + }) + } +} + +// TestConfig_ValidateAnalyticsPayloadMigration_SkippedWhenAnalyticsDisabled guards +// against a stale analytics.allow_payloads left over from a disabled analytics +// setup silently turning on body capture for an unrelated consumer (traffic +// logging) enabled later. The deprecated capture aliases belong to analytics, so +// they must only be honored while analytics itself is enabled. +func TestConfig_ValidateAnalyticsPayloadMigration_SkippedWhenAnalyticsDisabled(t *testing.T) { + cfg := validConfig() + cfg.Analytics.Enabled = false + cfg.TrafficLogging.Enabled = true // an unrelated consumer activates the collector + cfg.Analytics.AllowPayloads = true + + err := cfg.Validate() + require.NoError(t, err) + assert.False(t, cfg.Collector.RequestBody) + assert.False(t, cfg.Collector.ResponseBody) +} + +// TestConfig_ValidateAnalyticsTransportMigration_SkippedWhenAnalyticsDisabled guards +// against a stale analytics.grpc_event_server override left over from a disabled +// analytics setup silently reconfiguring the transport for an unrelated consumer +// (traffic logging) enabled later. The deprecated transport alias belongs to +// analytics, so it must only be honored while analytics itself is enabled. +func TestConfig_ValidateAnalyticsTransportMigration_SkippedWhenAnalyticsDisabled(t *testing.T) { + cfg := validConfig() + cfg.Analytics.Enabled = false + cfg.TrafficLogging.Enabled = true // an unrelated consumer activates the collector + cfg.Analytics.GRPCEventServerCfg.Mode = "uds" + cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 1000 + cfg.Analytics.GRPCEventServerCfg.BufferSizeBytes = 16384 + cfg.Analytics.GRPCEventServerCfg.GRPCRequestTimeout = 5000 + + err := cfg.Validate() + require.NoError(t, err) + assert.Equal(t, defaultGRPCEventServerConfig(), cfg.Collector.Server) +} + +func TestConfig_ValidateCollectorConfig_IgnorePathPrefixesNormalized(t *testing.T) { + tests := []struct { + name string + input []string + want []string + }{ + {name: "absent -> nil", input: nil, want: nil}, + {name: "empty slice -> nil", input: []string{}, want: nil}, + {name: "all blank -> nil", input: []string{"", " "}, want: nil}, + {name: "trims whitespace and drops empties", input: []string{" /health ", "", "/metrics"}, want: []string{"/health", "/metrics"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := validConfig() + cfg.Collector.IgnorePathPrefixes = tt.input + + require.NoError(t, cfg.Validate()) + assert.Equal(t, tt.want, cfg.Collector.IgnorePathPrefixes) }) } } diff --git a/gateway/gateway-controller/pkg/utils/system_policies.go b/gateway/gateway-controller/pkg/utils/system_policies.go index 8ad1da067f..01038b5eac 100644 --- a/gateway/gateway-controller/pkg/utils/system_policies.go +++ b/gateway/gateway-controller/pkg/utils/system_policies.go @@ -84,14 +84,15 @@ var defaultSystemPolicies = []systemPolicyConfig{ if cfg == nil { return false } - slog.Debug("Analytics state -> ", "state", cfg.Analytics.Enabled) - return cfg.Analytics.Enabled - }, - // Default parameters (can be overridden via additionalProps) - Parameters: map[string]interface{}{ - "send_request_body": false, - "send_response_body": false, + // The analytics system policy is the collector: it is injected whenever + // the collector is active — i.e. whenever any consumer (analytics or + // traffic logging) is enabled. + slog.Debug("Collector state -> ", "state", cfg.IsCollectorEnabled()) + return cfg.IsCollectorEnabled() }, + // No static defaults — all four capture flags are set at injection time + // from cfg.Collector (see InjectSystemPolicies below). + Parameters: nil, ExecutionCondition: nil, }, } @@ -197,10 +198,13 @@ func InjectSystemPolicies(policies []policyenginev1.PolicyInstance, cfg *config. for k, v := range sysPol.Parameters { effectiveDefaults[k] = v } - // For the analytics system policy, propagate the payload flags from runtime config. + // For the analytics (collector) system policy, propagate the payload and + // header capture flags from the collector config. if sysPol.Name == constants.ANALYTICS_SYSTEM_POLICY_NAME { - effectiveDefaults["send_request_body"] = cfg.Analytics.SendRequestBody - effectiveDefaults["send_response_body"] = cfg.Analytics.SendResponseBody + effectiveDefaults["request_body"] = cfg.Collector.RequestBody + effectiveDefaults["response_body"] = cfg.Collector.ResponseBody + effectiveDefaults["request_headers"] = cfg.Collector.RequestHeaders + effectiveDefaults["response_headers"] = cfg.Collector.ResponseHeaders } // Merge parameters efficiently diff --git a/gateway/gateway-controller/pkg/utils/system_policies_test.go b/gateway/gateway-controller/pkg/utils/system_policies_test.go index d0a0f1a61e..747d02bc7e 100644 --- a/gateway/gateway-controller/pkg/utils/system_policies_test.go +++ b/gateway/gateway-controller/pkg/utils/system_policies_test.go @@ -124,12 +124,9 @@ func TestInjectSystemPolicies_NilConfig(t *testing.T) { assert.Equal(t, policies, result) } -func TestInjectSystemPolicies_AnalyticsDisabled(t *testing.T) { - cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ - Enabled: false, - }, - } +func TestInjectSystemPolicies_CollectorDisabled(t *testing.T) { + // No consumer enabled -> collector implicitly off -> no system policy injected. + cfg := &config.Config{} policies := []policyenginev1.PolicyInstance{ {Name: "existing", Version: "v1.0.0"}, } @@ -139,11 +136,10 @@ func TestInjectSystemPolicies_AnalyticsDisabled(t *testing.T) { assert.Equal(t, "existing", result[0].Name) } -func TestInjectSystemPolicies_AnalyticsEnabled(t *testing.T) { +func TestInjectSystemPolicies_CollectorEnabled(t *testing.T) { + // A consumer enabled -> collector implicitly on -> system policy injected. cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ - Enabled: true, - }, + Analytics: config.AnalyticsConfig{Enabled: true}, } policies := []policyenginev1.PolicyInstance{ {Name: "existing", Version: "v1.0.0"}, @@ -159,43 +155,89 @@ func TestInjectSystemPolicies_AnalyticsEnabled(t *testing.T) { assert.Equal(t, "existing", result[1].Name) } -func TestInjectSystemPolicies_AllowPayloadsTrue(t *testing.T) { +func TestInjectSystemPolicies_TrafficLoggingOnlyEnablesCollector(t *testing.T) { + // Traffic logging alone (no analytics) -> collector implicitly on -> system + // policy injected. Guards against a regression that gates injection on + // cfg.Analytics.Enabled alone instead of cfg.IsCollectorEnabled(). cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ - Enabled: true, - SendRequestBody: true, - SendResponseBody: true, + TrafficLogging: config.TrafficLoggingConfig{Enabled: true}, + } + policies := []policyenginev1.PolicyInstance{ + {Name: "existing", Version: "v1.0.0"}, + } + + result := InjectSystemPolicies(policies, cfg, nil) + assert.Len(t, result, 2) + assert.Equal(t, constants.ANALYTICS_SYSTEM_POLICY_NAME, result[0].Name) + assert.Equal(t, constants.ANALYTICS_SYSTEM_POLICY_VERSION, result[0].Version) + assert.True(t, result[0].Enabled) + assert.Equal(t, "existing", result[1].Name) +} + +func TestInjectSystemPolicies_BodyFlagsPropagated(t *testing.T) { + cfg := &config.Config{ + Analytics: config.AnalyticsConfig{Enabled: true}, + Collector: config.CollectorConfig{ + RequestBody: true, + ResponseBody: true, }, } result := InjectSystemPolicies(nil, cfg, nil) assert.Len(t, result, 1) assert.Equal(t, constants.ANALYTICS_SYSTEM_POLICY_NAME, result[0].Name) - assert.Equal(t, true, result[0].Parameters["send_request_body"]) - assert.Equal(t, true, result[0].Parameters["send_response_body"]) + assert.Equal(t, true, result[0].Parameters["request_body"]) + assert.Equal(t, true, result[0].Parameters["response_body"]) } -func TestInjectSystemPolicies_AllowPayloadsFalse(t *testing.T) { +func TestInjectSystemPolicies_BodyFlagsDefaultFalse(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ - Enabled: true, - SendRequestBody: false, - SendResponseBody: false, + Analytics: config.AnalyticsConfig{Enabled: true}, + Collector: config.CollectorConfig{ + RequestBody: false, + ResponseBody: false, }, } result := InjectSystemPolicies(nil, cfg, nil) assert.Len(t, result, 1) - assert.Equal(t, false, result[0].Parameters["send_request_body"]) - assert.Equal(t, false, result[0].Parameters["send_response_body"]) + assert.Equal(t, false, result[0].Parameters["request_body"]) + assert.Equal(t, false, result[0].Parameters["response_body"]) } -func TestInjectSystemPolicies_WithAdditionalProps(t *testing.T) { +func TestInjectSystemPolicies_HeaderFlagsPropagated(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ - Enabled: true, + Analytics: config.AnalyticsConfig{Enabled: true}, + Collector: config.CollectorConfig{ + RequestHeaders: true, + ResponseHeaders: true, }, } + + result := InjectSystemPolicies(nil, cfg, nil) + assert.Len(t, result, 1) + assert.Equal(t, constants.ANALYTICS_SYSTEM_POLICY_NAME, result[0].Name) + assert.Equal(t, true, result[0].Parameters["request_headers"]) + assert.Equal(t, true, result[0].Parameters["response_headers"]) +} + +func TestInjectSystemPolicies_HeaderFlagsDefaultFalse(t *testing.T) { + // Zero-value collector (headers unset) with a consumer on: propagation passes the + // struct's false through, matching the production default (see defaultConfig). + cfg := &config.Config{ + Analytics: config.AnalyticsConfig{Enabled: true}, + } + + result := InjectSystemPolicies(nil, cfg, nil) + assert.Len(t, result, 1) + assert.Equal(t, false, result[0].Parameters["request_headers"]) + assert.Equal(t, false, result[0].Parameters["response_headers"]) +} + +func TestInjectSystemPolicies_WithAdditionalProps(t *testing.T) { + cfg := &config.Config{ + Analytics: config.AnalyticsConfig{Enabled: true}, + } additionalProps := map[string]any{ constants.ANALYTICS_SYSTEM_POLICY_NAME: map[string]interface{}{ "custom_param": "custom_value", @@ -209,9 +251,7 @@ func TestInjectSystemPolicies_WithAdditionalProps(t *testing.T) { func TestInjectSystemPolicies_WithSharedParams(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ - Enabled: true, - }, + Analytics: config.AnalyticsConfig{Enabled: true}, } additionalProps := map[string]any{ SharedParamsKey: map[string]interface{}{ @@ -226,9 +266,7 @@ func TestInjectSystemPolicies_WithSharedParams(t *testing.T) { func TestInjectSystemPolicies_EmptyPolicies(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ - Enabled: true, - }, + Analytics: config.AnalyticsConfig{Enabled: true}, } result := InjectSystemPolicies([]policyenginev1.PolicyInstance{}, cfg, nil) @@ -238,9 +276,7 @@ func TestInjectSystemPolicies_EmptyPolicies(t *testing.T) { func TestInjectSystemPolicies_PreservesExistingPolicies(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ - Enabled: true, - }, + Analytics: config.AnalyticsConfig{Enabled: true}, } policies := []policyenginev1.PolicyInstance{ {Name: "policy1", Version: "v1.0.0"}, diff --git a/gateway/gateway-controller/pkg/xds/translator.go b/gateway/gateway-controller/pkg/xds/translator.go index 408a6dda7a..a6dbf76d4d 100644 --- a/gateway/gateway-controller/pkg/xds/translator.go +++ b/gateway/gateway-controller/pkg/xds/translator.go @@ -31,6 +31,7 @@ import ( "strings" "time" + "github.com/wso2/api-platform/common/collector" commonconstants "github.com/wso2/api-platform/common/constants" accesslog "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v3" @@ -561,10 +562,10 @@ func (t *Translator) TranslateConfigs( policyEngineCluster := t.createPolicyEngineCluster() clusters = append(clusters, policyEngineCluster) - // Add ALS cluster if gRPC access log is enabled - log.Debug("gRPC event server config", slog.Any("config", t.config.Analytics.GRPCEventServerCfg)) - if t.config.Analytics.Enabled { - log.Info("gRPC access log is enabled, creating ALS cluster") + // Add ALS cluster if the collector is active (it ships access logs over gRPC) + log.Debug("gRPC event server config", slog.Any("config", t.config.Collector.Server)) + if t.config.IsCollectorEnabled() { + log.Info("collector is enabled, creating ALS cluster") alsCluster := t.createALSCluster() clusters = append(clusters, alsCluster) } @@ -2156,20 +2157,26 @@ func (t *Translator) createPolicyEngineCluster() *cluster.Cluster { // createALSCluster creates an Envoy cluster for the gRPC access log service func (t *Translator) createALSCluster() *cluster.Cluster { - grpcConfig := t.config.Analytics.GRPCEventServerCfg + grpcConfig := t.config.Collector.Server // Build the endpoint address (UDS or TCP) var address *core.Address if grpcConfig.Mode == "tcp" { - // TCP mode - use host:port + // TCP mode - use host:port. grpcConfig.Port is a deprecated override + // (see its doc comment); the fixed collector.ServerPort constant is used + // unless a config already sets it explicitly. + port := collector.ServerPort + if grpcConfig.Port != 0 { + port = grpcConfig.Port + } address = &core.Address{ Address: &core.Address_SocketAddress{ SocketAddress: &core.SocketAddress{ Protocol: core.SocketAddress_TCP, Address: t.config.Router.PolicyEngine.Host, PortSpecifier: &core.SocketAddress_PortValue{ - PortValue: uint32(grpcConfig.Port), + PortValue: uint32(port), }, }, }, @@ -2785,8 +2792,8 @@ func (t *Translator) createAccessLogConfig() ([]*accesslog.AccessLog, error) { }, }) - // If gRPC access log is enabled, create the configuration and append to existing access logs - if t.config.Analytics.Enabled { + // If the collector is active, create the gRPC access log config and append to existing access logs + if t.config.IsCollectorEnabled() { t.logger.Info("Creating gRPC access log configuration") grpcAccessLog, err := t.createGRPCAccessLog() if err != nil { @@ -2802,8 +2809,8 @@ func (t *Translator) createAccessLogConfig() ([]*accesslog.AccessLog, error) { // createGRPCAccessLog creates a gRPC access log configuration for the gateway controller func (t *Translator) createGRPCAccessLog() (*accesslog.AccessLog, error) { - grpcConfig := t.config.Analytics.GRPCEventServerCfg - bufferSizeBytes, err := checkedUInt32FromPositiveInt("analytics.grpc_event_server.buffer_size_bytes", grpcConfig.BufferSizeBytes) + grpcConfig := t.config.Collector.Server + bufferSizeBytes, err := checkedUInt32FromPositiveInt("collector.server.buffer_size_bytes", grpcConfig.BufferSizeBytes) if err != nil { return nil, err } @@ -2831,13 +2838,102 @@ func (t *Translator) createGRPCAccessLog() (*accesslog.AccessLog, error) { } return &accesslog.AccessLog{ - Name: "envoy.access_loggers.http_grpc", + Name: "envoy.access_loggers.http_grpc", + Filter: buildIgnorePathsAccessLogFilter(t.config.Collector.IgnorePathPrefixes), ConfigType: &accesslog.AccessLog_TypedConfig{ TypedConfig: grpcAccessLogAny, }, }, nil } +// envoyOriginalPathHeader is the header Envoy's router sets to the pre-rewrite, +// client-facing path whenever a route applies a path rewrite (PrefixRewrite/ +// RegexRewrite) — which every proxied API route in this gateway does, for +// context-path stripping. buildIgnorePathsAccessLogFilter matches against this +// header rather than :path so that collector.ignore_path_prefixes is evaluated +// against what the client actually called, not the rewritten backend path. +const envoyOriginalPathHeader = "x-envoy-original-path" + +// buildIgnorePathsAccessLogFilter builds an Envoy AccessLogFilter that suppresses +// the ALS access-log entry entirely for requests whose client-facing path matches +// one of the configured prefixes, so the policy-engine never receives them (no +// analytics event, no traffic-log line). Returns nil (no filter, log everything) +// when no usable prefixes are configured. +func buildIgnorePathsAccessLogFilter(prefixes []string) *accesslog.AccessLogFilter { + filters := make([]*accesslog.AccessLogFilter, 0, len(prefixes)) + for _, prefix := range prefixes { + prefix = strings.TrimSpace(prefix) + if prefix == "" { + continue + } + filters = append(filters, notEffectivelyMatchesPrefix(prefix)) + } + switch len(filters) { + case 0: + return nil + case 1: + return filters[0] + default: + return &accesslog.AccessLogFilter{ + FilterSpecifier: &accesslog.AccessLogFilter_AndFilter{ + AndFilter: &accesslog.AndFilter{Filters: filters}, + }, + } + } +} + +// notEffectivelyMatchesPrefix builds the per-prefix filter: OR(origAbsent, +// origDoesNotMatch) — i.e. suppress the log entry (filter evaluates false) only +// when x-envoy-original-path is present AND matches the prefix; log (filter +// evaluates true) in every other case, including when the header is absent +// entirely. There is deliberately no fallback to matching bare :path: without +// the original-path header we cannot distinguish a route that genuinely never +// rewrites (where :path is the real client path) from an upstream/backend path +// that merely happens to share the prefix, so we always log rather than risk a +// false-positive suppression of legitimate traffic. +func notEffectivelyMatchesPrefix(prefix string) *accesslog.AccessLogFilter { + origAbsent := headerPresentFilter(envoyOriginalPathHeader, false) + origDoesNotMatch := headerPrefixFilter(envoyOriginalPathHeader, prefix, true) + return &accesslog.AccessLogFilter{ + FilterSpecifier: &accesslog.AccessLogFilter_OrFilter{ + OrFilter: &accesslog.OrFilter{ + Filters: []*accesslog.AccessLogFilter{origAbsent, origDoesNotMatch}, + }, + }, + } +} + +// headerPrefixFilter builds an AccessLogFilter matching (or, when invert is +// true, not matching) a header's value against a prefix. +func headerPrefixFilter(headerName, prefix string, invert bool) *accesslog.AccessLogFilter { + return &accesslog.AccessLogFilter{ + FilterSpecifier: &accesslog.AccessLogFilter_HeaderFilter{ + HeaderFilter: &accesslog.HeaderFilter{ + Header: &route.HeaderMatcher{ + Name: headerName, + InvertMatch: invert, + HeaderMatchSpecifier: &route.HeaderMatcher_PrefixMatch{PrefixMatch: prefix}, + }, + }, + }, + } +} + +// headerPresentFilter builds an AccessLogFilter matching whether a header is +// present (present=true) or absent (present=false). +func headerPresentFilter(headerName string, present bool) *accesslog.AccessLogFilter { + return &accesslog.AccessLogFilter{ + FilterSpecifier: &accesslog.AccessLogFilter_HeaderFilter{ + HeaderFilter: &accesslog.HeaderFilter{ + Header: &route.HeaderMatcher{ + Name: headerName, + HeaderMatchSpecifier: &route.HeaderMatcher_PresentMatch{PresentMatch: present}, + }, + }, + }, + } +} + // createTracingConfig creates tracing configuration for HCM if tracing is enabled func (t *Translator) createTracingConfig() (*hcm.HttpConnectionManager_Tracing, error) { // Return nil if tracing is not enabled diff --git a/gateway/gateway-controller/pkg/xds/translator_test.go b/gateway/gateway-controller/pkg/xds/translator_test.go index dc637583e8..13b196aec0 100644 --- a/gateway/gateway-controller/pkg/xds/translator_test.go +++ b/gateway/gateway-controller/pkg/xds/translator_test.go @@ -27,6 +27,7 @@ import ( "testing" "time" + accesslog "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v3" cluster "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" core "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" listener "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" @@ -1330,7 +1331,7 @@ func TestTranslator_CreateALSCluster(t *testing.T) { routerCfg := testRouterConfig() cfg := testConfig() cfg.Analytics.Enabled = true - cfg.Analytics.GRPCEventServerCfg = config.GRPCEventServerConfig{ + cfg.Collector.Server = config.GRPCEventServerConfig{ Mode: "uds", BufferFlushInterval: 1000000000, BufferSizeBytes: 16384, @@ -1357,7 +1358,7 @@ func TestTranslator_CreateALSCluster(t *testing.T) { routerCfg := testRouterConfig() cfg := testConfig() cfg.Analytics.Enabled = true - cfg.Analytics.GRPCEventServerCfg = config.GRPCEventServerConfig{ + cfg.Collector.Server = config.GRPCEventServerConfig{ Mode: "", BufferFlushInterval: 1000000000, BufferSizeBytes: 16384, @@ -1383,9 +1384,8 @@ func TestTranslator_CreateALSCluster(t *testing.T) { routerCfg := testRouterConfig() cfg := testConfig() cfg.Analytics.Enabled = true - cfg.Analytics.GRPCEventServerCfg = config.GRPCEventServerConfig{ + cfg.Collector.Server = config.GRPCEventServerConfig{ Mode: "tcp", - Port: 18090, BufferFlushInterval: 1000000000, BufferSizeBytes: 16384, GRPCRequestTimeout: 20000000000, @@ -1409,37 +1409,77 @@ func TestTranslator_CreateALSCluster(t *testing.T) { assert.Equal(t, "policy-engine", socketAddr.Address) assert.Equal(t, uint32(18090), socketAddr.GetPortValue()) }) + + t.Run("TCP mode honors deprecated port override (backward compat)", func(t *testing.T) { + routerCfg := testRouterConfig() + cfg := testConfig() + cfg.Analytics.Enabled = true + cfg.Collector.Server = config.GRPCEventServerConfig{ + Mode: "tcp", + Port: 9099, + BufferFlushInterval: 1000000000, + BufferSizeBytes: 16384, + GRPCRequestTimeout: 20000000000, + } + cfg.Router.PolicyEngine.Host = "policy-engine" + translator := NewTranslator(logger, routerCfg, nil, cfg) + + c := translator.createALSCluster() + assert.NotNil(t, c) + + lbEndpoint := c.LoadAssignment.Endpoints[0].LbEndpoints[0] + socketAddr := lbEndpoint.GetEndpoint().Address.GetSocketAddress() + assert.NotNil(t, socketAddr) + assert.Equal(t, uint32(9099), socketAddr.GetPortValue()) + }) } func TestTranslator_CreateGRPCAccessLog(t *testing.T) { logger := createTestLogger() routerCfg := testRouterConfig() cfg := testConfig() - cfg.Analytics.GRPCEventServerCfg = config.GRPCEventServerConfig{ + cfg.Collector.Server = config.GRPCEventServerConfig{ + Mode: "tcp", + BufferFlushInterval: 1000, + BufferSizeBytes: 16384, + GRPCRequestTimeout: 5000, + } + translator := NewTranslator(logger, routerCfg, nil, cfg) + + accessLog, err := translator.createGRPCAccessLog() + assert.NoError(t, err) + assert.NotNil(t, accessLog) + assert.Nil(t, accessLog.Filter, "no ignore_path_prefixes configured -> no filter") +} + +func TestTranslator_CreateGRPCAccessLog_WithIgnorePathPrefixes(t *testing.T) { + logger := createTestLogger() + routerCfg := testRouterConfig() + cfg := testConfig() + cfg.Collector.Server = config.GRPCEventServerConfig{ Mode: "tcp", - Port: 18090, BufferFlushInterval: 1000, BufferSizeBytes: 16384, GRPCRequestTimeout: 5000, } + cfg.Collector.IgnorePathPrefixes = []string{"/health"} translator := NewTranslator(logger, routerCfg, nil, cfg) accessLog, err := translator.createGRPCAccessLog() assert.NoError(t, err) assert.NotNil(t, accessLog) + assert.NotNil(t, accessLog.Filter, "ignore_path_prefixes configured -> filter attached") } func TestTranslator_CreateGRPCAccessLog_BufferSizeOverflow(t *testing.T) { logger := createTestLogger() routerCfg := testRouterConfig() cfg := testConfig() - cfg.Analytics.GRPCEventServerCfg = config.GRPCEventServerConfig{ + cfg.Collector.Server = config.GRPCEventServerConfig{ Mode: "tcp", - Port: 18090, BufferFlushInterval: 1000, BufferSizeBytes: math.MaxInt, GRPCRequestTimeout: 5000, - ServerPort: 18090, } translator := NewTranslator(logger, routerCfg, nil, cfg) @@ -1449,6 +1489,142 @@ func TestTranslator_CreateGRPCAccessLog_BufferSizeOverflow(t *testing.T) { assert.Contains(t, err.Error(), "buffer_size_bytes") } +// evalAccessLogFilter walks a constructed AccessLogFilter tree and evaluates it +// against a synthetic header set, mirroring how Envoy itself would evaluate the +// filter. This proves actual matching behavior, not just proto shape. +func evalAccessLogFilter(t *testing.T, filter *accesslog.AccessLogFilter, headers map[string]string) bool { + t.Helper() + switch fs := filter.FilterSpecifier.(type) { + case *accesslog.AccessLogFilter_HeaderFilter: + return evalHeaderMatcher(t, fs.HeaderFilter.Header, headers) + case *accesslog.AccessLogFilter_AndFilter: + for _, f := range fs.AndFilter.Filters { + if !evalAccessLogFilter(t, f, headers) { + return false + } + } + return true + case *accesslog.AccessLogFilter_OrFilter: + for _, f := range fs.OrFilter.Filters { + if evalAccessLogFilter(t, f, headers) { + return true + } + } + return false + default: + t.Fatalf("evalAccessLogFilter: unsupported filter specifier %T", fs) + return false + } +} + +func evalHeaderMatcher(t *testing.T, m *route.HeaderMatcher, headers map[string]string) bool { + t.Helper() + val, present := headers[m.Name] + var result bool + switch spec := m.HeaderMatchSpecifier.(type) { + case *route.HeaderMatcher_PresentMatch: + result = present == spec.PresentMatch + case *route.HeaderMatcher_PrefixMatch: + result = present && strings.HasPrefix(val, spec.PrefixMatch) + default: + t.Fatalf("evalHeaderMatcher: unsupported header match specifier %T", spec) + } + if m.InvertMatch { + result = !result + } + return result +} + +func TestBuildIgnorePathsAccessLogFilter(t *testing.T) { + t.Run("nil prefixes -> nil filter", func(t *testing.T) { + assert.Nil(t, buildIgnorePathsAccessLogFilter(nil)) + }) + + t.Run("empty prefixes -> nil filter", func(t *testing.T) { + assert.Nil(t, buildIgnorePathsAccessLogFilter([]string{})) + }) + + t.Run("whitespace-only entries -> nil filter", func(t *testing.T) { + assert.Nil(t, buildIgnorePathsAccessLogFilter([]string{"", " "})) + }) + + t.Run("single prefix -> unwrapped per-prefix filter", func(t *testing.T) { + filter := buildIgnorePathsAccessLogFilter([]string{"/health"}) + require.NotNil(t, filter) + _, isAnd := filter.FilterSpecifier.(*accesslog.AccessLogFilter_AndFilter) + assert.False(t, isAnd, "single prefix should not be wrapped in an outer AndFilter") + + assert.False(t, evalAccessLogFilter(t, filter, map[string]string{ + "x-envoy-original-path": "/health/live", + }), "matching original path -> suppressed") + assert.True(t, evalAccessLogFilter(t, filter, map[string]string{ + "x-envoy-original-path": "/orders", + }), "non-matching original path -> logged") + assert.True(t, evalAccessLogFilter(t, filter, map[string]string{ + ":path": "/health/live", + }), "no original-path header -> logged regardless of :path") + }) + + t.Run("multiple prefixes -> outer AndFilter", func(t *testing.T) { + filter := buildIgnorePathsAccessLogFilter([]string{"/health", "/metrics", ""}) + require.NotNil(t, filter) + andFilter, isAnd := filter.FilterSpecifier.(*accesslog.AccessLogFilter_AndFilter) + require.True(t, isAnd, "multiple prefixes should be wrapped in an outer AndFilter") + assert.Len(t, andFilter.AndFilter.Filters, 2, "blank entry must be dropped") + + assert.False(t, evalAccessLogFilter(t, filter, map[string]string{ + "x-envoy-original-path": "/health/live", + }), "matches first prefix -> suppressed") + assert.False(t, evalAccessLogFilter(t, filter, map[string]string{ + "x-envoy-original-path": "/metrics/scrape", + }), "matches second prefix -> suppressed") + assert.True(t, evalAccessLogFilter(t, filter, map[string]string{ + "x-envoy-original-path": "/orders", + }), "matches neither prefix -> logged") + }) +} + +func TestNotEffectivelyMatchesPrefix(t *testing.T) { + tests := []struct { + name string + headers map[string]string + wantLog bool + }{ + { + name: "original present and has prefix -> suppress even if :path (rewritten backend path) differs", + headers: map[string]string{envoyOriginalPathHeader: "/health/live", ":path": "/some/rewritten/backend/path"}, + wantLog: false, + }, + { + name: "original present and does not have prefix -> log, original is authoritative", + headers: map[string]string{envoyOriginalPathHeader: "/orders", ":path": "/health"}, + wantLog: true, + }, + { + name: "original absent, :path happens to have prefix -> log anyway, no :path fallback", + headers: map[string]string{":path": "/health/live"}, + wantLog: true, + }, + { + name: "original absent, :path does not have prefix -> log", + headers: map[string]string{":path": "/orders"}, + wantLog: true, + }, + { + name: "no headers at all -> log", + headers: map[string]string{}, + wantLog: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + filter := notEffectivelyMatchesPrefix("/health") + assert.Equal(t, tt.wantLog, evalAccessLogFilter(t, filter, tt.headers)) + }) + } +} + func TestTranslator_CreateDynamicForwardProxyCluster(t *testing.T) { logger := createTestLogger() routerCfg := testRouterConfig() diff --git a/gateway/gateway-runtime/policy-engine/ANALYTICS_LOG_PUBLISHER_DESIGN.md b/gateway/gateway-runtime/policy-engine/ANALYTICS_LOG_PUBLISHER_DESIGN.md new file mode 100644 index 0000000000..7639bb2c5d --- /dev/null +++ b/gateway/gateway-runtime/policy-engine/ANALYTICS_LOG_PUBLISHER_DESIGN.md @@ -0,0 +1,366 @@ +# Design: Collector & Stdout Traffic Logging + +> **Maintenance:** This document describes the shared **collector** (data-capture pipeline) +> and its stdout **traffic-logging** consumer, both built on the analytics pipeline. **Keep +> it in sync with the code — every change to this feature (new config, new capture point, +> behavior change) must update this document in the same PR**, including the Change Log at +> the bottom. + +_Last updated: 2026-07-07_ + +--- + +## 1. Problem we are trying to solve + +Operators running a **self-hosted** API Platform gateway want to observe API traffic — +who called what, status/latency, the request/response **headers**, and optionally the +**bodies** — as structured logs on **stdout** + +Constraints / sub-problems: + +- **Bodies and arbitrary headers cannot come from the Envoy access log.** Envoy's file + access logger has no body operator, and headers must each be named explicitly — there is + no "all headers" or "mask these" capability. So enriching `[router.access_logs]` is not + enough. +- **Stdout logging must not require "enabling analytics."** Data capture and external + analytics (Moesif) used to be a single `analytics.enabled` switch, so an operator who only + wanted stdout logs had to turn on the whole analytics feature. Capture is a *shared* + concern; the consumers (analytics, traffic logging) are independent. +- **Payload size control.** Bodies can be large; operators need a cap. +- **Sensitive data.** Header values such as `Authorization` must be redactable, both + globally and per-API. +- **Per-API opt-in, not all-or-nothing.** Emitting a stdout line for *every* API is too + noisy and rarely wanted. Operators need to log only specific APIs, chosen the same way + other behavior is chosen — by **attaching a policy** to those APIs — while still getting + the access-log-derived fields (latencies, status) that inline policies cannot see. + +## 2. Proposed Solution + +Split the existing analytics pipeline into a shared **collector** (data capture + transport) +and independent **consumers** that read what the collector gathers: + +- **`[collector]`** — the capture pipeline: the auto-injected analytics system policy + (captures headers/bodies) plus the Envoy→policy-engine ALS transport. It has **no on/off + flag of its own**: it is **implicit**, active whenever a consumer is enabled and off + otherwise (see `IsCollectorEnabled`). +- **`[analytics]`** — consumer 1: external analytics publishers (e.g. Moesif). The + collector captures data for **all** publishers uniformly; masking/redaction is each + publisher's own concern — the collector does not filter on a publisher's behalf (see D4 + in prior designs; Moesif receives raw headers regardless of `traffic_logging.masked_headers`). +- **`[traffic_logging]`** — consumer 2 (this feature): serializes a collected event to + stdout as a single JSON line. No external SaaS needed. It is **per-API opt-in**: a line + is emitted only for APIs that attach the **`log-message` policy with + `enableTrafficLogging: true`**. + +Capture settings (`send_*_body`, `send_*_headers`) live under `[collector]` and are shared +by all consumers — capture of each is **off by default**; operators enable what they need. +Output/presentation settings — `masked_headers` (redaction) and `max_payload_size` (payload +byte cap) — live under `[traffic_logging]` and apply only to the stdout publisher; other +consumers (e.g. Moesif) are unaffected. + +### Per-API gating: the `log-message` policy (`enableTrafficLogging`) + +The stdout publisher does **not** log every API. An API's traffic is logged only when a +two-way AND holds: `[traffic_logging].enabled` **and** the **`log-message` policy with +`enableTrafficLogging: true`** is attached to that API/route. (The collector is implicit — +enabling `[traffic_logging]` activates it automatically.) + +`log-message` (in `gateway/dev-policies/log-message`, v1.1+) has two modes. The default +(`enableTrafficLogging: false`, or absent) logs in real time during mediation via `slog` +(including per-chunk streaming) with no dependency on the collector. Setting +`enableTrafficLogging: true` turns the policy into a lightweight **signal**: it emits no log +line itself — latencies only exist at access-log time, beyond an inline policy's reach. +Instead, in `OnRequestHeaders` it stamps a marker (`AnalyticsMetadata["traffic_log"] = +`) carrying its per-API presentation config, and `Mode()` requests only the +request-header phase (no body buffering). The kernel merges that into `analytics_data`, +Envoy carries it onto the `HTTPAccessLogEntry`, and `prepareAnalyticEvent` reads it back +into `event.TrafficLog`. `Log.Publish` emits **only** when `event.TrafficLog != nil`, and +uses the directive to shape the line. Because capture happens globally in the collector +*before* any user policy runs, the directive can **filter/mask** what was captured but +cannot enable capture the collector skipped. + +### Policy parameters (`log-message` v1.1, `enableTrafficLogging`) + +Attach the policy to an API's definition YAML with `enableTrafficLogging: true` to opt that +API into stdout traffic logging. Both `request` and `response` are optional but at least +one, or `fields`, should be set for the line to carry anything beyond the base event fields. + +```yaml +policies: + request: + - policyName: log-message + policyVersion: v1.1.0 + parameters: + enableTrafficLogging: true # required to activate stdout traffic logging + request: + payload: true # include request body in the log line + headers: true # include request headers + response: + payload: false + headers: true + # Optional: additional header names (case-insensitive) redacted for this API, + # merged with the global traffic_logging.masked_headers. + maskedHeaders: + - x-internal-token + # Optional: extra key/value pairs added under the top-level "properties" object. + # "$ctx:" values are resolved from the request context at request time; other + # values are literal. + properties: + subject: "$ctx:auth.subject" + authType: "$ctx:auth.type" + tenant: "$ctx:auth.property.tenant" + env: prod + # Optional: fine-grained field projection over the emitted JSON line. + # When set, fields is authoritative — request/response payload & header booleans + # are ignored (maskedHeaders still applies). + fields: + exclude: + - requestBody +``` + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `enableTrafficLogging` | bool | `false` | **Must be `true`** to opt in. When `false` (default), the policy logs inline in real time via `slog` during mediation — independent of the collector. | +| `request.payload` | bool | `false` | Include the request body. No-op if `[collector].send_request_body = false`. Ignored when `fields` is set. | +| `request.headers` | bool | `false` | Include request headers. No-op if `[collector].send_request_headers = false`. Ignored when `fields` is set. | +| `response.payload` | bool | `false` | Include the response body. No-op if `[collector].send_response_body = false`. Ignored when `fields` is set. | +| `response.headers` | bool | `false` | Include response headers. No-op if `[collector].send_response_headers = false`. Ignored when `fields` is set. | +| `maskedHeaders` | `[]string` | `[]` | Additional header names (case-insensitive) redacted to `****` in `requestHeaders`/`responseHeaders`, merged with the global `traffic_logging.masked_headers`. Always applied, even when `fields` is set. To drop a header entirely instead of redacting it, use a dotted `fields.exclude` path (e.g. `requestHeaders.X-Secret`). | +| `properties` | `object` | `{}` | Extra key→value pairs emitted under a top-level `properties` object. String values prefixed `$ctx:` are resolved from the request context at request time (see below); non-`$ctx` strings are literals; non-string values pass through as-is. Unresolvable `$ctx:` refs are skipped. | +| `fields.only` | `[]string` | `[]` | Keep only the listed keys (top-level, e.g. `latencies`, `requestHeaders`, `properties`, or dotted sub-paths, e.g. `requestHeaders.authorization`, `properties.env`). Authoritative over `request`/`response` payload & header booleans. If both `only` and `exclude` are set, `only` wins. | +| `fields.exclude` | `[]string` | `[]` | Drop the listed keys/dotted sub-paths and keep everything else. | + +**`properties` `$ctx:` references** (resolved in the policy's `OnRequestHeaders`, so only request-phase context is reachable; fixed names are case-insensitive, `auth.property.` is case-sensitive): + +| Reference | Resolves to | +|---|---| +| `$ctx:request.path` / `.method` / `.authority` / `.scheme` / `.vhost` | request line fields | +| `$ctx:request.id` | correlation request ID (`RequestHeaderContext.RequestID`) | +| `$ctx:request.header.` | first value of that request header (skipped if absent) | +| `$ctx:api.id` / `.name` / `.version` / `.context` / `.kind` / `.operation_path` | API metadata | +| `$ctx:project.id` | project the API belongs to | +| `$ctx:auth.subject` / `.type` / `.issuer` / `.credential_id` / `.token_id` | `AuthContext` string fields (`token_id` = jwt `jti`, skipped when empty) | +| `$ctx:auth.authenticated` / `.authorized` | `"true"` / `"false"` | +| `$ctx:auth.audience` | `AuthContext.Audience` joined by `,` | +| `$ctx:auth.scopes` | granted scopes, space-joined, sorted | +| `$ctx:auth.property.` | `AuthContext.Properties[""]` (custom claims) | + +> `auth.*` references require an authentication policy to have run **before** `log-message` in the request-header phase; when `AuthContext` is nil they resolve to nothing and the property is skipped. `properties` is only meaningful with `enableTrafficLogging: true` (ignored in inline mode). + +> **Precedence:** `fields` (when present) overrides `request.payload`, `request.headers`, +> `response.payload`, and `response.headers`. `maskedHeaders` (per-API, merged with the +> global `traffic_logging.masked_headers`) always applies regardless. +> +> **Capture limitation:** the directive can only filter/mask what the collector already +> captured. `request.payload: true` has no effect if `[collector].send_request_body = false`. + +### Data flow (gating in **bold**) + +``` +client → Envoy (ALS cluster + gRPC access log attached when COLLECTOR enabled) + │ ext_proc → policy-engine system policies + │ • collector system policy (injected when COLLECTOR enabled) captures + │ headers/body into dynamic metadata "analytics_data" + ▼ + Envoy gRPC ALS ── HTTPAccessLogEntry ──▶ policy-engine ALS server + (started when COLLECTOR enabled) + │ + ▼ + Analytics.Process(entry) + • prepareAnalyticEvent → dto.Event + • for each registered publisher: Publish(event) + │ + ┌───────────────────────┴───────────────────────┐ + ▼ ▼ + Moesif publisher (ANALYTICS enabled) Log publisher (TRAFFIC_LOGGING enabled) + (logs every event, raw headers) **if event.TrafficLog == nil → skip** + (per-API: set only when log-message + with enableTrafficLogging is attached) + → shape per directive (filter/mask, + truncate to max_payload_size) + → JSON line (incl. latencies) → stdout +``` + +The `log-message` policy (with `enableTrafficLogging: true`) runs on the ext_proc side +(above, "collector system policy" step): it stamps `analytics_data["traffic_log"]` +alongside the captured headers/bodies, so by the time the entry reaches +`prepareAnalyticEvent` the opt-in marker is present. + +## 3. Test scenarios + +| Area | Scenario | Where | +|---|---|---| +| Log publisher | Per-API gate: an event **without** `TrafficLog` is not logged | `publishers/log_test.go: TestLog_Publish_SkipsWhenNoDirective` | +| Log publisher | Emits one JSON line incl. ALS-derived `latencies`; event fields present | `TestLog_Publish_WritesJSONLineWithLatencies` | +| Log publisher | `headers:false`/nil flow omits the property; `payload:false` omits the payload | `TestLog_Publish_DisabledFieldsOmitted` | +| Directive extraction | `prepareAnalyticEvent` reads `traffic_log` marker into `event.TrafficLog` (nil when absent; opts in with defaults when malformed); never leaks into `Properties` | `analytics_test.go: TestPrepareAnalyticEvent_TrafficLogMarker`, `_NoTrafficLogMarker`, `_MalformedTrafficLogMarker` | +| log-message (traffic logging) | `enableTrafficLogging` parsing, `Mode()` (request-header only in that mode), marker JSON from params (incl. `fields`, `properties`, `maskedHeaders`), no inline log, safe degradation | `gateway/dev-policies/log-message/logmessage_test.go` | +| `properties` `$ctx:` | `resolveContextValue` resolves request/api/auth refs (incl. `auth.property.`, joined audience/scopes); literals pass through; unresolvable/nil-auth refs skipped; non-string passthrough; empty result omits the marker field | policy `logmessage_test.go: TestResolveContextValue`, `_NilAuthSkipped`, marker-build tests | +| Properties emission | resolved properties appear under a top-level `properties` object; absent when the directive has none; projectable via `fields` (`properties`/`properties.`) | `publishers/log_test.go: TestLog_Publish_PropertiesTopLevel`, `_NoPropertiesWhenAbsent`, `_PropertiesProjectableViaFields` | +| Field projection | `fields.only` keeps only named keys/dotted sub-paths; `fields.exclude` drops them; authoritative over presence (flow booleans ignored) while `maskedHeaders` still applies | `publishers/log_test.go: TestLog_Publish_FieldsInclude`, `_FieldsExclude`, `_FieldsIncludeRequestBodyAndProperties` | +| Log publisher | `masked_headers` (global) redacts values to `****`, case-insensitive, for request & response headers | `TestLog_Publish_MasksHeaders` | +| Log publisher | Per-API `maskedHeaders` merges with the global list (with and without a global list configured) | `TestLog_Publish_PerAPIMaskedHeadersMergedWithGlobal`, `_PerAPIMaskedHeadersNoGlobal` | +| Log publisher | A dotted `fields.exclude` path (e.g. `requestHeaders.X-Secret`) drops a header entirely, vs. masking which redacts it | `TestLog_Publish_ExcludeHeadersDrops` | +| Log publisher | Masking does **not** mutate the shared event seen by other publishers | `TestLog_Publish_DoesNotMutateSharedEvent` | +| Log publisher | `nil` event is a no-op (no panic, no output) | `TestLog_Publish_NilEvent` | +| Log publisher | A header value that is not valid JSON is passed through unchanged / dropped | `TestLog_Publish_UnparseableHeadersDropped` | +| Log publisher | `NewLog(nil)` returns a usable publisher with no masked headers | `TestNewLog_NilConfig` | +| Publisher wiring | `traffic_logging.enabled` registers the stdout publisher | `analytics_test.go: TestNewAnalytics_TrafficLoggingEnabled` | +| Collector (implicit) | `IsCollectorEnabled()` is false with no consumer, true when analytics or traffic logging is on; config validates in each case | engine `config_test.go: TestIsCollectorEnabled`; controller `config_test.go: TestConfig_IsCollectorEnabled` | +| Deprecated aliases | `analytics.allow_payloads`/`send_*_body` migrate onto `collector.send_*_body`; directional flags win; skipped when analytics is disabled | `TestValidate_AnalyticsPayloadMigration(_SkippedWhenAnalyticsDisabled)` (engine), `TestConfig_ValidateAnalyticsPayloadMigration(_SkippedWhenAnalyticsDisabled)` (controller) | +| Deprecated transport aliases | `analytics.access_logs_service`/`grpc_event_server` migrate onto `collector.als` only while analytics is enabled, so a traffic-logging-only deployment never re-applies a stale analytics override | `TestValidate_AnalyticsTransportMigration_SkippedWhenAnalyticsDisabled` (engine), `TestConfig_ValidateAnalyticsTransportMigration_SkippedWhenAnalyticsDisabled` (controller) | +| Max payload (publisher) | Log publisher truncates `request_payload`/`response_payload` to `traffic_logging.max_payload_size` (0 = no limit) | `publishers/log_test.go: TestLog_Publish_TruncatesPayload`, `_NoTruncationWhenZero`; validation: `config_test.go: TestValidate_TrafficLoggingMaxPayloadSize` | +| Collector gating | system policy injected only when a consumer is enabled (`IsCollectorEnabled`); capture flags (body and header, both off by default) propagate from `[collector]` | `gateway-controller .../system_policies_test.go: TestInjectSystemPolicies_Collector*`, `_BodyFlagsPropagated`, `_BodyFlagsDefaultFalse`, `_HeaderFlagsPropagated`, `_HeaderFlagsDefaultFalse`, `_TrafficLoggingOnlyEnablesCollector` | +| Header capture | `getHeaderFlags` parses bool/string forms; `serializeHeaders` emits a JSON object (lower-cased keys, multi-value joined) and "" for empty | `system-policies/analytics/analytics_headers_test.go` | + +**Manual / end-to-end (not yet automated):** with `traffic_logging.enabled = true`, +`analytics.enabled = false` (the collector activates implicitly), and +`collector.send_request_body`/`send_response_body`/`send_request_headers`/`send_response_headers` +turned on, deploy two APIs and attach the **`log-message`** policy with +**`enableTrafficLogging: true`** to only one. A request to the API with the policy produces +one JSON line on the **policy-engine** stdout containing the latencies, headers (auth +masked), and body (≤ cap), with **no** Moesif activity; a request to the API **without** the +policy produces **no** line; disabling `traffic_logging.enabled` (with analytics also off) +produces no lines even with the policy attached — the collector goes inactive. + +## 4. Implementation + +| Component | File | Responsibility | +|---|---|---| +| Log publisher | `internal/analytics/publishers/log.go` | `Log` implements `Publisher.Publish`; **per-API gate** (`event.TrafficLog == nil → skip`); `toTrafficLogEvent` (in `traffic_log_event.go`) shapes the per-flow headers/payload per the directive, merges per-API `MaskedHeaders` with the global config list, and truncates to `traffic_logging.max_payload_size` (0 = no limit) on a locally-built `TrafficLogEvent`, never mutating the shared `dto.Event`/`Properties`. When `Fields` is set, `applyFieldsProjection` shallow-decodes the marshaled line into `map[string]json.RawMessage` and projects to (`Only`) or without (`Exclude`) the named top-level keys / dotted sub-paths — decoding and re-encoding only the touched nested objects, leaving untouched top-level fields as raw bytes. `sync.Mutex` serializes writes; `NewLog(*config.TrafficLoggingConfig)` | +| Publisher selection | `internal/analytics/analytics.go` | `NewAnalytics` registers Moesif when `analytics.enabled` and the stdout `Log` publisher when `traffic_logging.enabled` (independently) | +| Event enrichment | `internal/analytics/analytics.go` | `prepareAnalyticEvent` attaches `requestHeaders`/`responseHeaders` and `request_payload`/`response_payload`; reads the `traffic_log` marker and parses it per-request via `parseTrafficLogDirective` (no caching — the marker's `Properties` are already request-specific, so caching by raw JSON would leak one API's properties into another request that happens to share the same static portion) into `event.TrafficLog` (gating state, kept off `Properties`) | +| log-message policy (`enableTrafficLogging`) | `gateway/dev-policies/log-message/logmessage.go` (v1.1+, in `build-manifest.yaml`/`build.yaml`) | `enableTrafficLogging: true` turns the policy into a per-API opt-in signal: `Mode()` = request-header phase only; `OnRequestHeaders` → `stampTrafficLogMarker` stamps `AnalyticsMetadata["traffic_log"] = ` (request/response `payload`/`headers`, `fields`, `properties`, `maskedHeaders`). `buildProperties`/`resolveContextValue` expand `$ctx:` refs in `properties` at request time and bake the resolved values into the marker. `false` (default) keeps the original real-time slog logging. Registered via build-manifest | +| Traffic-log directive | `internal/analytics/dto/event.go` | `Event.TrafficLog *TrafficLogDirective` (`json:"-"`); `TrafficLogDirective{Request,Response *TrafficLogFlow; Fields *TrafficLogFields; Properties map[string]interface{}; MaskedHeaders []string}`, `TrafficLogFlow{Payload,Headers bool}`, `TrafficLogFields{Only,Exclude []string}` | +| Engine ALS startup | `cmd/policy-engine/main.go` | ALS gRPC server started when `IsCollectorEnabled()` (a consumer is on) | +| Policy-engine config | `internal/config/config.go` | `CollectorConfig{SendRequestBody, SendResponseBody, AccessLogsServiceCfg}` (no `Enabled` — implicit; no header flags — header capture is a controller-side/capture-time concern); `IsCollectorEnabled() = Analytics.Enabled \|\| TrafficLogging.Enabled`; `TrafficLoggingConfig{Enabled, MaskedHeaders, MaxPayloadSize}`; `validateCollectorConfig` (deprecated-alias migration for body flags **and** `access_logs_service`, the latter only while `Analytics.Enabled`; validates the ALS receiver when the collector is active); `traffic_logging.max_payload_size >= 0` validated in `Validate` | +| Controller config | `gateway-controller/pkg/config/config.go` | `CollectorConfig{SendRequest/ResponseBody, SendRequest/ResponseHeaders, GRPCEventServerCfg}` (no `Enabled`; body and header capture both default **false** — off until an operator opts in); `TrafficLoggingConfig{Enabled}` (so the controller knows traffic logging activates the collector); `IsCollectorEnabled() = Analytics.Enabled \|\| TrafficLogging.Enabled`; `validateCollectorConfig` (alias migration for body flags **and** `grpc_event_server`, the latter only while `Analytics.Enabled`; validates the ALS sink via `validateGRPCEventServerConfig` when the collector is active) | +| Controller gating | `gateway-controller/pkg/xds/translator.go` | ALS cluster + gRPC access-log config attached when `IsCollectorEnabled()`, built from `Collector.GRPCEventServerCfg` | +| Engine ALS receiver | `internal/utils/access_logger_server.go` | Reads `Collector.AccessLogsServiceCfg` (mode, port, TLS, message/header limits) to start the ALS gRPC server | +| System-policy wiring | `gateway-controller/pkg/utils/system_policies.go` | The analytics (collector) system policy is injected when `IsCollectorEnabled()`; propagates `send_*_body`, `send_*_headers` from `[collector]` into policy params | +| Analytics system policy | `gateway/system-policies/analytics/analytics.go` | Captures headers (`serializeHeaders`, gated by `getHeaderFlags`, off by default) into `request_headers`/`response_headers` and request/response bodies (gated by the body flags, off by default) at all capture points (request, buffered response, streaming chunks). No size cap at capture — payload truncation is output-side in the Log publisher | +| Config docs | `gateway/configs/config-template.toml`, `config.toml` | Document/example `[collector]`, `[analytics]`, `[traffic_logging]` | + +### Configuration surface + +```toml +[collector] # implicit — shared capture pipeline + transport (no enabled flag) +send_request_body = false # capture bodies in full (controller→capture AND engine→attach); off by default +send_response_body = false +send_request_headers = false # capture ALL headers (capture side, controller only); off by default +send_response_headers = false + +# ALS transport tuning (advanced; sensible defaults). One shared section read by +# BOTH ends: the controller (Envoy's gRPC ALS sink — buffers, ports, TLS) and the +# policy-engine (the receiving ALS server — mode, port, limits). Envoy-sender-only +# keys (buffer_*, grpc_request_timeout) are ignored by the engine. +[collector.als] + +[analytics] # consumer 1 — enabling it activates the collector +enabled = false +enabled_publishers = ["moesif"] + +[traffic_logging] # consumer 2 — enabling it activates the collector +enabled = true +masked_headers = ["authorization"] # output-side redaction (log publisher only) +max_payload_size = 2048 # 0 = no limit; truncates the stdout line only +``` + +**Where each key is read (two-sided config note):** + +- `collector.send_*_body` — **both** sides: controller config drives the system policy to + *capture* the (full) payload; policy-engine config drives `prepareAnalyticEvent` to *attach* it. +- `collector.send_*_headers` — **capture side only** (controller config → system policy param). +- `traffic_logging.max_payload_size` — **policy-engine** config, applied output-side by the + Log publisher (per-consumer; other consumers unaffected). +- `collector.als` — **both** sides (one shared section): the controller reads it to tune + Envoy's gRPC ALS sink (validated in the controller's `validateCollectorConfig`), and the + policy-engine reads it to configure/validate its receiving ALS server. Shared keys + (`server_port`, TLS, `max_message_size`/`max_header_limit`) must agree on both ends; + Envoy-sender-only keys (`buffer_*`, `grpc_request_timeout`) are ignored by the engine. +- collector activation — **derived on both sides** from the consumer flags + (`IsCollectorEnabled() = analytics.enabled || traffic_logging.enabled`): the controller + injects the system policy + ALS sink and the engine starts the ALS server when it is true. + There is no `collector.enabled` key. +- `traffic_logging.enabled` / `masked_headers` — **policy-engine** config, consumed by the + log publisher. Note `enabled` only *arms* the publisher; whether a given API is logged is + decided per-API by the marker (`event.TrafficLog`) from the attached `log-message` policy + with `enableTrafficLogging: true`. +- `log-message` **policy with `enableTrafficLogging: true`** (per-API attachment) — + data-plane opt-in: its `OnRequestHeaders` stamps the `analytics_data["traffic_log"]` + marker read back in `prepareAnalyticEvent`. + +## 4a. Configuration changes & backward compatibility + +Relative to the previous release, this PR **adds** the `[collector]`, `[collector.als]`, and +`[traffic_logging]` sections and turns `[analytics]` into a consumer of an **implicit** +collector (the collector has no `enabled` flag — it activates whenever a consumer is on). +The only pre-existing (released) `[analytics]` keys affected are the payload-capture flags +and the ALS transport tuning, which are **deprecated and migrated** onto `[collector]`. + +**Which side reads each section:** `[collector]` capture flags split by side — the +controller owns header capture (`send_*_headers`), the engine owns body-attach +(`send_*_body`); `[collector.als]` is read by **both**; collector *activation* is derived on +both sides from the consumer flags; `[traffic_logging]` (including its output caps) is +**engine-only**, except `[traffic_logging].enabled`, which the controller also reads so it +can activate the collector when only traffic logging is on. + +### Added + +| Key | Read by | Purpose | +|---|---|---| +| `[collector].send_request_body` / `send_response_body` | both | Capture (controller) + attach (engine) request/response bodies (in full). Default `false`. | +| `[collector].send_request_headers` / `send_response_headers` | controller | Capture all request/response headers. Default `false`. | +| `[collector.als].*` | both | ALS transport tuning (one shared section — see §4) | +| `[traffic_logging].enabled` | both | Arm the stdout traffic-logging publisher (engine); activate the collector (controller) | +| `[traffic_logging].masked_headers` | engine | Output-side header value redaction | +| `[traffic_logging].max_payload_size` | engine | Truncate the logged payload (`0` = no limit); output-side, publisher-only | + +### Deprecated & migrated (from `[analytics]`) + +These released `[analytics]` keys still work: they are migrated onto `[collector]` at load +time with a `slog.Warn`, so existing configs keep functioning unchanged. + +| Deprecated key | Migrated onto | Notes | +|---|---|---| +| `analytics.grpc_event_server.*` (controller) | `collector.als.*` | Whole-struct migration; only applied when `[collector.als]` is still at its default **and** `analytics.enabled = true`, so an explicit `[collector.als]` is never clobbered, and a traffic-logging-only deployment never re-applies a stale analytics override | +| `analytics.access_logs_service.*` (engine) | `collector.als.*` | Same whole-struct migration, same `analytics.enabled` guard | +| `analytics.allow_payloads` | `collector.send_request_body` + `collector.send_response_body` | Directional flags win over `allow_payloads` | +| `analytics.send_request_body` / `send_response_body` | `collector.send_*_body` | — | + +### Backward-compatibility verdict + +- **Deprecated `[analytics]` keys — backward compatible.** The keys above are migrated onto + `[collector]` at load with a deprecation warning; nothing an operator set previously stops + working. +- **Implicit collector — upgrades are zero-touch, and there is no `collector.enabled` key.** + A previously-valid config that sets `analytics.enabled = true` keeps working: the collector + activates automatically because a consumer is on (`IsCollectorEnabled`). A stale + `collector.enabled = true` left in an old config is harmless — the key is no longer bound, + so koanf ignores it. Nothing an operator must edit on upgrade. + +## 5. Open items / future considerations + +- Automated end-to-end test (containerized run) is not yet in place; covered manually. +- No `max_headers` / allow-list cap on header capture (all captured headers are attached); + add if header cardinality becomes a concern. +- Cross-publisher masking is out of scope: masking/redaction is a per-publisher presentation + concern, not a collector guarantee. The collector hands every publisher the same raw + captured data; if Moesif is enabled alongside traffic logging, it receives unmasked + headers regardless of `traffic_logging.masked_headers` or a policy's `maskedHeaders`. If + cross-publisher masking is ever needed, it would have to move to capture time. + +## Change Log + +| Date | Change | +|---|---| +| 2026-07-02 | Initial version — PR "Add Access Logging support". Introduced the shared **`[collector]`** capture pipeline (request/response header & body capture, plus ALS transport tuning under **`[collector.als]`**) as a prerequisite for its consumers. `[analytics]` (Moesif) became an explicit consumer; its released capture/transport keys (`allow_payloads`, `send_*_body`, `grpc_event_server`/`access_logs_service`) are deprecated and migrated onto `[collector]` with warnings (§4a). Added **`[traffic_logging]`**, a stdout JSON consumer that is **per-API**: the `log-message` policy stamps a `traffic_log` marker that gates and shapes the emitted line — including a `fields` include/exclude projection and header masking — enriched with ALS-derived latencies. | +| 2026-07-03 | Simplified the collector config. **Removed `collector.enabled`** — the collector is now **implicit**, derived from whether a consumer is enabled (`IsCollectorEnabled() = analytics.enabled \|\| traffic_logging.enabled`); the controller gained a minimal `[traffic_logging].enabled` binding so it activates the collector when only traffic logging is on. **Renamed the `log-message` activation param** from `destination: inline\|access-log` to the boolean **`enableTrafficLogging`** (default `false`; `true` ≡ the old `access-log`). Per-API stdout logging is now a two-way AND (publisher + policy). | +| 2026-07-03 | Added **`customProperties`** to `log-message` (traffic-logging mode): user-defined key→value pairs, later renamed to **`labels`** (see 2026-07-07), resolved from `$ctx:`-prefixed request-context references and emitted on the log line. Resolved values are baked into the `traffic_log` marker. | +| 2026-07-03 | Expanded the request-context `$ctx:` surface with five more request-phase scalars: `request.vhost`, `request.id`, `api.kind`, `api.operation_path`, and `project.id`. The inter-policy `Metadata` map and all response-phase fields remain intentionally unexposed (the latter is a hard phase limit — the marker is stamped at request-header time). | +| 2026-07-03 | Audited the `$ctx:` surface against the latest SDK (`sdk/core` v0.2.16). Bumped `log-message` from v0.2.4 → v0.2.16 and exposed `$ctx:auth.token_id` (jwt `jti`, added in v0.2.15). | +| 2026-07-07 | Header capture flags (`collector.send_request_headers` / `send_response_headers`) now default to **`false`**, matching body capture — nothing is captured until an operator opts in. Renamed `customProperties` → **`labels`** and dropped the separate per-flow `excludeHeaders` param in favor of a single per-API **`maskedHeaders`** list (merged with the global `traffic_logging.masked_headers`); dropping a header entirely (rather than redacting it) is done via a dotted `fields.exclude` path. `TrafficLogDirective` gained `MaskedHeaders`. | +| 2026-07-07 | Removed the per-request `directiveCache` from `analytics.go`: the marker's resolved `Labels` are request-specific, so caching the parsed `TrafficLogDirective` by raw JSON string risked leaking one request's labels onto another that happened to produce the same cache key. `parseTrafficLogDirective` now parses the marker fresh on every request. | +| 2026-07-07 | Log publisher field projection (`applyFieldsProjection`) now shallow-decodes the marshaled line into `map[string]json.RawMessage` instead of a full `map[string]interface{}`, and only decodes/re-encodes the specific nested objects a dotted `fields` path references — avoiding a deep unmarshal/remarshal of the whole event on every projected line. | +| 2026-07-07 | Renamed the `log-message` **`labels`** param to **`properties`** (and the emitted top-level log-line key, the marker key, `TrafficLogDirective.Properties`, and the policy's `buildProperties`) — a straight rename of the concept, no behavioral change. `fields` dotted paths are now `properties.`. | +| 2026-07-07 | Gave the traffic log its own **microsecond** latencies block (`dto.TrafficLogLatencies`: `durationUs`, `requestMediationLatencyUs`, `responseMediationLatencyUs`, `backendLatencyUs`), computed from the ALS `CommonProperties` timepoints at full precision via a new `toUs` in `analytics.go`. It no longer embeds the Moesif-oriented `dto.Latencies` (milliseconds), so the Moesif-oriented `*Latency` fields no longer leak into the traffic-log line and Moesif's millisecond units are unchanged. Dropped the now-unused millisecond `Duration`/`BackendProcDuration`/`ResponseProcDuration` fields (and `Get/SetDuration`) from `dto.Latencies`. | diff --git a/gateway/gateway-runtime/policy-engine/cmd/policy-engine/main.go b/gateway/gateway-runtime/policy-engine/cmd/policy-engine/main.go index 031f527745..e63886b085 100644 --- a/gateway/gateway-runtime/policy-engine/cmd/policy-engine/main.go +++ b/gateway/gateway-runtime/policy-engine/cmd/policy-engine/main.go @@ -268,10 +268,12 @@ func main() { metrics.StartMemoryMetricsUpdater(ctx, 15*time.Second) } - // Start access log service server if enabled + // Start the access log service server when the collector is enabled. The + // collector is the shared transport that carries collected data to its + // consumers (analytics, traffic logging). var alsServer *grpc.Server - slog.DebugContext(ctx, "Policy engine ALS server config", "config", cfg.Analytics.AccessLogsServiceCfg) - if cfg.Analytics.Enabled { + slog.DebugContext(ctx, "Policy engine ALS server config", "config", cfg.Collector.Server) + if cfg.IsCollectorEnabled() { // Start the access log service server slog.Info("Starting the ALS gRPC server...") alsServer = utils.StartAccessLogServiceServer(cfg) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go index 2b84536570..39a9896981 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go @@ -64,6 +64,12 @@ const ( RequestHeadersKey = "request_headers" ResponseHeadersKey = "response_headers" + // TrafficLogMetadataKey is the analytics-metadata key under which the + // log-message policy (in access-log mode) stamps its per-API opt-in marker. + // Its presence gates the stdout traffic-logging publisher; its value (a JSON + // directive) shapes the emitted line. Must match the key used by that policy. + TrafficLogMetadataKey = "traffic_log" + // PromptTokenCountMetadataKey represents the prompt token count metadata key. PromptTokenCountMetadataKey string = "aitoken:prompttokencount" // CompletionTokenCountMetadataKey represents the completion token count metadata key. @@ -90,14 +96,18 @@ type Analytics struct { publishers []analytics_publisher.Publisher } -// NewAnalytics creates a new instance of Analytics. +// NewAnalytics creates a new instance of Analytics. Publishers are assembled from +// each independently-configured consumer of the collected data: the analytics +// consumer ([analytics], e.g. Moesif) and the traffic-logging consumer +// ([traffic_logging], stdout JSON). Both rely on the collector being enabled to +// receive any events. func NewAnalytics(cfg *config.Config) *Analytics { analyticsCfg := cfg.Analytics publishers := make([]analytics_publisher.Publisher, 0) if analyticsCfg.Enabled { for _, publisherName := range analyticsCfg.EnabledPublishers { switch publisherName { - case "moesif": + case MoesifAnalyticsPublisher: publisher := analytics_publisher.NewMoesif(&analyticsCfg.Publishers.Moesif) if publisher != nil { publishers = append(publishers, publisher) @@ -109,8 +119,14 @@ func NewAnalytics(cfg *config.Config) *Analytics { } } + // Traffic logging is a standalone consumer, independent of analytics. + if cfg.TrafficLogging.Enabled { + publishers = append(publishers, analytics_publisher.NewLog(&cfg.TrafficLogging)) + slog.Info("Traffic logging (stdout) publisher added") + } + if len(publishers) == 0 { - slog.Debug("No analytics publishers found. Analytics will not be published.") + slog.Debug("No analytics publishers found. Collected events will not be published.") } return &Analytics{ cfg: cfg, @@ -133,7 +149,6 @@ func (c *Analytics) Process(event *v3.HTTPAccessLogEntry) { return } - // Add logic to publish the event analyticEvent := c.prepareAnalyticEvent(event) for _, publisher := range c.publishers { publisher.Publish(analyticEvent) @@ -197,6 +212,17 @@ func (c *Analytics) prepareAnalyticEvent(logEntry *v3.HTTPAccessLogEntry) *dto.E for key, value := range keyValuePairsFromMetadata { slog.Debug(fmt.Sprintf("Metadata key: %v -> value: %+v", key, value)) } + + // Extract the per-API traffic-log opt-in marker stamped by the log-message + // policy (access-log mode). Its presence marks the API as opted in to stdout + // traffic logging; + // its value shapes the emitted line. It is intentionally kept off + // event.Properties so it never reaches the serialized output or other + // publishers (e.g. Moesif). A malformed marker still opts the API in with + // default (empty) presentation, since attachment alone signals intent. + if raw, exists := keyValuePairsFromMetadata[TrafficLogMetadataKey]; exists { + event.TrafficLog = c.parseTrafficLogDirective(raw) + } // Prepare extended API extendedAPI := dto.ExtendedAPI{} extendedAPI.APIType = keyValuePairsFromMetadata[APITypeKey] @@ -247,34 +273,53 @@ func (c *Analytics) prepareAnalyticEvent(logEntry *v3.HTTPAccessLogEntry) *dto.E properties.TimeToFirstUpstreamTxByte != nil && properties.TimeToFirstUpstreamRxByte != nil && properties.TimeToLastUpstreamRxByte != nil && properties.TimeToLastDownstreamTxByte != nil { - lastRx := - (properties.TimeToLastRxByte.Seconds * 1000) + - (int64(properties.TimeToLastRxByte.Nanos) / 1_000_000) - - firstUpTx := - (properties.TimeToFirstUpstreamTxByte.Seconds * 1000) + - (int64(properties.TimeToFirstUpstreamTxByte.Nanos) / 1_000_000) - - firstUpRx := - (properties.TimeToFirstUpstreamRxByte.Seconds * 1000) + - (int64(properties.TimeToFirstUpstreamRxByte.Nanos) / 1_000_000) - - lastUpRx := - (properties.TimeToLastUpstreamRxByte.Seconds * 1000) + - (int64(properties.TimeToLastUpstreamRxByte.Nanos) / 1_000_000) + toMs := func(secs int64, nanos int32) int64 { + return (secs * 1000) + int64(nanos)/1_000_000 + } + toUs := func(secs int64, nanos int32) int64 { + return (secs * 1_000_000) + int64(nanos)/1000 + } - lastDownTx := - (properties.TimeToLastDownstreamTxByte.Seconds * 1000) + - (int64(properties.TimeToLastDownstreamTxByte.Nanos) / 1_000_000) + // Moesif-oriented latencies (milliseconds). + lastRx := toMs(properties.TimeToLastRxByte.Seconds, properties.TimeToLastRxByte.Nanos) + firstUpTx := toMs(properties.TimeToFirstUpstreamTxByte.Seconds, properties.TimeToFirstUpstreamTxByte.Nanos) + firstUpRx := toMs(properties.TimeToFirstUpstreamRxByte.Seconds, properties.TimeToFirstUpstreamRxByte.Nanos) + lastUpRx := toMs(properties.TimeToLastUpstreamRxByte.Seconds, properties.TimeToLastUpstreamRxByte.Nanos) + lastDownTx := toMs(properties.TimeToLastDownstreamTxByte.Seconds, properties.TimeToLastDownstreamTxByte.Nanos) - latencies := dto.Latencies{ + event.Latencies = &dto.Latencies{ BackendLatency: lastUpRx - firstUpTx, RequestMediationLatency: firstUpTx - lastRx, ResponseLatency: lastDownTx - firstUpRx, ResponseMediationLatency: lastDownTx - lastUpRx, } - event.Latencies = &latencies + // Traffic-log latencies (microseconds), derived from the same timepoints + // at full precision. Kept separate from the millisecond Latencies above so + // Moesif's units are unaffected. + lastRxUs := toUs(properties.TimeToLastRxByte.Seconds, properties.TimeToLastRxByte.Nanos) + firstUpTxUs := toUs(properties.TimeToFirstUpstreamTxByte.Seconds, properties.TimeToFirstUpstreamTxByte.Nanos) + firstUpRxUs := toUs(properties.TimeToFirstUpstreamRxByte.Seconds, properties.TimeToFirstUpstreamRxByte.Nanos) + lastDownTxUs := toUs(properties.TimeToLastDownstreamTxByte.Seconds, properties.TimeToLastDownstreamTxByte.Nanos) + + trafficLatencies := dto.TrafficLogLatencies{ + DurationUs: lastDownTxUs, // DS_RX_BEG → DS_TX_END + RequestMediationLatencyUs: firstUpTxUs - lastRxUs, // DS_RX_END → US_TX_BEG + } + + // US_TX_END → US_RX_BEG: time the backend spent before sending the first response byte (TTFB). + if properties.TimeToLastUpstreamTxByte != nil { + lastUpTxUs := toUs(properties.TimeToLastUpstreamTxByte.Seconds, properties.TimeToLastUpstreamTxByte.Nanos) + trafficLatencies.BackendLatencyUs = firstUpRxUs - lastUpTxUs + } + + // US_RX_BEG → DS_TX_BEG: gateway overhead processing the first response byte before writing downstream. + if properties.TimeToFirstDownstreamTxByte != nil { + firstDownTxUs := toUs(properties.TimeToFirstDownstreamTxByte.Seconds, properties.TimeToFirstDownstreamTxByte.Nanos) + trafficLatencies.ResponseMediationLatencyUs = firstDownTxUs - firstUpRxUs + } + + event.TrafficLogLatencies = &trafficLatencies } // prepare metaInfo @@ -439,22 +484,22 @@ func (c *Analytics) prepareAnalyticEvent(logEntry *v3.HTTPAccessLogEntry) *dto.E //Adding request and response headers for the analytics event if requestHeaders, exists := keyValuePairsFromMetadata[RequestHeadersKey]; exists { - event.Properties["requestHeaders"] = requestHeaders + event.Properties[dto.PropKeyRequestHeaders] = requestHeaders } if responseHeaders, exists := keyValuePairsFromMetadata[ResponseHeadersKey]; exists { - event.Properties["responseHeaders"] = responseHeaders + event.Properties[dto.PropKeyResponseHeaders] = responseHeaders } - // Optionally attach request and response payloads when enabled via configuration. - if c.cfg.Analytics.SendRequestBody { - if requestPayload, ok := keyValuePairsFromMetadata["request_payload"]; ok && requestPayload != "" { - event.Properties["request_payload"] = requestPayload + // Optionally attach request and response payloads when enabled via the collector. + if c.cfg.Collector.RequestBody { + if requestPayload, ok := keyValuePairsFromMetadata[dto.PropKeyRequestPayload]; ok && requestPayload != "" { + event.Properties[dto.PropKeyRequestPayload] = requestPayload slog.Debug("Analytics request payload captured", "size_bytes", len(requestPayload)) } } - if c.cfg.Analytics.SendResponseBody { - if responsePayload, ok := keyValuePairsFromMetadata["response_payload"]; ok && responsePayload != "" { - event.Properties["response_payload"] = responsePayload + if c.cfg.Collector.ResponseBody { + if responsePayload, ok := keyValuePairsFromMetadata[dto.PropKeyResponsePayload]; ok && responsePayload != "" { + event.Properties[dto.PropKeyResponsePayload] = responsePayload slog.Debug("Analytics response payload captured", "size_bytes", len(responsePayload)) } } @@ -505,6 +550,18 @@ func (c *Analytics) prepareAnalyticEvent(logEntry *v3.HTTPAccessLogEntry) *dto.E return event } +// parseTrafficLogDirective parses the raw traffic_log directive JSON for the +// current request. +func (c *Analytics) parseTrafficLogDirective(raw string) *dto.TrafficLogDirective { + dir := &dto.TrafficLogDirective{} + if raw != "" { + if err := json.Unmarshal([]byte(raw), dir); err != nil { + slog.Warn("Failed to parse traffic_log directive; opting in with defaults", "error", err) + } + } + return dir +} + func (c *Analytics) getAnonymousApp() *dto.Application { application := &dto.Application{} application.ApplicationID = anonymousValue diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go index 99e6b4122e..268572ad93 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go @@ -63,8 +63,8 @@ func validAnalyticsConfigForValidation(analytics config.AnalyticsConfig) *config }, Analytics: analytics, } - cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = config.AccessLogsServiceConfig{ + cfg.Analytics.Enabled = true // a consumer being on makes the collector implicit + cfg.Collector.Server = config.AccessLogsServiceConfig{ Mode: "uds", ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -138,6 +138,18 @@ func TestNewAnalytics_EnabledWithUnknownPublisherType(t *testing.T) { assert.Empty(t, analytics.publishers) // Unknown type should not be added } +func TestNewAnalytics_TrafficLoggingEnabled(t *testing.T) { + // Traffic logging is a standalone consumer, independent of analytics. + cfg := &config.Config{ + TrafficLogging: config.TrafficLoggingConfig{Enabled: true}, + } + + analytics := NewAnalytics(cfg) + + require.NotNil(t, analytics) + assert.Len(t, analytics.publishers, 1) // traffic-logging publisher should be registered +} + // ============================================================================= // isInvalid Tests // ============================================================================= @@ -253,6 +265,24 @@ func TestProcess_WithMockPublisher(t *testing.T) { assert.Equal(t, "TestAPI", mockPub.event.API.APIName) } +// Path-based suppression moved to the traffic-logging publisher, so +// Analytics.Process no longer filters by path — every valid event reaches the +// publishers (each decides what to do with it). Ignored-path coverage lives in +// publishers/log_test.go. +func TestProcess_PublishesEvent(t *testing.T) { + analytics := NewAnalytics(&config.Config{}) + mockPub := &mockPublisher{} + analytics.publishers = append(analytics.publishers, mockPub) + + logEntry := &v3.HTTPAccessLogEntry{ + Response: &v3.HTTPResponseProperties{ResponseCode: wrapperspb.UInt32(200)}, + Request: &v3.HTTPRequestProperties{OriginalPath: "/api/v1/orders", RequestMethod: corev3.RequestMethod_GET}, + } + analytics.Process(logEntry) + + assert.True(t, mockPub.called, "publisher must be called") +} + func TestProcess_PanicRecovery(t *testing.T) { cfg := &config.Config{} analytics := NewAnalytics(cfg) @@ -373,6 +403,59 @@ func TestPrepareAnalyticEvent_WithFilterMetadata(t *testing.T) { assert.Equal(t, "TestApp", event.Application.ApplicationName) } +func TestPrepareAnalyticEvent_TrafficLogMarker(t *testing.T) { + cfg := &config.Config{} + analytics := NewAnalytics(cfg) + + logEntry := createLogEntryWithMetadata(map[string]string{ + APINameKey: "TestAPI", + TrafficLogMetadataKey: `{"request":{"payload":false,"headers":true}}`, + }) + + event := analytics.prepareAnalyticEvent(logEntry) + + require.NotNil(t, event) + require.NotNil(t, event.TrafficLog, "traffic_log marker should populate event.TrafficLog") + require.NotNil(t, event.TrafficLog.Request) + assert.True(t, event.TrafficLog.Request.Headers) + assert.False(t, event.TrafficLog.Request.Payload) + assert.Nil(t, event.TrafficLog.Response, "unset flow stays nil") + + // The marker must never leak into serialized properties (or other publishers). + _, inProps := event.Properties[TrafficLogMetadataKey] + assert.False(t, inProps, "traffic_log must not appear in event.Properties") +} + +func TestPrepareAnalyticEvent_NoTrafficLogMarker(t *testing.T) { + cfg := &config.Config{} + analytics := NewAnalytics(cfg) + + logEntry := createLogEntryWithMetadata(map[string]string{APINameKey: "TestAPI"}) + + event := analytics.prepareAnalyticEvent(logEntry) + + require.NotNil(t, event) + assert.Nil(t, event.TrafficLog, "no marker -> event.TrafficLog stays nil (publisher skips)") +} + +func TestPrepareAnalyticEvent_MalformedTrafficLogMarker(t *testing.T) { + cfg := &config.Config{} + analytics := NewAnalytics(cfg) + + logEntry := createLogEntryWithMetadata(map[string]string{ + TrafficLogMetadataKey: "not-json", + }) + + event := analytics.prepareAnalyticEvent(logEntry) + + require.NotNil(t, event) + // Presence alone is the opt-in signal; a malformed marker still opts in with + // default (empty) presentation. + require.NotNil(t, event.TrafficLog) + assert.Nil(t, event.TrafficLog.Request) + assert.Nil(t, event.TrafficLog.Response) +} + func TestPrepareAnalyticEvent_WithAnonymousApp(t *testing.T) { cfg := &config.Config{} analytics := NewAnalytics(cfg) @@ -446,6 +529,11 @@ func TestPrepareAnalyticEvent_WithLatencies(t *testing.T) { require.NotNil(t, event) require.NotNil(t, event.Latencies) assert.True(t, event.Latencies.BackendLatency >= 0) + + // Traffic-log latencies are computed in microseconds from the same timepoints. + require.NotNil(t, event.TrafficLogLatencies) + assert.Equal(t, int64(250000), event.TrafficLogLatencies.DurationUs) // DS_RX_BEG → DS_TX_END = 250ms + assert.Equal(t, int64(50000), event.TrafficLogLatencies.RequestMediationLatencyUs) // 100ms - 50ms } func TestPrepareAnalyticEvent_WithUserID(t *testing.T) { @@ -469,8 +557,8 @@ func TestPrepareAnalyticEvent_WithLLMCost(t *testing.T) { analytics := NewAnalytics(cfg) logEntry := createLogEntryWithMetadata(map[string]string{ - AIProviderNameMetadataKey: "openai", - ModelIDMetadataKey: "gpt-4", + AIProviderNameMetadataKey: "openai", + ModelIDMetadataKey: "gpt-4", constants.LLMCostMetadataKey: "0.0000423100", }) @@ -525,9 +613,9 @@ func TestPrepareAnalyticEvent_WithRequestResponseHeaders(t *testing.T) { func TestPrepareAnalyticEvent_WithPayloadsEnabled(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ - SendRequestBody: true, - SendResponseBody: true, + Collector: config.CollectorConfig{ + RequestBody: true, + ResponseBody: true, }, } analytics := NewAnalytics(cfg) @@ -551,9 +639,9 @@ func TestPrepareAnalyticEvent_WithPayloadsEnabled(t *testing.T) { func TestPrepareAnalyticEvent_WithPayloadsDisabled(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ - SendRequestBody: false, - SendResponseBody: false, + Collector: config.CollectorConfig{ + RequestBody: false, + ResponseBody: false, }, } analytics := NewAnalytics(cfg) @@ -574,9 +662,9 @@ func TestPrepareAnalyticEvent_WithPayloadsDisabled(t *testing.T) { func TestPrepareAnalyticEvent_RequestPayloadOnly(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ - SendRequestBody: true, - SendResponseBody: false, + Collector: config.CollectorConfig{ + RequestBody: true, + ResponseBody: false, }, } analytics := NewAnalytics(cfg) @@ -598,9 +686,9 @@ func TestPrepareAnalyticEvent_RequestPayloadOnly(t *testing.T) { func TestPrepareAnalyticEvent_ResponsePayloadOnly(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ - SendRequestBody: false, - SendResponseBody: true, + Collector: config.CollectorConfig{ + RequestBody: false, + ResponseBody: true, }, } analytics := NewAnalytics(cfg) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/dto_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/dto_test.go index c544ddc0e5..b771083d1d 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/dto_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/dto_test.go @@ -147,14 +147,6 @@ func TestLatencies_GetSetResponseMediationLatency(t *testing.T) { assert.Equal(t, int64(30), lat.GetResponseMediationLatency()) } -func TestLatencies_GetSetDuration(t *testing.T) { - lat := &Latencies{} - assert.Equal(t, int64(0), lat.GetDuration()) - - lat.SetDuration(200) - assert.Equal(t, int64(200), lat.GetDuration()) -} - // ============================================================================= // AIMetadata Tests // ============================================================================= diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go index 6b2f05829b..7cc1c2f158 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go @@ -19,6 +19,16 @@ package dto import "time" +// Property keys written into Event.Properties by the analytics pipeline and read +// back by publishers (e.g. the Log publisher's masking and field-projection paths). +// Both sides must use these constants so a rename stays in one place. +const ( + PropKeyRequestHeaders = "requestHeaders" + PropKeyResponseHeaders = "responseHeaders" + PropKeyRequestPayload = "request_payload" + PropKeyResponsePayload = "response_payload" +) + // Event represents analytics event data. type Event struct { API *ExtendedAPI `json:"api,omitempty" bson:"api"` @@ -36,4 +46,60 @@ type Event struct { UserIP string `json:"userIp,omitempty" bson:"user_ip"` ErrorType string `json:"errorType,omitempty" bson:"error_type"` Properties map[string]interface{} `json:"properties,omitempty" bson:"properties"` + + // TrafficLogLatencies carries microsecond-precision gateway/backend timings + // for the stdout traffic-logging publisher. It is computed from the same ALS + // CommonProperties timepoints as Latencies but at full precision, and is kept + // separate so Moesif's millisecond units are unaffected. Never serialized + // (json:"-") and not sent to other publishers. + TrafficLogLatencies *TrafficLogLatencies `json:"-" bson:"-"` + + // TrafficLog carries the per-API stdout traffic-logging opt-in marker stamped + // by the log-message policy (access-log mode). When nil, the API has not opted + // in and the stdout traffic-logging publisher skips the event. It is + // gating/presentation state only and is never serialized (json:"-") nor sent + // to other publishers. + TrafficLog *TrafficLogDirective `json:"-" bson:"-"` +} + +// TrafficLogDirective is the presentation config carried in the traffic-log +// marker. Field names mirror the policy's marker JSON so it round-trips. A nil +// flow means that flow was not configured. +type TrafficLogDirective struct { + Request *TrafficLogFlow `json:"request,omitempty"` + Response *TrafficLogFlow `json:"response,omitempty"` + Fields *TrafficLogFields `json:"fields,omitempty"` + // Properties holds the policy's resolved properties (context references already + // expanded at request time). The Log publisher emits them as a top-level + // "properties" object on the log line. + Properties map[string]interface{} `json:"properties,omitempty"` + // MaskedHeaders lists lower-cased header names whose values are redacted + // in the emitted log line. Merged with the global masked_headers config. + MaskedHeaders []string `json:"maskedHeaders,omitempty"` } + +// TrafficLogFlow is the per-flow (request or response) presentation config. +type TrafficLogFlow struct { + Payload bool `json:"payload"` + Headers bool `json:"headers"` + // ExcludeHeaders lists header names (case-insensitive) dropped entirely from + // this flow's headers in the emitted line. It is orthogonal to Fields and to + // masking: it always applies when the flow's headers are present, and removes + // the header rather than redacting it (use the global/per-API masked headers to + // redact instead). The log-message policy lower-cases these names when stamping + // the marker; the publisher matches case-insensitively regardless. + ExcludeHeaders []string `json:"excludeHeaders,omitempty"` +} + +// TrafficLogFields selects which fields appear in the emitted line. Exactly one +// of Only or Exclude should be set. Only keeps exactly the named fields; Exclude +// drops the named fields and keeps everything else. Names are top-level keys +// (e.g. "latencies", "requestHeaders") or dotted sub-key paths within map fields +// (e.g. "requestHeaders.authorization", "properties.env"). When set, this is +// authoritative over field presence; per-flow Payload/Headers booleans are ignored +// (global header masking still applies). If both are set, Only takes precedence. +type TrafficLogFields struct { + Only []string `json:"only,omitempty"` + Exclude []string `json:"exclude,omitempty"` +} + diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/latencies.go b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/latencies.go index 412f95f460..df1e98ebbd 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/latencies.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/latencies.go @@ -14,10 +14,13 @@ * limitations under the License. * */ - + package dto -// Latencies represents latency attributes in an analytics event. +// Latencies represents latency attributes in an analytics event. All values are +// in milliseconds: this shape feeds Moesif, which treats them as milliseconds. +// Finer-grained gateway/backend timings for the traffic log live in the separate +// TrafficLogLatencies (microseconds) so the units Moesif depends on never change. type Latencies struct { ResponseLatency int64 `json:"responseLatency"` BackendLatency int64 `json:"backendLatency"` @@ -26,6 +29,27 @@ type Latencies struct { Duration int64 `json:"duration"` } +// TrafficLogLatencies holds gateway/backend timings for a traffic-log event, in +// microseconds. It is intentionally separate from Latencies (which is shaped for +// Moesif and expressed in milliseconds) so the traffic log can carry +// finer-grained timings without changing the units Moesif depends on. The +// timepoint labels (DS_RX_BEG, US_TX_BEG, …) refer to Envoy's request/response +// timeline as reported through the ALS CommonProperties. +type TrafficLogLatencies struct { + // DurationUs is the total request duration: downstream request received → + // downstream response sent (DS_RX_BEG → DS_TX_END). + DurationUs int64 `json:"durationUs"` + // RequestMediationLatencyUs is the gateway request overhead: downstream request + // fully received → first byte sent upstream (DS_RX_END → US_TX_BEG). + RequestMediationLatencyUs int64 `json:"requestMediationLatencyUs"` + // ResponseMediationLatencyUs is the gateway response overhead: first upstream + // response byte → first downstream response byte (US_RX_BEG → DS_TX_BEG). + ResponseMediationLatencyUs int64 `json:"responseMediationLatencyUs"` + // BackendLatencyUs is the backend TTFB: upstream request fully sent → + // first upstream response byte (US_TX_END → US_RX_BEG). + BackendLatencyUs int64 `json:"backendLatencyUs"` +} + // GetResponseLatency returns the response latency. func (l *Latencies) GetResponseLatency() int64 { return l.ResponseLatency diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go new file mode 100644 index 0000000000..40e8132888 --- /dev/null +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go @@ -0,0 +1,268 @@ +/* + * Copyright (c) 2025, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package publishers + +import ( + "encoding/json" + "fmt" + "log/slog" + "os" + "strings" + "sync" + + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/analytics/dto" + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/config" +) + +// maskedHeaderValue is the placeholder written in place of a masked header value. +const maskedHeaderValue = "****" + +// Log is an analytics publisher that writes each enriched analytics event to +// stdout as a single JSON line. It is intended for log-scraping pipelines +// (Fluent Bit, Loki, ELK, etc.) and as a lightweight alternative to a SaaS +// analytics backend. The event already carries the rich metadata, headers and +// (when request_body/response_body are enabled) payloads attached by +// the analytics engine, so this publisher only serializes it. +type Log struct { + // maskedHeaders holds lower-cased header names whose values are redacted in + // the requestHeaders/responseHeaders properties before logging. + maskedHeaders map[string]bool + // maxPayloadSize caps the number of request/response payload bytes written to + // the log line (0 = no limit). Truncation is output-side only. + maxPayloadSize int + // mu serializes writes to stdout so concurrent ALS streams do not interleave. + mu sync.Mutex + // out is the destination writer; defaults to os.Stdout (overridable in tests). + out *os.File +} + +// NewLog creates a new stdout traffic-logging publisher. +func NewLog(logCfg *config.TrafficLoggingConfig) *Log { + if logCfg == nil { + logCfg = &config.TrafficLoggingConfig{} + } + + masked := make(map[string]bool, len(logCfg.MaskedHeaders)) + for _, h := range logCfg.MaskedHeaders { + h = strings.ToLower(strings.TrimSpace(h)) + if h != "" { + masked[h] = true + } + } + + return &Log{ + maskedHeaders: masked, + maxPayloadSize: logCfg.MaxPayloadSize, + out: os.Stdout, + } +} + +// Publish writes the event to stdout as JSON. Traffic logging is per-API: only +// events carrying a traffic-log directive (stamped by the log-message policy in +// access-log mode on APIs that opted in) are emitted; all others are skipped. +func (l *Log) Publish(event *dto.Event) { + if event == nil || event.TrafficLog == nil { + return + } + + dir := event.TrafficLog + tl := l.toTrafficLogEvent(event, dir) + + data, err := json.Marshal(tl) + if err != nil { + slog.Error("Failed to marshal traffic-log event", "error", err) + return + } + + if fields := dir.Fields; fields != nil && (len(fields.Only) > 0 || len(fields.Exclude) > 0) { + // Shallow-decode only the top level; untouched fields stay as raw JSON + // bytes and are never deep-decoded or re-encoded. + var m map[string]json.RawMessage + if err := json.Unmarshal(data, &m); err != nil { + slog.Error("Failed to unmarshal for field projection; emitting as-is", "error", err) + } else { + applyFieldsProjection(m, fields) + if projected, merr := json.Marshal(m); merr == nil { + data = projected + } else { + slog.Error("Failed to remarshal after field projection; emitting as-is", "error", merr) + } + } + } + + l.write(data) +} + +func (l *Log) write(data []byte) { + l.mu.Lock() + defer l.mu.Unlock() + if _, err := fmt.Fprintln(l.out, string(data)); err != nil { + slog.Error("Failed to write analytics event to stdout", "error", err) + } +} + +// parseHeadersFromString converts the JSON-encoded header value stored in +// event.Properties (a map[string]string or map[string][]string serialized by the +// ext_proc layer) into a map[string]string so it embeds as a plain JSON object +// in the log line. Other publishers (e.g. Moesif) read the raw string directly; +// the Log publisher calls this only on the local TrafficLogEvent it builds, so +// the shared event is never modified. Multi-value headers are flattened to their +// first value. Returns nil on empty input or parse failure. +func parseHeadersFromString(raw string) map[string]string { + if raw == "" { + return nil + } + var single map[string]string + if err := json.Unmarshal([]byte(raw), &single); err == nil { + return single + } + // Fallback: multi-value wire format — flatten to first value. + var multi map[string][]string + if err := json.Unmarshal([]byte(raw), &multi); err == nil { + out := make(map[string]string, len(multi)) + for k, vs := range multi { + if len(vs) > 0 { + out[k] = vs[0] + } + } + return out + } + return nil +} + +// truncatePayload returns up to maxPayloadSize bytes of the payload (0 = no +// limit). Truncation is on a byte boundary, matching the previous capture-time +// behavior. +func (l *Log) truncatePayload(s string) string { + if l.maxPayloadSize <= 0 || len(s) <= l.maxPayloadSize { + return s + } + return s[:l.maxPayloadSize] +} + +// applyFieldsProjection mutates m in place to restrict it to the configured +// fields. Names are top-level keys (e.g. "latencies", "requestHeaders") or +// dotted sub-key paths within map fields (e.g. "requestHeaders.authorization", +// "labels.env"). Only keeps exactly the named fields; Exclude drops the named +// fields and keeps everything else. If both are set, Only takes precedence. +// Top-level values are kept as raw JSON bytes; only the specific nested +// objects referenced by a dotted path are decoded and re-encoded. +func applyFieldsProjection(m map[string]json.RawMessage, fields *dto.TrafficLogFields) { + if len(fields.Only) > 0 { + directKeys := make(map[string]bool) + subKeys := make(map[string][]string) // topKey → sub-keys to keep + for _, name := range fields.Only { + if top, sub, found := strings.Cut(name, "."); found { + subKeys[top] = append(subKeys[top], sub) + } else { + directKeys[name] = true + } + } + for key := range m { + if !directKeys[key] && subKeys[key] == nil { + delete(m, key) + } + } + for top, subs := range subKeys { + if directKeys[top] { + continue // whole key kept; don't filter sub-keys + } + keep := make(map[string]bool, len(subs)) + for _, s := range subs { + keep[s] = true + } + filterNestedKeys(m, top, func(k string) bool { return keep[k] }) + } + return + } + for _, name := range fields.Exclude { + if top, sub, found := strings.Cut(name, "."); found { + filterNestedKeys(m, top, func(k string) bool { return k != sub }) + } else { + delete(m, name) + } + } +} + +// filterNestedKeys decodes the JSON object stored at m[top], keeps only the +// sub-keys for which keep returns true, and re-encodes the result back into +// m[top]. Deletes m[top] entirely if no sub-keys survive, or if m[top] is +// absent or not a JSON object. +func filterNestedKeys(m map[string]json.RawMessage, top string, keep func(string) bool) { + raw, ok := m[top] + if !ok { + return + } + var nested map[string]json.RawMessage + if err := json.Unmarshal(raw, &nested); err != nil { + return + } + for k := range nested { + if !keep(k) { + delete(nested, k) + } + } + if len(nested) == 0 { + delete(m, top) + return + } + filtered, err := json.Marshal(nested) + if err != nil { + return + } + m[top] = filtered +} + +// maskHeaders redacts header values whose names appear in mask (case-insensitive). +// Returns a new map; the input is not modified. To drop a header entirely rather +// than redacting its value, prefer the per-flow excludeHeaders directive (see +// dropHeaders), which matches header names case-insensitively. A dotted +// fields.exclude path (e.g. "requestHeaders.Authorization") can also drop a field, +// but it matches the emitted key case-sensitively — so it must reproduce the exact +// casing Envoy delivered, and is not a reliable way to drop a header by name. +func (l *Log) maskHeaders(headers map[string]string, mask map[string]bool) map[string]string { + result := make(map[string]string, len(headers)) + for name, value := range headers { + if mask[strings.ToLower(name)] { + result[name] = maskedHeaderValue + } else { + result[name] = value + } + } + return result +} + +// dropHeaders deletes, in place, any header whose name (case-insensitive) appears +// in exclude. It is a no-op when exclude is empty. Unlike masking (which redacts +// the value) this removes the key entirely, matching the log-message policy's +// per-flow excludeHeaders semantics. Callers pass the freshly-built, non-shared +// map returned by maskHeaders, so in-place mutation never affects other publishers. +func dropHeaders(headers map[string]string, exclude []string) { + if len(exclude) == 0 { + return + } + drop := make(map[string]bool, len(exclude)) + for _, h := range exclude { + drop[strings.ToLower(strings.TrimSpace(h))] = true + } + for name := range headers { + if drop[strings.ToLower(name)] { + delete(headers, name) + } + } +} diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go new file mode 100644 index 0000000000..cca18ea9bb --- /dev/null +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go @@ -0,0 +1,555 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package publishers + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/analytics/dto" + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/config" +) + +// newLogToFile builds a Log publisher that writes to a temp file, returning the +// publisher and a function that reads back what was written. +func newLogToFile(t *testing.T, cfg *config.TrafficLoggingConfig) (*Log, func() string) { + t.Helper() + path := filepath.Join(t.TempDir(), "out.log") + f, err := os.Create(path) + require.NoError(t, err) + t.Cleanup(func() { _ = f.Close() }) + + l := NewLog(cfg) + l.out = f + return l, func() string { + require.NoError(t, f.Sync()) + data, err := os.ReadFile(path) + require.NoError(t, err) + return string(data) + } +} + +// bothFlows returns a directive that opts in to logging both request and response +// headers and payloads — the "log everything captured" case. +func bothFlows() *dto.TrafficLogDirective { + return &dto.TrafficLogDirective{ + Request: &dto.TrafficLogFlow{Payload: true, Headers: true}, + Response: &dto.TrafficLogFlow{Payload: true, Headers: true}, + } +} + +// decodeLine parses the single JSON line emitted by the Log publisher. +func decodeLine(t *testing.T, out string) map[string]interface{} { + t.Helper() + var decoded map[string]interface{} + require.NoError(t, json.Unmarshal([]byte(out), &decoded)) + return decoded +} + +// headerMap returns the JSON-decoded header object as map[string]interface{} for +// easy key/value assertions. Fails the test if v is not the expected type. +func headerMap(t *testing.T, v interface{}) map[string]interface{} { + t.Helper() + m, ok := v.(map[string]interface{}) + require.True(t, ok, "expected header object (map[string]interface{}), got %T", v) + return m +} + +func TestNewLog_NilConfig(t *testing.T) { + l := NewLog(nil) + require.NotNil(t, l) + assert.Empty(t, l.maskedHeaders) +} + +// Per-API gating: an event without a traffic-log directive is never emitted. +func TestLog_Publish_SkipsWhenNoDirective(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() // no TrafficLog set + event.Properties["requestHeaders"] = `{"x-foo":"bar"}` + + l.Publish(event) + + assert.Empty(t, read(), "event without a traffic-log directive must not be logged") +} + +func TestLog_Publish_WritesJSONLineWithLatencies(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.TrafficLog = bothFlows() + event.Properties["requestHeaders"] = `{"x-foo":"bar"}` + + l.Publish(event) + + out := read() + // Single line (one trailing newline). + assert.Equal(t, 1, strings.Count(strings.TrimRight(out, "\n"), "\n")+1) + + decoded := decodeLine(t, out) + api := decoded["api"].(map[string]interface{}) + assert.Equal(t, "test-api", api["name"]) + reqH := headerMap(t, decoded["requestHeaders"]) + assert.Equal(t, "bar", reqH["x-foo"]) + + // ALS-derived latencies are always present in the line — the key improvement + // over the inline log-message policy, which could never see them. The traffic + // log carries microsecond-precision timings, separate from Moesif's ms fields. + latencies := decoded["latencies"].(map[string]interface{}) + assert.Equal(t, float64(250000), latencies["durationUs"]) +} + +// Properties from the directive are emitted as a top-level "properties" object. +func TestLog_Publish_PropertiesTopLevel(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.TrafficLog = &dto.TrafficLogDirective{ + Request: &dto.TrafficLogFlow{Headers: true}, + Properties: map[string]interface{}{ + "who": "alice", + "authType": "jwt", + "retryCount": float64(3), + }, + } + event.Properties["requestHeaders"] = `{"x-foo":"bar"}` + + l.Publish(event) + + decoded := decodeLine(t, read()) + props, ok := decoded["properties"].(map[string]interface{}) + require.True(t, ok, "expected top-level properties object, got %T", decoded["properties"]) + assert.Equal(t, "alice", props["who"]) + assert.Equal(t, "jwt", props["authType"]) + assert.Equal(t, float64(3), props["retryCount"]) + reqH := headerMap(t, decoded["requestHeaders"]) + assert.Equal(t, "bar", reqH["x-foo"]) +} + +// A directive with no properties emits no "properties" key. +func TestLog_Publish_NoPropertiesWhenAbsent(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.TrafficLog = bothFlows() + + l.Publish(event) + + decoded := decodeLine(t, read()) + _, present := decoded["properties"] + assert.False(t, present, "no properties key expected when directive has no properties") +} + +// The fields projection can select "properties" like any other top-level key. +func TestLog_Publish_PropertiesProjectableViaFields(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.TrafficLog = &dto.TrafficLogDirective{ + Properties: map[string]interface{}{"who": "alice"}, + Fields: &dto.TrafficLogFields{Only: []string{"properties"}}, + } + event.Properties["requestHeaders"] = `{"x-foo":"bar"}` + + l.Publish(event) + + decoded := decodeLine(t, read()) + props, ok := decoded["properties"].(map[string]interface{}) + require.True(t, ok, "expected properties retained by include projection") + assert.Equal(t, "alice", props["who"]) + _, hasHeaders := decoded["requestHeaders"] + assert.False(t, hasHeaders, "requestHeaders not in Only list -> dropped") +} + +func TestLog_Publish_MasksHeaders(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{MaskedHeaders: []string{"Authorization"}}) + event := createBaseEvent() + event.TrafficLog = bothFlows() + event.Properties["requestHeaders"] = `{"Authorization":"Bearer secret","x-foo":"bar"}` + event.Properties["responseHeaders"] = `{"authorization":"Bearer secret2"}` + + l.Publish(event) + + decoded := decodeLine(t, read()) + reqH := headerMap(t, decoded["requestHeaders"]) + assert.Equal(t, "****", reqH["Authorization"]) // masked + assert.Equal(t, "bar", reqH["x-foo"]) // untouched + + resH := headerMap(t, decoded["responseHeaders"]) + assert.Equal(t, "****", resH["authorization"]) // case-insensitive match +} + +// Per-API maskedHeaders are merged with the global config mask; either source alone redacts. +func TestLog_Publish_PerAPIMaskedHeadersMergedWithGlobal(t *testing.T) { + // Global config masks "authorization"; per-API directive adds "x-secret". + l, read := newLogToFile(t, &config.TrafficLoggingConfig{MaskedHeaders: []string{"authorization"}}) + event := createBaseEvent() + event.TrafficLog = &dto.TrafficLogDirective{ + Request: &dto.TrafficLogFlow{Headers: true}, + MaskedHeaders: []string{"x-secret"}, + } + event.Properties["requestHeaders"] = `{"Authorization":"Bearer s","X-Secret":"top","x-foo":"bar"}` + + l.Publish(event) + + decoded := decodeLine(t, read()) + reqH := headerMap(t, decoded["requestHeaders"]) + assert.Equal(t, "****", reqH["Authorization"], "global masked header still redacted") + assert.Equal(t, "****", reqH["X-Secret"], "per-API masked header redacted") + assert.Equal(t, "bar", reqH["x-foo"], "unmasked header unchanged") +} + +// Per-API maskedHeaders work even when no global headers are configured. +func TestLog_Publish_PerAPIMaskedHeadersNoGlobal(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.TrafficLog = &dto.TrafficLogDirective{ + Request: &dto.TrafficLogFlow{Headers: true}, + MaskedHeaders: []string{"X-Token"}, + } + event.Properties["requestHeaders"] = `{"X-Token":"secret","x-foo":"bar"}` + + l.Publish(event) + + decoded := decodeLine(t, read()) + reqH := headerMap(t, decoded["requestHeaders"]) + assert.Equal(t, "****", reqH["X-Token"]) + assert.Equal(t, "bar", reqH["x-foo"]) +} + +// fields.exclude with a dotted path drops a specific header entirely (vs global masking which redacts). +func TestLog_Publish_ExcludeHeadersDrops(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{MaskedHeaders: []string{"authorization"}}) + event := createBaseEvent() + event.TrafficLog = &dto.TrafficLogDirective{ + Request: &dto.TrafficLogFlow{Headers: true}, + Fields: &dto.TrafficLogFields{Exclude: []string{"requestHeaders.X-Secret"}}, + } + event.Properties["requestHeaders"] = `{"Authorization":"Bearer s","X-Secret":"top","x-foo":"bar"}` + + l.Publish(event) + + decoded := decodeLine(t, read()) + reqH := headerMap(t, decoded["requestHeaders"]) + + _, hasSecret := reqH["X-Secret"] + assert.False(t, hasSecret, "excluded header must be dropped") + assert.Equal(t, "****", reqH["Authorization"], "masked header still redacted") + assert.Equal(t, "bar", reqH["x-foo"]) +} + +// Per-flow excludeHeaders drops a header entirely (case-insensitive), in both the +// request and response flows, while masking still redacts other headers. This is +// the traffic-logging counterpart of the inline excludeHeaders param. +func TestLog_Publish_PerFlowExcludeHeadersDrops(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{MaskedHeaders: []string{"authorization"}}) + event := createBaseEvent() + event.TrafficLog = &dto.TrafficLogDirective{ + // Directive carries lower-cased names (as the policy stamps them); the + // captured header keys use mixed case to prove case-insensitive matching. + Request: &dto.TrafficLogFlow{Headers: true, ExcludeHeaders: []string{"x-secret"}}, + Response: &dto.TrafficLogFlow{Headers: true, ExcludeHeaders: []string{"set-cookie"}}, + } + event.Properties["requestHeaders"] = `{"Authorization":"Bearer s","X-Secret":"top","x-foo":"bar"}` + event.Properties["responseHeaders"] = `{"Set-Cookie":"sid=1","x-bar":"baz"}` + + l.Publish(event) + + decoded := decodeLine(t, read()) + + reqH := headerMap(t, decoded["requestHeaders"]) + _, hasSecret := reqH["X-Secret"] + assert.False(t, hasSecret, "excluded request header must be dropped entirely") + assert.Equal(t, "****", reqH["Authorization"], "masked header still redacted") + assert.Equal(t, "bar", reqH["x-foo"], "untouched header retained") + + resH := headerMap(t, decoded["responseHeaders"]) + _, hasCookie := resH["Set-Cookie"] + assert.False(t, hasCookie, "excluded response header must be dropped entirely") + assert.Equal(t, "baz", resH["x-bar"], "untouched response header retained") +} + +// Per-flow excludeHeaders must not mutate the shared event, so other publishers +// (e.g. Moesif) still see the full captured header set. +func TestLog_Publish_PerFlowExcludeHeadersDoesNotMutateSharedEvent(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.TrafficLog = &dto.TrafficLogDirective{ + Request: &dto.TrafficLogFlow{Headers: true, ExcludeHeaders: []string{"x-secret"}}, + } + const raw = `{"X-Secret":"top","x-foo":"bar"}` + event.Properties["requestHeaders"] = raw + + l.Publish(event) + + decoded := decodeLine(t, read()) + reqH := headerMap(t, decoded["requestHeaders"]) + _, hasSecret := reqH["X-Secret"] + assert.False(t, hasSecret, "header dropped from the emitted line") + assert.Equal(t, raw, event.Properties["requestHeaders"], "shared event.Properties must be untouched") +} + +// headers:false omits the headers property; a nil flow omits its whole side. +func TestLog_Publish_DisabledFieldsOmitted(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.TrafficLog = &dto.TrafficLogDirective{ + Request: &dto.TrafficLogFlow{Headers: false, Payload: true}, + // Response flow nil -> both response props dropped. + } + event.Properties["requestHeaders"] = `{"x-foo":"bar"}` + event.Properties["request_payload"] = "req-body" + event.Properties["responseHeaders"] = `{"x-bar":"baz"}` + event.Properties["response_payload"] = "resp-body" + + l.Publish(event) + + decoded := decodeLine(t, read()) + _, hasReqHeaders := decoded["requestHeaders"] + assert.False(t, hasReqHeaders, "request headers disabled -> omitted") + assert.Equal(t, "req-body", decoded["requestBody"], "request payload enabled -> kept") + + _, hasRespHeaders := decoded["responseHeaders"] + _, hasRespPayload := decoded["responseBody"] + assert.False(t, hasRespHeaders, "nil response flow -> response headers omitted") + assert.False(t, hasRespPayload, "nil response flow -> response payload omitted") +} + +func TestLog_Publish_DoesNotMutateSharedEvent(t *testing.T) { + l, _ := newLogToFile(t, &config.TrafficLoggingConfig{MaskedHeaders: []string{"authorization"}}) + event := createBaseEvent() + event.TrafficLog = bothFlows() + original := `{"authorization":"Bearer secret"}` + event.Properties["requestHeaders"] = original + + l.Publish(event) + + // The shared event (read by other publishers) must be untouched. + assert.Equal(t, original, event.Properties["requestHeaders"]) +} + +func TestLog_Publish_NilEvent(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + assert.NotPanics(t, func() { l.Publish(nil) }) + assert.Empty(t, read()) +} + +// Field selection (include): only named top-level keys survive; fields.only is +// authoritative over presence (request.headers boolean is ignored, masking still applies). +func TestLog_Publish_FieldsInclude(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{MaskedHeaders: []string{"authorization"}}) + event := createBaseEvent() + event.Properties["requestHeaders"] = `{"Authorization":"Bearer s","X-Keep":"k"}` + event.Properties["responseHeaders"] = `{"x-bar":"baz"}` + event.TrafficLog = &dto.TrafficLogDirective{ + Request: &dto.TrafficLogFlow{Headers: false}, // boolean ignored when fields set + Fields: &dto.TrafficLogFields{Only: []string{"latencies", "requestHeaders"}}, + } + + l.Publish(event) + decoded := decodeLine(t, read()) + + _, hasAPI := decoded["api"] + _, hasOp := decoded["operation"] + assert.False(t, hasAPI, "api not listed -> dropped") + assert.False(t, hasOp, "operation not listed -> dropped") + assert.Contains(t, decoded, "latencies", "latencies listed -> kept") + + _, hasResp := decoded["responseHeaders"] + assert.False(t, hasResp, "responseHeaders not listed -> dropped") + + require.NotNil(t, decoded["requestHeaders"], "requestHeaders present (fields authoritative, boolean ignored)") + reqH := headerMap(t, decoded["requestHeaders"]) + assert.Equal(t, "****", reqH["Authorization"], "masking still applies") + assert.Equal(t, "k", reqH["X-Keep"]) +} + +// Field selection (exclude): named keys are dropped, everything else remains. +func TestLog_Publish_FieldsExclude(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.Properties["requestHeaders"] = `{"x-foo":"bar"}` + event.Properties["request_payload"] = "secret-body" + event.TrafficLog = &dto.TrafficLogDirective{ + Fields: &dto.TrafficLogFields{Exclude: []string{"operation", "requestBody"}}, + } + + l.Publish(event) + decoded := decodeLine(t, read()) + + _, hasOp := decoded["operation"] + assert.False(t, hasOp, "operation excluded") + assert.Contains(t, decoded, "api", "api kept") + assert.Contains(t, decoded, "latencies", "latencies kept") + _, hasPayload := decoded["requestBody"] + assert.False(t, hasPayload, "requestBody excluded") + assert.Contains(t, decoded, "requestHeaders", "requestHeaders kept (not excluded)") +} + +// An unrelated fields.exclude entry (dropping one header sub-key) must not defeat +// an explicitly configured flow's payload:false/headers:true booleans. Regression +// test for a bug where any fields.exclude entry made the per-flow booleans +// globally inert, silently re-enabling payload logging that request.payload:false +// was supposed to suppress. +func TestLog_Publish_FieldsExcludeDoesNotOverrideExplicitFlowBooleans(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.Properties["requestHeaders"] = `{"Cookie":"sid=1","x-foo":"bar"}` + event.Properties["request_payload"] = "secret-request-body" + event.Properties["responseHeaders"] = `{"x-bar":"baz"}` + event.Properties["response_payload"] = "secret-response-body" + event.TrafficLog = &dto.TrafficLogDirective{ + Request: &dto.TrafficLogFlow{Headers: true, Payload: false}, + Response: &dto.TrafficLogFlow{Headers: true, Payload: false}, + Fields: &dto.TrafficLogFields{Exclude: []string{"requestHeaders.Cookie"}}, + } + + l.Publish(event) + decoded := decodeLine(t, read()) + + _, hasReqBody := decoded["requestBody"] + assert.False(t, hasReqBody, "request.payload:false must still suppress the request body") + _, hasRespBody := decoded["responseBody"] + assert.False(t, hasRespBody, "response.payload:false must still suppress the response body") + + reqH := headerMap(t, decoded["requestHeaders"]) + _, hasCookie := reqH["Cookie"] + assert.False(t, hasCookie, "excluded header sub-key still dropped") + assert.Equal(t, "bar", reqH["x-foo"], "other request headers still present") + + respH := headerMap(t, decoded["responseHeaders"]) + assert.Equal(t, "baz", respH["x-bar"], "response headers still present per headers:true") +} + +// requestBody and properties are top-level keys like any other and can be selected +// explicitly via fields.only. +func TestLog_Publish_FieldsIncludeRequestBodyAndProperties(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.Properties["requestHeaders"] = `{"x-foo":"bar"}` + event.Properties["request_payload"] = "body-data" + event.TrafficLog = &dto.TrafficLogDirective{ + Properties: map[string]interface{}{"env": "prod"}, + Fields: &dto.TrafficLogFields{Only: []string{"requestBody", "properties"}}, + } + + l.Publish(event) + decoded := decodeLine(t, read()) + + _, hasAPI := decoded["api"] + assert.False(t, hasAPI, "api not listed -> dropped") + _, hasHeaders := decoded["requestHeaders"] + assert.False(t, hasHeaders, "requestHeaders not in Only list -> dropped") + assert.Equal(t, "body-data", decoded["requestBody"]) + props, ok := decoded["properties"].(map[string]interface{}) + require.True(t, ok) + assert.Equal(t, "prod", props["env"]) +} + +// Output-side payload truncation (0 = no limit). +func TestLog_Publish_TruncatesPayload(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{MaxPayloadSize: 5}) + event := createBaseEvent() + event.TrafficLog = bothFlows() + event.Properties["request_payload"] = "hello world" + event.Properties["response_payload"] = "goodbye world" + + l.Publish(event) + decoded := decodeLine(t, read()) + assert.Equal(t, "hello", decoded["requestBody"]) + assert.Equal(t, "goodb", decoded["responseBody"]) +} + +func TestLog_Publish_NoTruncationWhenZero(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{MaxPayloadSize: 0}) + event := createBaseEvent() + event.TrafficLog = bothFlows() + event.Properties["request_payload"] = "hello world" + + l.Publish(event) + decoded := decodeLine(t, read()) + assert.Equal(t, "hello world", decoded["requestBody"]) +} + +func TestLog_Publish_UnparseableHeadersDropped(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{MaskedHeaders: []string{"authorization"}}) + event := createBaseEvent() + event.TrafficLog = bothFlows() + event.Properties["requestHeaders"] = "not-json" + + l.Publish(event) + + decoded := decodeLine(t, read()) + _, hasHeaders := decoded["requestHeaders"] + assert.False(t, hasHeaders, "unparseable header value must be silently dropped") +} + +// Application is omitted entirely for unauthenticated requests (all fields empty). +func TestLog_Publish_UnauthenticatedRequestOmitsApplication(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.Application = &dto.Application{} // all fields are "" + event.TrafficLog = bothFlows() + + l.Publish(event) + + decoded := decodeLine(t, read()) + _, hasApp := decoded["application"] + assert.False(t, hasApp, "application with all-empty fields must be absent") +} + +// TrafficLogAPI uses clean field names (id/name/kind) not the Moesif apiId/apiName/apiType. +func TestLog_Publish_APIFieldNames(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.TrafficLog = bothFlows() + + l.Publish(event) + + decoded := decodeLine(t, read()) + api, ok := decoded["api"].(map[string]interface{}) + require.True(t, ok, "api object must be present") + assert.Equal(t, "api-123", api["id"]) + assert.Equal(t, "test-api", api["name"]) + assert.Equal(t, "v1.0", api["version"]) + assert.Equal(t, "Rest", api["kind"]) + assert.Equal(t, "/test", api["context"]) + assert.Equal(t, "project-123", api["projectId"]) + // Moesif-specific keys must not bleed into traffic logs. + for _, moesifKey := range []string{"apiId", "apiName", "apiCreator", "apiCreatorTenantDomain", "organizationId"} { + _, present := api[moesifKey] + assert.False(t, present, "Moesif field %q must not appear in traffic log", moesifKey) + } +} + +// Top-level fields — status, correlationId, client — are always present when set. +func TestLog_Publish_TopLevelFieldsPresent(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.TrafficLog = bothFlows() + + l.Publish(event) + + decoded := decodeLine(t, read()) + assert.Equal(t, float64(200), decoded["status"], "proxy response code must appear as status") + assert.Equal(t, "corr-123", decoded["correlationId"]) + client, ok := decoded["client"].(map[string]interface{}) + require.True(t, ok, "client object must be present") + assert.Equal(t, "192.168.1.1", client["ip"]) + assert.Equal(t, "test-agent", client["userAgent"]) +} diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/moesif_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/moesif_test.go index 63941d4615..1a5afa8197 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/moesif_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/moesif_test.go @@ -85,6 +85,12 @@ func createBaseEvent() *dto.Event { Latencies: &dto.Latencies{ ResponseLatency: 100, }, + TrafficLogLatencies: &dto.TrafficLogLatencies{ + DurationUs: 250000, + RequestMediationLatencyUs: 50000, + ResponseMediationLatencyUs: 30000, + BackendLatencyUs: 40000, + }, } } diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/traffic_log_event.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/traffic_log_event.go new file mode 100644 index 0000000000..669345c473 --- /dev/null +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/traffic_log_event.go @@ -0,0 +1,234 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package publishers + +import ( + "strings" + + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/analytics/dto" +) + +// trafficLogTimestampFormat is RFC 3339 with millisecond precision. +const trafficLogTimestampFormat = "2006-01-02T15:04:05.000Z07:00" + +// TrafficLogEvent is the JSON shape written to stdout by the Log publisher. +// It is intentionally separate from dto.Event (shaped for Moesif) so its field +// names, schema, and presence rules can evolve independently. All string fields +// carry omitempty so absent or unknown values produce no key rather than "". +type TrafficLogEvent struct { + Timestamp string `json:"timestamp,omitempty"` + CorrelationID string `json:"correlationId,omitempty"` + Status int `json:"status,omitempty"` + API *TrafficLogAPI `json:"api,omitempty"` + Operation *TrafficLogOperation `json:"operation,omitempty"` + Target *TrafficLogTarget `json:"target,omitempty"` + Application *TrafficLogApplication `json:"application,omitempty"` + Client *TrafficLogClient `json:"client,omitempty"` + Latencies *dto.TrafficLogLatencies `json:"latencies,omitempty"` + RequestHeaders map[string]string `json:"requestHeaders,omitempty"` + ResponseHeaders map[string]string `json:"responseHeaders,omitempty"` + RequestBody string `json:"requestBody,omitempty"` + ResponseBody string `json:"responseBody,omitempty"` + Properties map[string]interface{} `json:"properties,omitempty"` +} + +// TrafficLogAPI identifies the API that processed the request. +type TrafficLogAPI struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Version string `json:"version,omitempty"` + Context string `json:"context,omitempty"` + Kind string `json:"kind,omitempty"` + ProjectID string `json:"projectId,omitempty"` +} + +// TrafficLogOperation describes the matched operation within the API. +type TrafficLogOperation struct { + Method string `json:"method,omitempty"` + Path string `json:"path,omitempty"` +} + +// TrafficLogTarget holds upstream response information. +type TrafficLogTarget struct { + StatusCode int `json:"statusCode,omitempty"` + Destination string `json:"destination,omitempty"` +} + +// TrafficLogApplication is present only for authenticated requests. +type TrafficLogApplication struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Owner string `json:"owner,omitempty"` + KeyType string `json:"keyType,omitempty"` +} + +// TrafficLogClient holds downstream caller information. +type TrafficLogClient struct { + IP string `json:"ip,omitempty"` + UserAgent string `json:"userAgent,omitempty"` +} + +// toTrafficLogEvent translates a dto.Event and its traffic-log directive into the +// traffic-log-specific output shape, applying per-flow header filtering, global +// header masking, and payload truncation. +func (l *Log) toTrafficLogEvent(event *dto.Event, dir *dto.TrafficLogDirective) *TrafficLogEvent { + tl := &TrafficLogEvent{ + Status: event.ProxyResponseCode, + Latencies: event.TrafficLogLatencies, + } + + if !event.RequestTimestamp.IsZero() { + tl.Timestamp = event.RequestTimestamp.UTC().Format(trafficLogTimestampFormat) + } + + if event.MetaInfo != nil { + tl.CorrelationID = event.MetaInfo.CorrelationID + } + + if event.API != nil { + tl.API = &TrafficLogAPI{ + ID: event.API.APIID, + Name: event.API.APIName, + Version: event.API.APIVersion, + Context: event.API.APIContext, + Kind: event.API.APIType, + ProjectID: event.API.ProjectID, + } + } + + if event.Operation != nil { + tl.Operation = &TrafficLogOperation{ + Method: event.Operation.APIMethod, + Path: event.Operation.APIResourceTemplate, + } + } + + if event.Target != nil { + tl.Target = &TrafficLogTarget{ + StatusCode: event.Target.TargetResponseCode, + Destination: event.Target.Destination, + } + } + + // Application is only meaningful for authenticated requests. + if a := event.Application; a != nil && (a.ApplicationID != "" || a.ApplicationName != "") { + tl.Application = &TrafficLogApplication{ + ID: a.ApplicationID, + Name: a.ApplicationName, + Owner: a.ApplicationOwner, + KeyType: a.KeyType, + } + } + + if event.UserIP != "" || event.UserAgentHeader != "" { + tl.Client = &TrafficLogClient{ + IP: event.UserIP, + UserAgent: event.UserAgentHeader, + } + } + + // fields.only is a whitelist and is always authoritative over presence: every + // field gets attached here and applyFieldsProjection in Publish alone decides + // what survives, regardless of the per-flow Headers/Payload booleans. + // + // fields.exclude is different: when a flow (Request/Response) is explicitly + // configured, its Headers/Payload booleans keep governing presence as normal — + // exclude only trims already-present fields/sub-keys (e.g. one header) on top + // of that. This is what makes an unrelated `fields.exclude: + // [requestHeaders.cookie]` entry not silently defeat `request.payload: false`. + // Only when a flow isn't configured at all does exclude fall back to its + // "present unless named" convenience behavior, so `fields.exclude` alone (with + // no request/response block) still works as a blanket "log everything except + // X" shorthand. + hasOnlySelection := dir.Fields != nil && len(dir.Fields.Only) > 0 + hasExcludeSelection := dir.Fields != nil && len(dir.Fields.Only) == 0 && len(dir.Fields.Exclude) > 0 + + // Effective header mask: global config merged with any per-API additions. + mask := l.maskedHeaders + if len(dir.MaskedHeaders) > 0 { + merged := make(map[string]bool, len(l.maskedHeaders)+len(dir.MaskedHeaders)) + for k, v := range l.maskedHeaders { + merged[k] = v + } + for _, h := range dir.MaskedHeaders { + merged[strings.ToLower(h)] = true + } + mask = merged + } + + // Request flow + if raw, ok := event.Properties[dto.PropKeyRequestHeaders].(string); ok { + headersOn := dir.Request != nil && dir.Request.Headers + if fieldEnabled(hasOnlySelection, hasExcludeSelection, dir.Request, headersOn) { + if headers := parseHeadersFromString(raw); headers != nil { + masked := l.maskHeaders(headers, mask) + if dir.Request != nil { + dropHeaders(masked, dir.Request.ExcludeHeaders) + } + tl.RequestHeaders = masked + } + } + } + if p, ok := event.Properties[dto.PropKeyRequestPayload].(string); ok && p != "" { + payloadOn := dir.Request != nil && dir.Request.Payload + if fieldEnabled(hasOnlySelection, hasExcludeSelection, dir.Request, payloadOn) { + tl.RequestBody = l.truncatePayload(p) + } + } + + // Response flow + if raw, ok := event.Properties[dto.PropKeyResponseHeaders].(string); ok { + headersOn := dir.Response != nil && dir.Response.Headers + if fieldEnabled(hasOnlySelection, hasExcludeSelection, dir.Response, headersOn) { + if headers := parseHeadersFromString(raw); headers != nil { + masked := l.maskHeaders(headers, mask) + if dir.Response != nil { + dropHeaders(masked, dir.Response.ExcludeHeaders) + } + tl.ResponseHeaders = masked + } + } + } + if p, ok := event.Properties[dto.PropKeyResponsePayload].(string); ok && p != "" { + payloadOn := dir.Response != nil && dir.Response.Payload + if fieldEnabled(hasOnlySelection, hasExcludeSelection, dir.Response, payloadOn) { + tl.ResponseBody = l.truncatePayload(p) + } + } + + if len(dir.Properties) > 0 { + tl.Properties = dir.Properties + } + + return tl +} + +// fieldEnabled decides whether a request/response header or payload field should +// be attached to the traffic-log event, before any Fields projection trims it +// back down. See the comment above hasOnlySelection/hasExcludeSelection in +// toTrafficLogEvent for the reasoning. +func fieldEnabled(hasOnlySelection, hasExcludeSelection bool, flow *dto.TrafficLogFlow, boolValue bool) bool { + switch { + case hasOnlySelection: + return true + case flow != nil: + return boolValue + default: + return hasExcludeSelection + } +} diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config.go b/gateway/gateway-runtime/policy-engine/internal/config/config.go index 5dfdc688ed..cc5a1ba6d2 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config.go @@ -23,6 +23,7 @@ import ( "log/slog" "math" "net/url" + "strconv" "strings" "time" @@ -31,6 +32,7 @@ import ( "github.com/knadh/koanf/providers/env" "github.com/knadh/koanf/providers/file" "github.com/knadh/koanf/v2" + "github.com/wso2/api-platform/common/collector" ) const ( @@ -42,24 +44,44 @@ type Config struct { PolicyEngine PolicyEngine `koanf:"policy_engine"` GatewayController map[string]interface{} `koanf:"gateway_controller"` PolicyConfigurations map[string]interface{} `koanf:"policy_configurations"` + Collector CollectorConfig `koanf:"collector"` Analytics AnalyticsConfig `koanf:"analytics"` + TrafficLogging TrafficLoggingConfig `koanf:"traffic_logging"` TracingConfig TracingConfig `koanf:"tracing"` } +// CollectorConfig holds the data-collection ("collector") configuration. The +// collector is the shared capture pipeline that gathers request/response headers +// and bodies and ships them to the policy-engine over ALS. It underpins every +// consumer of that data (analytics and traffic logging) and is implicitly active +// whenever a consumer is enabled — see Config.IsCollectorEnabled. This section +// tunes capture and transport; it has no on/off flag of its own. +type CollectorConfig struct { + // RequestBody / ResponseBody attach captured request/response bodies + // onto the collected event. + RequestBody bool `koanf:"request_body"` + ResponseBody bool `koanf:"response_body"` + // Server tunes the policy-engine ALS receiver (the gRPC server that ingests + // collected access logs). It is part of the collector transport and is + // configured under the shared [collector.server] section (the controller + // reads the same section to configure Envoy's sender side). + Server AccessLogsServiceConfig `koanf:"server"` +} + // AnalyticsConfig holds analytics configuration type AnalyticsConfig struct { - Enabled bool `koanf:"enabled"` - EnabledPublishers []string `koanf:"enabled_publishers"` - Publishers AnalyticsPublishersConfig `koanf:"publishers"` - GRPCEventServerCfg map[string]interface{} `koanf:"grpc_event_server"` - AccessLogsServiceCfg AccessLogsServiceConfig `koanf:"access_logs_service"` - // AllowPayloads controls whether request and response bodies are captured - // into analytics metadata and forwarded to analytics publishers. - // Deprecated: use SendRequestBody and SendResponseBody instead. - // When true, validateAnalyticsConfig maps both SendRequestBody and SendResponseBody - // to true if both are false. Because bools cannot represent "unset", this also - // applies when both new flags are explicitly false; remove allow_payloads when - // migrating and set the directional flags directly. + Enabled bool `koanf:"enabled"` + EnabledPublishers []string `koanf:"enabled_publishers"` + Publishers AnalyticsPublishersConfig `koanf:"publishers"` + GRPCEventServerCfg map[string]interface{} `koanf:"grpc_event_server"` + // AccessLogsServiceCfg is a deprecated alias. ALS receiver tuning moved to + // [collector.server]; when set here it is migrated onto the collector during + // validation (with a warning). Prefer [collector.server]. + AccessLogsServiceCfg AccessLogsServiceConfig `koanf:"access_logs_service"` + // AllowPayloads, SendRequestBody and SendResponseBody are deprecated aliases. + // Body capture now lives under [collector]. When set, these are mapped onto + // collector.request_body / collector.response_body during validation + // (with a warning). Prefer the [collector] fields directly. AllowPayloads bool `koanf:"allow_payloads"` SendRequestBody bool `koanf:"send_request_body"` SendResponseBody bool `koanf:"send_response_body"` @@ -70,6 +92,22 @@ type AnalyticsPublishersConfig struct { Moesif MoesifPublisherConfig `koanf:"moesif"` } +// TrafficLoggingConfig holds configuration for the stdout traffic-logging feature, +// which writes each collected event to stdout as a JSON line. It is a consumer of +// the collector; enabling it implicitly activates the collector. +type TrafficLoggingConfig struct { + // Enabled turns stdout JSON traffic logging on. + Enabled bool `koanf:"enabled"` + // MaskedHeaders lists header names (case-insensitive) whose values are + // redacted in the logged requestHeaders/responseHeaders. + MaskedHeaders []string `koanf:"masked_headers"` + // MaxPayloadSize caps the number of bytes of request/response payload written + // to the log line (0 = no limit). Truncation is applied at the publisher, so + // the collector still captures the full body and other consumers (e.g. Moesif) + // are unaffected. + MaxPayloadSize int `koanf:"max_payload_size"` +} + // MoesifPublisherConfig holds Moesif-specific configuration type MoesifPublisherConfig struct { ApplicationID string `koanf:"application_id"` @@ -82,13 +120,13 @@ type MoesifPublisherConfig struct { // Config represents the complete policy engine configuration type PolicyEngine struct { - Server ServerConfig `koanf:"server"` - Admin AdminConfig `koanf:"admin"` - Metrics MetricsConfig `koanf:"metrics"` - ConfigMode ConfigModeConfig `koanf:"config_mode"` - XDS XDSConfig `koanf:"xds"` - FileConfig FileConfigConfig `koanf:"file_config"` - Logging LoggingConfig `koanf:"logging"` + Server ServerConfig `koanf:"server"` + Admin AdminConfig `koanf:"admin"` + Metrics MetricsConfig `koanf:"metrics"` + ConfigMode ConfigModeConfig `koanf:"config_mode"` + XDS XDSConfig `koanf:"xds"` + FileConfig FileConfigConfig `koanf:"file_config"` + Logging LoggingConfig `koanf:"logging"` PythonExecutor PythonExecutorConfig `koanf:"python_executor"` // Tracing holds OpenTelemetry exporter configuration TracingServiceName string `koanf:"tracing_service_name"` @@ -235,7 +273,13 @@ type LoggingConfig struct { // AccessLogsServiceConfig holds access logs service configuration type AccessLogsServiceConfig struct { - Mode string `koanf:"mode"` // Connection mode: "uds" (default) or "tcp" + Mode string `koanf:"mode"` // Connection mode: "uds" (default) or "tcp" + // ServerPort overrides the fixed ALS listener port (collector.ServerPort, 18090), + // used only in "tcp" mode. Deprecated: no longer defaulted here or documented in + // config-template.toml/Helm charts, so new deployments have no way to discover or + // set it. Kept solely so a config that already sets it explicitly keeps working; + // leave unset (0) to use the fixed port. Must match the gateway-controller's + // collector.server.port override, or the two sides will fail to connect. ServerPort int `koanf:"server_port"` ShutdownTimeout time.Duration `koanf:"shutdown_timeout"` PublicKeyPath string `koanf:"public_key_path"` @@ -304,6 +348,21 @@ func Load(configPath string) (*Config, error) { return cfg, nil } +// defaultAccessLogsServiceConfig returns the default policy-engine ALS receiver tuning. +// Shared by the collector (canonical) and the deprecated [analytics].access_logs_service +// alias so a partial alias override migrates cleanly. +func defaultAccessLogsServiceConfig() AccessLogsServiceConfig { + return AccessLogsServiceConfig{ + Mode: "", + ShutdownTimeout: 600 * time.Second, + PublicKeyPath: "", + PrivateKeyPath: "", + ALSPlainText: true, + ExtProcMaxMessageSize: 1000000000, + ExtProcMaxHeaderLimit: 8192, + } +} + // defaultConfig returns a Config struct with default configuration values func defaultConfig() *Config { return &Config{ @@ -350,6 +409,16 @@ func defaultConfig() *Config { }, TracingServiceName: "policy-engine", }, + Collector: CollectorConfig{ + RequestBody: false, + ResponseBody: false, + Server: defaultAccessLogsServiceConfig(), + }, + TrafficLogging: TrafficLoggingConfig{ + Enabled: false, + MaskedHeaders: []string{}, + MaxPayloadSize: 0, + }, Analytics: AnalyticsConfig{ Enabled: false, EnabledPublishers: []string{"moesif"}, @@ -369,19 +438,12 @@ func defaultConfig() *Config { "buffer_size_bytes": 16384, "grpc_request_timeout": 20000000000, }, - AccessLogsServiceCfg: AccessLogsServiceConfig{ - Mode: "", - ServerPort: 18090, - ShutdownTimeout: 600 * time.Second, - PublicKeyPath: "", - PrivateKeyPath: "", - ALSPlainText: true, - ExtProcMaxMessageSize: 1000000000, - ExtProcMaxHeaderLimit: 8192, - }, - AllowPayloads: false, - SendRequestBody: false, - SendResponseBody: false, + // Deprecated alias: default mirrors the collector so a partial + // [analytics.access_logs_service] override migrates cleanly. + AccessLogsServiceCfg: defaultAccessLogsServiceConfig(), + AllowPayloads: false, + SendRequestBody: false, + SendResponseBody: false, }, TracingConfig: TracingConfig{ Enabled: false, @@ -480,6 +542,12 @@ func (c *Config) Validate() error { return fmt.Errorf("invalid logging.format: %s (must be json or text)", c.PolicyEngine.Logging.Format) } + if err := c.validateCollectorConfig(); err != nil { + return err + } + if c.TrafficLogging.MaxPayloadSize < 0 { + return fmt.Errorf("traffic_logging.max_payload_size must be >= 0, got %d", c.TrafficLogging.MaxPayloadSize) + } if c.Analytics.Enabled { if err := c.validateAnalyticsConfig(); err != nil { return fmt.Errorf("analytics configuration validation failed: %v", err) @@ -536,49 +604,101 @@ func (c *Config) validateXDSConfig() error { return nil } -// validateAnalyticsConfig validates the analytics configuration -func (c *Config) validateAnalyticsConfig() error { - // Validate analytics configuration - if c.Analytics.Enabled { - // Migration path for deprecated analytics.allow_payloads. - // Runs when both directional flags are false, which is indistinguishable - // from "not set" because bool fields cannot represent unset vs explicit false. - if c.Analytics.AllowPayloads { - slog.Warn("analytics.allow_payloads is deprecated; use analytics.send_request_body and analytics.send_response_body instead") - if !c.Analytics.SendRequestBody && !c.Analytics.SendResponseBody { - c.Analytics.SendRequestBody = true - c.Analytics.SendResponseBody = true - } +// validateCollectorConfig migrates deprecated analytics capture aliases onto the +// collector and enforces the collector prerequisite: a consumer (analytics or +// traffic logging) requires the collector that feeds it. The collector has no +// on/off flag of its own: it is implicitly active whenever a consumer is enabled +// (see IsCollectorEnabled), so its transport is validated only in that case. +func (c *Config) validateCollectorConfig() error { + c.migrateDeprecatedAnalyticsCapture() + c.migrateDeprecatedAnalyticsTransport() + + if c.IsCollectorEnabled() { + if err := validateAccessLogsServiceConfig(c.Collector.Server); err != nil { + return err } + } + return nil +} - // Validate ALS server config (policy-engine side) - als := c.Analytics.AccessLogsServiceCfg - - // Validate ALS connection mode - switch als.Mode { - case "uds", "": - // UDS mode (default) - port is unused - case "tcp": - // TCP mode - validate port - if als.ServerPort <= 0 || als.ServerPort > 65535 { - return fmt.Errorf("analytics.access_logs_service.server_port must be between 1 and 65535, got %d", als.ServerPort) - } - default: - return fmt.Errorf("analytics.access_logs_service.mode must be 'uds' or 'tcp', got: %s", als.Mode) - } - if als.ShutdownTimeout <= 0 { - return fmt.Errorf("analytics.access_logs_service.shutdown_timeout must be positive, got %s", als.ShutdownTimeout) - } - if als.ExtProcMaxMessageSize <= 0 { - return fmt.Errorf("analytics.access_logs_service.max_message_size must be positive, got %d", als.ExtProcMaxMessageSize) - } - if als.ExtProcMaxHeaderLimit <= 0 { - return fmt.Errorf("analytics.access_logs_service.max_header_limit must be positive, got %d", als.ExtProcMaxHeaderLimit) - } - if als.ExtProcMaxHeaderLimit > math.MaxUint32 { - return fmt.Errorf("analytics.access_logs_service.max_header_limit must be <= %d, got %d", uint64(math.MaxUint32), als.ExtProcMaxHeaderLimit) +// IsCollectorEnabled reports whether the collector should run. The collector is +// implicit: it is active whenever any consumer of the collected data is enabled +// (analytics or stdout traffic logging), and off otherwise. +func (c *Config) IsCollectorEnabled() bool { + return collector.IsEnabled(c.Analytics.Enabled, c.TrafficLogging.Enabled) +} + +// migrateDeprecatedAnalyticsTransport maps a deprecated [analytics].access_logs_service +// override onto the collector when the collector's receiver tuning is still at its +// default, so existing configs keep working after the transport moved to [collector]. +// See collector.MigrateDeprecatedTransport for the shared (with the gateway-controller) +// migration logic and its guarding-while-analytics-enabled rationale. +func (c *Config) migrateDeprecatedAnalyticsTransport() { + collector.MigrateDeprecatedTransport( + c.Analytics.Enabled, + c.Analytics.AccessLogsServiceCfg, + &c.Collector.Server, + defaultAccessLogsServiceConfig(), + "analytics.access_logs_service", + ) +} + +// validateAccessLogsServiceConfig validates the policy-engine ALS receiver tuning. +// The transport port is normally the fixed, non-configurable collector.ServerPort +// constant (see collector.ServerPort); als.ServerPort is a deprecated override honored +// only for backward compatibility with configs that already set it (see its doc comment). +func validateAccessLogsServiceConfig(als AccessLogsServiceConfig) error { + switch als.Mode { + case "uds", "tcp", "": + default: + return fmt.Errorf("collector.server.mode must be 'uds' or 'tcp', got: %s", als.Mode) + } + if als.ServerPort != 0 { + slog.Warn("collector.server.server_port is deprecated and no longer documented; the ALS port is fixed at " + + strconv.Itoa(collector.ServerPort) + " by default. Honoring the configured override for backward " + + "compatibility — ensure the gateway-controller's collector.server.port matches, or the two sides will fail to connect.") + if als.ServerPort < 0 || als.ServerPort > 65535 { + return fmt.Errorf("collector.server.server_port must be between 1 and 65535, got %d", als.ServerPort) } + } + if als.ShutdownTimeout <= 0 { + return fmt.Errorf("collector.server.shutdown_timeout must be positive, got %s", als.ShutdownTimeout) + } + if als.ExtProcMaxMessageSize <= 0 { + return fmt.Errorf("collector.server.max_message_size must be positive, got %d", als.ExtProcMaxMessageSize) + } + if als.ExtProcMaxHeaderLimit <= 0 { + return fmt.Errorf("collector.server.max_header_limit must be positive, got %d", als.ExtProcMaxHeaderLimit) + } + if als.ExtProcMaxHeaderLimit > math.MaxUint32 { + return fmt.Errorf("collector.server.max_header_limit must be <= %d, got %d", uint64(math.MaxUint32), als.ExtProcMaxHeaderLimit) + } + return nil +} +// migrateDeprecatedAnalyticsCapture maps the deprecated analytics.allow_payloads / +// analytics.send_request_body / analytics.send_response_body onto the collector's +// body-capture flags, so existing configs keep working after capture settings +// moved under [collector]. See collector.MigrateDeprecatedCapture for the shared +// (with the gateway-controller) migration logic and its guarding-while-analytics- +// enabled rationale. +func (c *Config) migrateDeprecatedAnalyticsCapture() { + collector.MigrateDeprecatedCapture( + c.Analytics.Enabled, + collector.CaptureFlags{ + SendRequestBody: c.Analytics.SendRequestBody, + SendResponseBody: c.Analytics.SendResponseBody, + AllowPayloads: c.Analytics.AllowPayloads, + }, + &c.Collector.RequestBody, + &c.Collector.ResponseBody, + ) +} + +// validateAnalyticsConfig validates the analytics consumer configuration (publishers). +// ALS transport validation lives in validateCollectorConfig. +func (c *Config) validateAnalyticsConfig() error { + if c.Analytics.Enabled { // Validate enabled publishers for _, publisherName := range c.Analytics.EnabledPublishers { switch publisherName { diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config_test.go b/gateway/gateway-runtime/policy-engine/internal/config/config_test.go index 3470b998c9..d582275275 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config_test.go @@ -69,8 +69,15 @@ func validConfig() *Config { Timeout: 30 * time.Second, }, }, + // The collector is implicit (active whenever a consumer is enabled). ALS + // receiver defaults mirror production so transport validation passes and the + // deprecated alias stays neutral (no spurious migration). + Collector: CollectorConfig{ + Server: defaultAccessLogsServiceConfig(), + }, Analytics: AnalyticsConfig{ - Enabled: false, + Enabled: false, + AccessLogsServiceCfg: defaultAccessLogsServiceConfig(), }, TracingConfig: TracingConfig{ Enabled: false, @@ -948,7 +955,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - valid UDS config (default)", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.Server = AccessLogsServiceConfig{ Mode: "uds", ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -961,7 +968,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - valid UDS config (empty mode defaults to uds)", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.Server = AccessLogsServiceConfig{ Mode: "", ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -974,9 +981,8 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - valid TCP config", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.Server = AccessLogsServiceConfig{ Mode: "tcp", - ServerPort: 18090, ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, ExtProcMaxHeaderLimit: 8192, @@ -988,7 +994,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - invalid mode", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.Server = AccessLogsServiceConfig{ Mode: "invalid", ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -999,39 +1005,41 @@ func TestValidate_AnalyticsConfig(t *testing.T) { errMsg: "mode must be 'uds' or 'tcp'", }, { - name: "analytics enabled - TCP mode invalid ALS port", + // Backward compat: an existing config that already sets a custom port + // (the deprecated ServerPort override) must keep working, not error. + name: "analytics enabled - deprecated server_port override still accepted (backward compat)", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.Server = AccessLogsServiceConfig{ Mode: "tcp", - ServerPort: 0, + ServerPort: 9099, ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, ExtProcMaxHeaderLimit: 8192, } }, - expectErr: true, - errMsg: "server_port must be between 1 and 65535", + expectErr: false, }, { - name: "analytics enabled - UDS mode skips port validation", + name: "analytics enabled - deprecated server_port override out of range still errors", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ - Mode: "uds", - ServerPort: 0, // Invalid port, but irrelevant in UDS mode + cfg.Collector.Server = AccessLogsServiceConfig{ + Mode: "tcp", + ServerPort: 70000, ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, ExtProcMaxHeaderLimit: 8192, } }, - expectErr: false, + expectErr: true, + errMsg: "server_port must be between 1 and 65535", }, { name: "analytics enabled - invalid shutdown timeout", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.Server = AccessLogsServiceConfig{ ShutdownTimeout: 0, ExtProcMaxMessageSize: 1000000, ExtProcMaxHeaderLimit: 8192, @@ -1044,7 +1052,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - invalid max message size", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.Server = AccessLogsServiceConfig{ ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 0, ExtProcMaxHeaderLimit: 8192, @@ -1057,7 +1065,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - invalid max header limit", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.Server = AccessLogsServiceConfig{ ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, ExtProcMaxHeaderLimit: 0, @@ -1070,7 +1078,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - max header limit exceeds uint32", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.Server = AccessLogsServiceConfig{ ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, ExtProcMaxHeaderLimit: math.MaxInt, @@ -1099,8 +1107,8 @@ func TestValidate_AnalyticsConfig(t *testing.T) { func TestValidate_AnalyticsPayloadMigration(t *testing.T) { setValidAnalyticsALS := func(cfg *Config) { - cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Analytics.Enabled = true // a consumer being on makes the collector implicit + cfg.Collector.Server = AccessLogsServiceConfig{ Mode: "uds", ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -1160,12 +1168,89 @@ func TestValidate_AnalyticsPayloadMigration(t *testing.T) { err := cfg.Validate() require.NoError(t, err) - assert.Equal(t, tt.wantSendReq, cfg.Analytics.SendRequestBody) - assert.Equal(t, tt.wantSendResp, cfg.Analytics.SendResponseBody) + // Deprecated analytics body aliases now migrate onto the collector. + assert.Equal(t, tt.wantSendReq, cfg.Collector.RequestBody) + assert.Equal(t, tt.wantSendResp, cfg.Collector.ResponseBody) }) } } +// TestValidate_AnalyticsPayloadMigration_SkippedWhenAnalyticsDisabled guards +// against a stale analytics.allow_payloads left over from a disabled analytics +// setup silently turning on body capture for an unrelated consumer (traffic +// logging) enabled later. The deprecated capture aliases belong to analytics, so +// they must only be honored while analytics itself is enabled. +func TestValidate_AnalyticsPayloadMigration_SkippedWhenAnalyticsDisabled(t *testing.T) { + cfg := validConfig() + cfg.Analytics.Enabled = false + cfg.TrafficLogging.Enabled = true // an unrelated consumer activates the collector + cfg.Analytics.AllowPayloads = true + + err := cfg.Validate() + require.NoError(t, err) + assert.False(t, cfg.Collector.RequestBody) + assert.False(t, cfg.Collector.ResponseBody) +} + +// TestValidate_AnalyticsTransportMigration_SkippedWhenAnalyticsDisabled guards +// against a stale analytics.access_logs_service override left over from a disabled +// analytics setup silently reconfiguring the transport for an unrelated consumer +// (traffic logging) enabled later. The deprecated transport alias belongs to +// analytics, so it must only be honored while analytics itself is enabled. +func TestValidate_AnalyticsTransportMigration_SkippedWhenAnalyticsDisabled(t *testing.T) { + cfg := validConfig() + cfg.Analytics.Enabled = false + cfg.TrafficLogging.Enabled = true // an unrelated consumer activates the collector + cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + Mode: "uds", + ShutdownTimeout: 600 * time.Second, + ExtProcMaxMessageSize: 1000000, + ExtProcMaxHeaderLimit: 8192, + } + + err := cfg.Validate() + require.NoError(t, err) + assert.Equal(t, defaultAccessLogsServiceConfig(), cfg.Collector.Server) +} + +// TestValidate_CollectorPrerequisite verifies that enabling a consumer (analytics, +// traffic logging) without the collector auto-enables the collector (a +// backward-compat soft prerequisite) rather than failing. +// TestIsCollectorEnabled covers the implicit collector: it is active iff a consumer +// (analytics or traffic logging) is enabled, and off otherwise. +func TestIsCollectorEnabled(t *testing.T) { + t.Run("no consumers -> off", func(t *testing.T) { + cfg := validConfig() + cfg.Analytics.Enabled = false + cfg.TrafficLogging.Enabled = false + assert.False(t, cfg.IsCollectorEnabled()) + }) + + t.Run("analytics on -> collector on", func(t *testing.T) { + cfg := validConfig() + cfg.Analytics.Enabled = true + cfg.Analytics.EnabledPublishers = []string{} + assert.True(t, cfg.IsCollectorEnabled()) + require.NoError(t, cfg.Validate()) + }) + + t.Run("traffic logging on -> collector on", func(t *testing.T) { + cfg := validConfig() + cfg.TrafficLogging.Enabled = true + assert.True(t, cfg.IsCollectorEnabled()) + require.NoError(t, cfg.Validate()) + }) +} + +func TestValidate_TrafficLoggingMaxPayloadSize(t *testing.T) { + cfg := validConfig() + cfg.TrafficLogging.Enabled = true + cfg.TrafficLogging.MaxPayloadSize = -1 + err := cfg.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "traffic_logging.max_payload_size") +} + // TestValidate_AnalyticsPublishers tests analytics publisher validation func TestValidate_AnalyticsPublishers(t *testing.T) { tests := []struct { @@ -1178,8 +1263,7 @@ func TestValidate_AnalyticsPublishers(t *testing.T) { name: "no publishers enabled", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ - ServerPort: 18090, + cfg.Collector.Server = AccessLogsServiceConfig{ ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, ExtProcMaxHeaderLimit: 8192, @@ -1192,8 +1276,7 @@ func TestValidate_AnalyticsPublishers(t *testing.T) { name: "unknown publisher type", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ - ServerPort: 18090, + cfg.Collector.Server = AccessLogsServiceConfig{ ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, ExtProcMaxHeaderLimit: 8192, @@ -1207,8 +1290,7 @@ func TestValidate_AnalyticsPublishers(t *testing.T) { name: "moesif publisher - missing application_id", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ - ServerPort: 18090, + cfg.Collector.Server = AccessLogsServiceConfig{ ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, ExtProcMaxHeaderLimit: 8192, @@ -1223,8 +1305,7 @@ func TestValidate_AnalyticsPublishers(t *testing.T) { name: "moesif publisher - invalid publish_interval", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ - ServerPort: 18090, + cfg.Collector.Server = AccessLogsServiceConfig{ ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, ExtProcMaxHeaderLimit: 8192, @@ -1240,8 +1321,7 @@ func TestValidate_AnalyticsPublishers(t *testing.T) { name: "moesif publisher - invalid base_url", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ - ServerPort: 18090, + cfg.Collector.Server = AccessLogsServiceConfig{ ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, ExtProcMaxHeaderLimit: 8192, @@ -1258,8 +1338,7 @@ func TestValidate_AnalyticsPublishers(t *testing.T) { name: "moesif publisher - valid config", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ - ServerPort: 18090, + cfg.Collector.Server = AccessLogsServiceConfig{ ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, ExtProcMaxHeaderLimit: 8192, diff --git a/gateway/gateway-runtime/policy-engine/internal/kernel/translator.go b/gateway/gateway-runtime/policy-engine/internal/kernel/translator.go index 8cd30d85a6..5c72e5e478 100644 --- a/gateway/gateway-runtime/policy-engine/internal/kernel/translator.go +++ b/gateway/gateway-runtime/policy-engine/internal/kernel/translator.go @@ -387,7 +387,41 @@ func TranslateRequestHeaderActions(result *executor.RequestHeaderExecutionResult }, }, } - analyticsStruct, err := buildAnalyticsStruct(immResp.AnalyticsMetadata, execCtx) + // Preserve request-header-phase analytics metadata from policies that + // executed before the short-circuit (e.g. the log-message traffic-log + // marker) so an immediate response like a 401 still carries it to the + // ALS access log. Without this, a short-circuiting policy (auth) drops + // the marker of any earlier policy and the traffic-log line is never + // emitted. Mirrors translateRequestActionsCore's short-circuit path. + shortCircuitAnalyticsData := make(map[string]any) + for key, value := range execCtx.analyticsMetadata { + shortCircuitAnalyticsData[key] = value + } + for _, policyResult := range result.Results { + if policyResult.Skipped || policyResult.Action == nil { + continue + } + mods, ok := policyResult.Action.(policy.UpstreamRequestHeaderModifications) + if !ok { + continue + } + if mods.AnalyticsMetadata != nil { + for key, value := range mods.AnalyticsMetadata { + shortCircuitAnalyticsData[key] = value + } + } + dropAction := mods.AnalyticsHeaderFilter + if dropAction.Action != "" || len(dropAction.Headers) > 0 { + originalHeaders := execCtx.requestBodyCtx.Headers.GetAll() + shortCircuitAnalyticsData["request_headers"] = finalizeAnalyticsHeaders(dropAction, originalHeaders) + } + } + if immResp.AnalyticsMetadata != nil { + for key, value := range immResp.AnalyticsMetadata { + shortCircuitAnalyticsData[key] = value + } + } + analyticsStruct, err := buildAnalyticsStruct(shortCircuitAnalyticsData, execCtx) if err != nil { return nil, fmt.Errorf("failed to build analytics metadata for immediate response: %w", err) } diff --git a/gateway/gateway-runtime/policy-engine/internal/kernel/translator_test.go b/gateway/gateway-runtime/policy-engine/internal/kernel/translator_test.go index 669d4e8c18..6aad945e10 100644 --- a/gateway/gateway-runtime/policy-engine/internal/kernel/translator_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/kernel/translator_test.go @@ -608,6 +608,72 @@ func TestTranslateRequestActionsCore_ShortCircuit_PreservesPriorRequestAnalytics assert.Equal(t, "immediate-response", analyticsData.GetFields()["source"].GetStringValue()) } +// TestTranslateRequestHeaderActions_ShortCircuit_PreservesPriorTrafficLogMarker +// covers the request-header phase short-circuit path: the log-message policy +// (traffic-logging mode) stamps its marker as request-header-phase analytics +// metadata, then an auth policy rejects the request with 401 and short-circuits +// the chain. The prior policy's marker must survive onto the immediate response's +// dynamic metadata, otherwise the ALS access-log entry carries no traffic_log key +// and the stdout traffic-log line is never emitted for rejected requests. +func TestTranslateRequestHeaderActions_ShortCircuit_PreservesPriorTrafficLogMarker(t *testing.T) { + kernel := NewKernel() + chainExecutor := executor.NewChainExecutor(nil, nil, nil) + server := NewExternalProcessorServer(kernel, chainExecutor, config.TracingConfig{}, "") + + chain := ®istry.PolicyChain{} + execCtx := newPolicyExecutionContext(server, "test-route", chain) + execCtx.requestBodyCtx = &policy.RequestContext{ + Path: "/api/test", + SharedContext: &policy.SharedContext{ + APIId: "api-1", + }, + } + + result := &executor.RequestHeaderExecutionResult{ + ShortCircuited: true, + Results: []executor.RequestHeaderPolicyResult{ + { + Skipped: false, + Action: policy.UpstreamRequestHeaderModifications{ + AnalyticsMetadata: map[string]any{ + "traffic_log": `{"request":{"headers":true}}`, + }, + }, + }, + }, + FinalAction: policy.ImmediateResponse{ + StatusCode: 401, + Headers: map[string]string{ + "content-type": "application/json", + }, + Body: []byte(`{"error":"unauthorized"}`), + AnalyticsMetadata: map[string]any{ + "source": "immediate-response", + }, + }, + } + + resp, err := TranslateRequestHeaderActions(result, chain, execCtx) + + assert.NoError(t, err) + require.NotNil(t, resp) + + immediate := resp.GetImmediateResponse() + require.NotNil(t, immediate) + assert.Equal(t, uint32(401), uint32(immediate.Status.Code)) + + extProcNamespace := resp.DynamicMetadata.GetFields()[constants.ExtProcFilterName].GetStructValue() + require.NotNil(t, extProcNamespace) + + analyticsData := extProcNamespace.GetFields()["analytics_data"].GetStructValue() + require.NotNil(t, analyticsData) + + // The traffic-log marker stamped by the earlier policy survives the short-circuit... + assert.Equal(t, `{"request":{"headers":true}}`, analyticsData.GetFields()["traffic_log"].GetStringValue()) + // ...and the immediate response's own analytics metadata is still present. + assert.Equal(t, "immediate-response", analyticsData.GetFields()["source"].GetStringValue()) +} + func TestTranslateRequestActionsCore_SkippedPolicy(t *testing.T) { kernel := NewKernel() chainExecutor := executor.NewChainExecutor(nil, nil, nil) diff --git a/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server.go b/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server.go index 87049d7d8a..e9715f7f9c 100644 --- a/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server.go +++ b/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server.go @@ -27,6 +27,7 @@ import ( "time" v3 "github.com/envoyproxy/go-control-plane/envoy/service/accesslog/v3" + "github.com/wso2/api-platform/common/collector" "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/analytics" "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/config" "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/constants" @@ -90,15 +91,15 @@ func StartAccessLogServiceServer(cfg *config.Config) *grpc.Server { Timeout: 20 * time.Second, } maxHeaderListSize, err := checkedUInt32FromPositiveInt( - "analytics.access_logs_service.max_header_limit", - cfg.Analytics.AccessLogsServiceCfg.ExtProcMaxHeaderLimit, + "collector.server.max_header_limit", + cfg.Collector.Server.ExtProcMaxHeaderLimit, ) if err != nil { panic(err) } - server, err := CreateGRPCServer(cfg.Analytics.AccessLogsServiceCfg.PublicKeyPath, - cfg.Analytics.AccessLogsServiceCfg.PrivateKeyPath, cfg.Analytics.AccessLogsServiceCfg.ALSPlainText, - grpc.MaxRecvMsgSize(cfg.Analytics.AccessLogsServiceCfg.ExtProcMaxMessageSize), + server, err := CreateGRPCServer(cfg.Collector.Server.PublicKeyPath, + cfg.Collector.Server.PrivateKeyPath, cfg.Collector.Server.ALSPlainText, + grpc.MaxRecvMsgSize(cfg.Collector.Server.ExtProcMaxMessageSize), grpc.MaxHeaderListSize(maxHeaderListSize), grpc.KeepaliveParams(kaParams)) if err != nil { @@ -109,7 +110,7 @@ func StartAccessLogServiceServer(cfg *config.Config) *grpc.Server { // Create listener based on mode (same pattern as ext_proc in main.go) var listener net.Listener - alsMode := cfg.Analytics.AccessLogsServiceCfg.Mode + alsMode := cfg.Collector.Server.Mode if alsMode == "" { alsMode = "uds" } @@ -139,14 +140,20 @@ func StartAccessLogServiceServer(cfg *config.Config) *grpc.Server { } }() case "tcp": - listener, err = net.Listen("tcp", fmt.Sprintf(":%d", cfg.Analytics.AccessLogsServiceCfg.ServerPort)) + // cfg.Collector.Server.ServerPort is a deprecated override (see its doc comment); + // the fixed collector.ServerPort constant is used unless a config already sets it. + port := collector.ServerPort + if cfg.Collector.Server.ServerPort != 0 { + port = cfg.Collector.Server.ServerPort + } + listener, err = net.Listen("tcp", fmt.Sprintf(":%d", port)) if err != nil { - slog.Error("Failed to listen on ALS TCP port", "port", cfg.Analytics.AccessLogsServiceCfg.ServerPort) + slog.Error("Failed to listen on ALS TCP port", "port", port) panic(err) } go func() { - slog.Info("Starting to serve access log service server", "mode", "tcp", "port", cfg.Analytics.AccessLogsServiceCfg.ServerPort) + slog.Info("Starting to serve access log service server", "mode", "tcp", "port", port) if err := server.Serve(listener); err != nil { slog.Error("ALS server exited", "error", err) } diff --git a/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server_test.go b/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server_test.go index fd6dba08d7..c0ed6a3db1 100644 --- a/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server_test.go @@ -210,13 +210,10 @@ func TestStreamAccessLogs_MultipleMessages(t *testing.T) { func TestStartAccessLogServiceServer_TCP(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ - Enabled: false, - EnabledPublishers: []string{}, - Publishers: config.AnalyticsPublishersConfig{}, - AccessLogsServiceCfg: config.AccessLogsServiceConfig{ + Collector: config.CollectorConfig{ + Server: config.AccessLogsServiceConfig{ Mode: "tcp", - ServerPort: 19001, // Use non-standard port to avoid conflicts + ServerPort: 19001, // Deprecated override; also exercises backward compat ALSPlainText: true, ExtProcMaxMessageSize: 1024 * 1024 * 4, ExtProcMaxHeaderLimit: 8192, diff --git a/gateway/system-policies/analytics/analytics.go b/gateway/system-policies/analytics/analytics.go index 261cef76cd..c197a991b8 100644 --- a/gateway/system-policies/analytics/analytics.go +++ b/gateway/system-policies/analytics/analytics.go @@ -152,6 +152,14 @@ func (a *AnalyticsPolicy) OnRequestHeaders(_ context.Context, reqCtx *policy.Req } } + // Capture all request headers when enabled, so they flow into analytics events + // (and the stdout/log publisher) without attaching a per-API header policy. + if sendReqHeaders, _ := getHeaderFlags(params); sendReqHeaders && reqCtx.Headers != nil { + if headers := serializeHeaders(reqCtx.Headers); headers != "" { + analyticsMetadata["request_headers"] = headers + } + } + if len(analyticsMetadata) > 0 { return policy.UpstreamRequestHeaderModifications{AnalyticsMetadata: analyticsMetadata} } @@ -207,6 +215,13 @@ func (a *AnalyticsPolicy) OnResponseHeaders(_ context.Context, respCtx *policy.R } } + // Capture all response headers when enabled. + if _, sendRespHeaders := getHeaderFlags(params); sendRespHeaders && respCtx.ResponseHeaders != nil { + if headers := serializeHeaders(respCtx.ResponseHeaders); headers != "" { + analyticsMetadata["response_headers"] = headers + } + } + if len(analyticsMetadata) > 0 { return policy.DownstreamResponseHeaderModifications{AnalyticsMetadata: analyticsMetadata} } @@ -890,7 +905,7 @@ func convertToInt64(value interface{}) (int64, error) { } // getPayloadFlags derives per-direction payload capture flags from policy parameters. -// New parameters send_request_body and send_response_body take precedence. When neither +// New parameters request_body and response_body take precedence. When neither // is provided, the deprecated allow_payloads flag is used as a fallback, mapping to // both directions for backward compatibility. func getPayloadFlags(params map[string]interface{}) (sendRequestBody, sendResponseBody bool) { @@ -898,25 +913,13 @@ func getPayloadFlags(params map[string]interface{}) (sendRequestBody, sendRespon return false, false } - parseBoolLike := func(v interface{}) bool { - switch val := v.(type) { - case bool: - return val - case string: - lower := strings.ToLower(strings.TrimSpace(val)) - return lower == "true" || lower == "1" || lower == "yes" - default: - return false - } - } - hasReq, hasResp := false, false - if raw, ok := params["send_request_body"]; ok { + if raw, ok := params["request_body"]; ok { sendRequestBody = parseBoolLike(raw) hasReq = true } - if raw, ok := params["send_response_body"]; ok { + if raw, ok := params["response_body"]; ok { sendResponseBody = parseBoolLike(raw) hasResp = true } @@ -937,6 +940,55 @@ func getPayloadFlags(params map[string]interface{}) (sendRequestBody, sendRespon return false, false } +// parseBoolLike interprets bool and common string ("true"/"1"/"yes") representations +// of a boolean policy parameter. +func parseBoolLike(v interface{}) bool { + switch val := v.(type) { + case bool: + return val + case string: + lower := strings.ToLower(strings.TrimSpace(val)) + return lower == "true" || lower == "1" || lower == "yes" + default: + return false + } +} + +// getHeaderFlags derives per-direction header capture flags from policy parameters. +func getHeaderFlags(params map[string]interface{}) (sendRequestHeaders, sendResponseHeaders bool) { + if params == nil { + return false, false + } + if raw, ok := params["request_headers"]; ok { + sendRequestHeaders = parseBoolLike(raw) + } + if raw, ok := params["response_headers"]; ok { + sendResponseHeaders = parseBoolLike(raw) + } + return sendRequestHeaders, sendResponseHeaders +} + +// serializeHeaders renders all headers as a JSON object string ({"name":"v1, v2"}), +// matching the request_headers/response_headers format the analytics engine reads. +// Returns "" when there are no headers. Sensitive values are not masked here; the +// stdout/log publisher applies masked_headers on output. +func serializeHeaders(headers *policy.Headers) string { + all := headers.GetAll() + if len(all) == 0 { + return "" + } + flat := make(map[string]string, len(all)) + for name, values := range all { + flat[name] = strings.Join(values, ", ") + } + data, err := json.Marshal(flat) + if err != nil { + slog.Error("Failed to marshal headers for analytics", "error", err) + return "" + } + return string(data) +} + // Helper to extract string values via JSONPath func extractStringFromJsonpath(payload map[string]interface{}, path string) string { val, err := utils.ExtractValueFromJsonpath(payload, path) diff --git a/gateway/system-policies/analytics/analytics_headers_test.go b/gateway/system-policies/analytics/analytics_headers_test.go new file mode 100644 index 0000000000..25a3b645cb --- /dev/null +++ b/gateway/system-policies/analytics/analytics_headers_test.go @@ -0,0 +1,63 @@ +package analytics + +import ( + "encoding/json" + "testing" + + policy "github.com/wso2/api-platform/sdk/core/policy/v1alpha2" +) + +func TestGetHeaderFlags(t *testing.T) { + cases := []struct { + name string + params map[string]interface{} + wantReq bool + wantResp bool + }{ + {"nil params", nil, false, false}, + {"absent", map[string]interface{}{}, false, false}, + {"bool true", map[string]interface{}{"request_headers": true, "response_headers": true}, true, true}, + {"string true", map[string]interface{}{"request_headers": "true"}, true, false}, + {"mixed", map[string]interface{}{"request_headers": false, "response_headers": "yes"}, false, true}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + gotReq, gotResp := getHeaderFlags(c.params) + if gotReq != c.wantReq || gotResp != c.wantResp { + t.Fatalf("getHeaderFlags(%v) = (%v, %v), want (%v, %v)", c.params, gotReq, gotResp, c.wantReq, c.wantResp) + } + }) + } +} + +func TestSerializeHeaders(t *testing.T) { + // Empty headers -> empty string. + if got := serializeHeaders(policy.NewHeaders(nil)); got != "" { + t.Fatalf("serializeHeaders(empty) = %q, want \"\"", got) + } + + h := policy.NewHeaders(map[string][]string{ + "Authorization": {"Bearer secret"}, + "X-Foo": {"a", "b"}, + }) + got := serializeHeaders(h) + if got == "" { + t.Fatal("serializeHeaders returned empty for non-empty headers") + } + + var decoded map[string]string + if err := json.Unmarshal([]byte(got), &decoded); err != nil { + t.Fatalf("output is not valid JSON: %v (%q)", err, got) + } + // NewHeaders lower-cases keys; multi-value headers are joined with ", ". + if decoded["authorization"] != "Bearer secret" { + t.Errorf("authorization = %q, want %q", decoded["authorization"], "Bearer secret") + } + if decoded["x-foo"] != "a, b" { + t.Errorf("x-foo = %q, want %q", decoded["x-foo"], "a, b") + } +} + +// Note: payload-size capping moved out of the capture path (the collector now +// captures full bodies); truncation is applied output-side by the traffic-logging +// publisher (traffic_logging.max_payload_size) and tested there. diff --git a/gateway/system-policies/analytics/policy-definition.yaml b/gateway/system-policies/analytics/policy-definition.yaml index 7fe915469d..8969448c27 100644 --- a/gateway/system-policies/analytics/policy-definition.yaml +++ b/gateway/system-policies/analytics/policy-definition.yaml @@ -10,14 +10,14 @@ parameters: type: object additionalProperties: false properties: - send_request_body: + request_body: type: boolean default: false description: > When true, the analytics system policy will capture the request body into analytics metadata as request_payload, which is then forwarded through the analytics pipeline (including publishers like Moesif). - send_response_body: + response_body: type: boolean default: false description: > @@ -25,14 +25,30 @@ parameters: into analytics metadata as response_payload, for both buffered and streaming responses, which is then forwarded through the analytics pipeline (including publishers like Moesif). + request_headers: + type: boolean + default: false + description: > + When true, the analytics system policy captures ALL request headers + into analytics metadata as request_headers, which is forwarded through + the analytics pipeline (including the stdout/log and Moesif publishers). + Sensitive values are not masked here; the log publisher's masked_headers + setting redacts them on output. + response_headers: + type: boolean + default: false + description: > + When true, the analytics system policy captures ALL response headers + into analytics metadata as response_headers, which is forwarded through + the analytics pipeline (including the stdout/log and Moesif publishers). allow_payloads: type: boolean default: false description: > - Deprecated: use send_request_body and send_response_body instead. - When true, and when send_request_body and send_response_body are not + Deprecated: use request_body and response_body instead. + When true, and when request_body and response_body are not explicitly set, the analytics system policy behaves as if both - send_request_body and send_response_body are true, capturing request + request_body and response_body are true, capturing request and response bodies into analytics metadata as request_payload and response_payload, which are then forwarded through the analytics pipeline (including publishers like Moesif). When false, payloads are diff --git a/kubernetes/helm/gateway-helm-chart/templates/gateway/gateway-config.yaml b/kubernetes/helm/gateway-helm-chart/templates/gateway/gateway-config.yaml index a90894b5cf..6d42baf607 100644 --- a/kubernetes/helm/gateway-helm-chart/templates/gateway/gateway-config.yaml +++ b/kubernetes/helm/gateway-helm-chart/templates/gateway/gateway-config.yaml @@ -256,6 +256,33 @@ data: {{- end }} {{- end }} + {{- if .Values.gateway.config.collector }} + {{- if .Values.gateway.config.collector.server }} + [collector.server] + buffer_flush_interval = {{ .Values.gateway.config.collector.server.buffer_flush_interval | quote }} + buffer_size_bytes = {{ .Values.gateway.config.collector.server.buffer_size_bytes | int64 }} + grpc_request_timeout = {{ .Values.gateway.config.collector.server.grpc_request_timeout | quote }} + shutdown_timeout = {{ .Values.gateway.config.collector.server.shutdown_timeout | quote }} + public_key_path = {{ .Values.gateway.config.collector.server.public_key_path | quote }} + private_key_path = {{ .Values.gateway.config.collector.server.private_key_path | quote }} + als_plain_text = {{ .Values.gateway.config.collector.server.als_plain_text }} + max_message_size = {{ .Values.gateway.config.collector.server.max_message_size | int64 }} + max_header_limit = {{ .Values.gateway.config.collector.server.max_header_limit | int64 }} + {{- end }} + {{- end }} + + {{- if .Values.gateway.config.traffic_logging }} + {{- $trafficLoggingEnabled := and (kindIs "bool" .Values.gateway.config.traffic_logging.enabled) .Values.gateway.config.traffic_logging.enabled }} + [traffic_logging] + enabled = {{ $trafficLoggingEnabled }} + {{- if .Values.gateway.config.traffic_logging.masked_headers }} + masked_headers = [{{- range $i, $h := .Values.gateway.config.traffic_logging.masked_headers }}{{- if gt $i 0 }}, {{ end }}{{ $h | quote }}{{- end }}] + {{- end }} + {{- if .Values.gateway.config.traffic_logging.max_payload_size }} + max_payload_size = {{ .Values.gateway.config.traffic_logging.max_payload_size | int64 }} + {{- end }} + {{- end }} + {{- if .Values.gateway.config.tracing }} [tracing] enabled = {{ .Values.gateway.config.tracing.enabled }}