diff --git a/api/v1alpha1/envoyproxy_metric_types.go b/api/v1alpha1/envoyproxy_metric_types.go index 82e75ec1a52..d386caec0d2 100644 --- a/api/v1alpha1/envoyproxy_metric_types.go +++ b/api/v1alpha1/envoyproxy_metric_types.go @@ -122,6 +122,11 @@ type ProxyOpenTelemetrySink struct { // // +optional ReportHistogramsAsDeltas *bool `json:"reportHistogramsAsDeltas,omitempty"` + // Prefix configures the OpenTelemetry sink to prepend the given prefix to emitted stat names, + // the full stat name will be `.`. + // + // +optional + Prefix *string `json:"prefix,omitempty"` // Headers is a list of additional headers to send with OTLP export requests. // These headers are added as gRPC initial metadata for the OTLP gRPC service. // +optional diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 7a998137dbb..8e8d414851a 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -7358,6 +7358,11 @@ func (in *ProxyOpenTelemetrySink) DeepCopyInto(out *ProxyOpenTelemetrySink) { *out = new(bool) **out = **in } + if in.Prefix != nil { + in, out := &in.Prefix, &out.Prefix + *out = new(string) + **out = **in + } if in.Headers != nil { in, out := &in.Headers, &out.Headers *out = make([]v1.HTTPHeader, len(*in)) diff --git a/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml index a9630d033a6..660f95e61e5 100644 --- a/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -17085,6 +17085,11 @@ spec: maximum: 65535 minimum: 0 type: integer + prefix: + description: |- + Prefix configures the OpenTelemetry sink to prepend the given prefix to emitted stat names, + the full stat name will be `.`. + type: string reportCountersAsDeltas: description: |- ReportCountersAsDeltas configures the OpenTelemetry sink to report diff --git a/charts/gateway-helm/charts/crds/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-helm/charts/crds/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml index 76159da3d20..32b6204579b 100644 --- a/charts/gateway-helm/charts/crds/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-helm/charts/crds/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -17084,6 +17084,11 @@ spec: maximum: 65535 minimum: 0 type: integer + prefix: + description: |- + Prefix configures the OpenTelemetry sink to prepend the given prefix to emitted stat names, + the full stat name will be `.`. + type: string reportCountersAsDeltas: description: |- ReportCountersAsDeltas configures the OpenTelemetry sink to report diff --git a/internal/gatewayapi/listener.go b/internal/gatewayapi/listener.go index 3d8376553ad..9652229b3df 100644 --- a/internal/gatewayapi/listener.go +++ b/internal/gatewayapi/listener.go @@ -1303,6 +1303,7 @@ func (t *Translator) processMetrics(gwCtx *GatewayContext, envoyproxy *egv1a1.En ResourceAttributes: sink.OpenTelemetry.ResourceAttributes, ReportCountersAsDeltas: ptr.Deref(sink.OpenTelemetry.ReportCountersAsDeltas, false), ReportHistogramsAsDeltas: ptr.Deref(sink.OpenTelemetry.ReportHistogramsAsDeltas, false), + Prefix: ptr.Deref(sink.OpenTelemetry.Prefix, ""), }) } } diff --git a/internal/infrastructure/common/proxy_metrics.go b/internal/infrastructure/common/proxy_metrics.go index 93360a9abfd..d1248b1ec1e 100644 --- a/internal/infrastructure/common/proxy_metrics.go +++ b/internal/infrastructure/common/proxy_metrics.go @@ -26,6 +26,7 @@ func ConvertResolvedMetricSinks(irSinks []ir.ResolvedMetricSink) []bootstrap.Met Authority: sink.Authority, ReportCountersAsDeltas: sink.ReportCountersAsDeltas, ReportHistogramsAsDeltas: sink.ReportHistogramsAsDeltas, + Prefix: sink.Prefix, Headers: sink.Headers, ResourceAttributes: sink.ResourceAttributes, } diff --git a/internal/infrastructure/common/proxy_metrics_test.go b/internal/infrastructure/common/proxy_metrics_test.go index 549998d7159..3032a7dcd7b 100644 --- a/internal/infrastructure/common/proxy_metrics_test.go +++ b/internal/infrastructure/common/proxy_metrics_test.go @@ -166,6 +166,31 @@ func TestResolvedMetricSinksConversion(t *testing.T) { }, }, }, + { + name: "sink with prefix", + irSinks: []ir.ResolvedMetricSink{ + { + Destination: ir.RouteDestination{ + Name: "metrics_otel_0", + Settings: []*ir.DestinationSetting{ + { + Endpoints: []*ir.DestinationEndpoint{ + {Host: "otel-collector.example.com", Port: 4317}, + }, + }, + }, + }, + Prefix: "envoy", + }, + }, + expected: []bootstrap.MetricSink{ + { + Address: "otel-collector.example.com", + Port: 4317, + Prefix: "envoy", + }, + }, + }, { name: "sink with resources", irSinks: []ir.ResolvedMetricSink{ diff --git a/internal/infrastructure/remote/conversion.go b/internal/infrastructure/remote/conversion.go index 482d476be07..fc3a855fb8e 100644 --- a/internal/infrastructure/remote/conversion.go +++ b/internal/infrastructure/remote/conversion.go @@ -140,7 +140,7 @@ func proxyListenerToProto(l *ir.ProxyListener) *remoteinfra.ProxyListener { // resolvedMetricSinkToProto translates every field of ir.ResolvedMetricSink // (Authority, Headers, ResourceAttributes, ReportCountersAsDeltas, -// ReportHistogramsAsDeltas). Only its Destination is narrowed to a subset; see +// ReportHistogramsAsDeltas, Prefix). Only its Destination is narrowed to a subset; see // routeDestinationToProto. func resolvedMetricSinkToProto(s *ir.ResolvedMetricSink) *remoteinfra.ResolvedMetricSink { out := &remoteinfra.ResolvedMetricSink{ @@ -149,6 +149,7 @@ func resolvedMetricSinkToProto(s *ir.ResolvedMetricSink) *remoteinfra.ResolvedMe ResourceAttributes: s.ResourceAttributes, ReportCountersAsDeltas: s.ReportCountersAsDeltas, ReportHistogramsAsDeltas: s.ReportHistogramsAsDeltas, + Prefix: s.Prefix, } for _, h := range s.Headers { @@ -376,6 +377,7 @@ func protoToResolvedMetricSink(s *remoteinfra.ResolvedMetricSink) ir.ResolvedMet ResourceAttributes: s.GetResourceAttributes(), ReportCountersAsDeltas: s.GetReportCountersAsDeltas(), ReportHistogramsAsDeltas: s.GetReportHistogramsAsDeltas(), + Prefix: s.GetPrefix(), } if s.GetDestination() != nil { diff --git a/internal/infrastructure/remote/conversion_test.go b/internal/infrastructure/remote/conversion_test.go index 1a3d910caed..e9b6688eeb7 100644 --- a/internal/infrastructure/remote/conversion_test.go +++ b/internal/infrastructure/remote/conversion_test.go @@ -109,6 +109,7 @@ func fullyPopulatedInfra() *ir.Infra { ResourceAttributes: map[string]string{"service.name": "eg"}, ReportCountersAsDeltas: true, ReportHistogramsAsDeltas: true, + Prefix: "envoy", Headers: []gwapiv1.HTTPHeader{ {Name: "x-header", Value: "header-value"}, }, diff --git a/internal/ir/infra.go b/internal/ir/infra.go index 5aff6c9aaae..bcc40d7640b 100644 --- a/internal/ir/infra.go +++ b/internal/ir/infra.go @@ -76,6 +76,8 @@ type ResolvedMetricSink struct { ReportCountersAsDeltas bool `json:"reportCountersAsDeltas,omitempty" yaml:"reportCountersAsDeltas,omitempty"` // ReportHistogramsAsDeltas configures histograms to use delta temporality. ReportHistogramsAsDeltas bool `json:"reportHistogramsAsDeltas,omitempty" yaml:"reportHistogramsAsDeltas,omitempty"` + // Prefix is prepended to emitted stat names, the full stat name will be `.`. + Prefix string `json:"prefix,omitempty" yaml:"prefix,omitempty"` } // InfraMetadata defines metadata for the managed proxy infrastructure. diff --git a/internal/xds/bootstrap/bootstrap.go b/internal/xds/bootstrap/bootstrap.go index 66c315f1e22..fa758eeb806 100644 --- a/internal/xds/bootstrap/bootstrap.go +++ b/internal/xds/bootstrap/bootstrap.go @@ -136,6 +136,9 @@ type MetricSink struct { ReportCountersAsDeltas bool // ReportHistogramsAsDeltas configures histograms to use delta temporality. ReportHistogramsAsDeltas bool + // Prefix is prepended to emitted stat names, the full stat name will be `.`. + // If empty, no prefix is added. + Prefix string // Headers is a list of headers to send with OTLP export requests. Headers []gwapiv1.HTTPHeader // ResourceAttributes is a map of resource attributes for the metrics sink. @@ -269,6 +272,7 @@ func GetRenderedBootstrapConfig(opts *RenderBootstrapConfigOptions) (string, err Port: port, ReportCountersAsDeltas: ptr.Deref(sink.OpenTelemetry.ReportCountersAsDeltas, false), ReportHistogramsAsDeltas: ptr.Deref(sink.OpenTelemetry.ReportHistogramsAsDeltas, false), + Prefix: ptr.Deref(sink.OpenTelemetry.Prefix, ""), Headers: sink.OpenTelemetry.Headers, }) } diff --git a/internal/xds/bootstrap/bootstrap.yaml.tpl b/internal/xds/bootstrap/bootstrap.yaml.tpl index 2a69fea32eb..9aba6a787c7 100644 --- a/internal/xds/bootstrap/bootstrap.yaml.tpl +++ b/internal/xds/bootstrap/bootstrap.yaml.tpl @@ -96,6 +96,9 @@ stats_sinks: {{- if $sink.ReportHistogramsAsDeltas }} report_histograms_as_deltas: true {{- end }} + {{- if $sink.Prefix }} + prefix: '{{ $sink.Prefix }}' + {{- end}} {{- if $sink.ResourceAttributes }} resource_detectors: - name: envoy.tracers.opentelemetry.resource_detectors.static_config diff --git a/internal/xds/bootstrap/bootstrap_test.go b/internal/xds/bootstrap/bootstrap_test.go index 9904fd623e6..37d4fa7f521 100644 --- a/internal/xds/bootstrap/bootstrap_test.go +++ b/internal/xds/bootstrap/bootstrap_test.go @@ -161,6 +161,28 @@ func TestGetRenderedBootstrapConfig(t *testing.T) { SdsConfig: sds, }, }, + { + name: "otel-metrics-prefix", + opts: &RenderBootstrapConfigOptions{ + ProxyMetrics: &egv1a1.ProxyMetrics{ + Prometheus: &egv1a1.ProxyPrometheusProvider{ + Disable: true, + }, + Sinks: []egv1a1.ProxyMetricSink{ + { + Type: egv1a1.MetricSinkTypeOpenTelemetry, + OpenTelemetry: &egv1a1.ProxyOpenTelemetrySink{ + Host: new("otel-collector.monitoring.svc"), + Port: 4317, + ReportHistogramsAsDeltas: new(true), + Prefix: new("envoy"), + }, + }, + }, + }, + SdsConfig: sds, + }, + }, { name: "otel-metrics-headers", opts: &RenderBootstrapConfigOptions{ diff --git a/internal/xds/bootstrap/testdata/render/otel-metrics-prefix.yaml b/internal/xds/bootstrap/testdata/render/otel-metrics-prefix.yaml new file mode 100644 index 00000000000..399dec375f9 --- /dev/null +++ b/internal/xds/bootstrap/testdata/render/otel-metrics-prefix.yaml @@ -0,0 +1,146 @@ +admin: + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 +cluster_manager: + local_cluster_name: local_cluster +node: + locality: + zone: $(ENVOY_SERVICE_ZONE) +deferredStatOptions: + enableDeferredCreationStats: true +stats_config: + use_all_default_tags: true + stats_tags: + - regex: \.zone(\.(([^\.]+)\.)) + tag_name: from_zone + - regex: \.zone\.[^\.]+\.(([^\.]+)\.) + tag_name: to_zone + - regex: "^cluster(\\..+\\.(.+))\\.total_match_count$" + tag_name: socket_match_name + - regex: "circuit_breakers\\.((.+?)\\.).+" + tag_name: priority +layered_runtime: + layers: + - name: global_config + static_layer: + envoy.reloadable_features.oauth2_legacy_cbc_decrypt_compat: false + envoy.reloadable_features.oauth2_use_gcm_encryption: true + re2.max_program_size.error_level: 4294967295 + re2.max_program_size.warn_level: 1000 +dynamic_resources: + ads_config: + api_type: DELTA_GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: xds_cluster + set_node_on_first_message_only: true + lds_config: + ads: {} + initial_fetch_timeout: 0s + resource_api_version: V3 + cds_config: + ads: {} + initial_fetch_timeout: 0s + resource_api_version: V3 +stats_sinks: +- name: "envoy.stat_sinks.open_telemetry" + typed_config: + "@type": type.googleapis.com/envoy.extensions.stat_sinks.open_telemetry.v3.SinkConfig + grpc_service: + envoy_grpc: + cluster_name: otel_metric_sink_0 + report_histograms_as_deltas: true + prefix: 'envoy' +static_resources: + clusters: + - name: otel_metric_sink_0 + connect_timeout: 0.250s + type: STRICT_DNS + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: otel_metric_sink_0 + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: otel-collector.monitoring.svc + port_value: 4317 + - connect_timeout: 10s + eds_cluster_config: + eds_config: + ads: {} + resource_api_version: 'V3' + service_name: local_cluster + load_balancing_policy: + policies: + - typed_extension_config: + name: 'envoy.load_balancing_policies.least_request' + typed_config: + '@type': 'type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest' + locality_lb_config: + zone_aware_lb_config: + min_cluster_size: '1' + name: local_cluster + type: EDS + - connect_timeout: 10s + load_assignment: + cluster_name: xds_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18000 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: + connection_keepalive: + interval: 30s + timeout: 5s + name: xds_cluster + type: STRICT_DNS + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: /sds/xds-certificate.json + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: /sds/xds-trusted-ca.json + resource_api_version: V3 +overload_manager: + refresh_interval: 0.25s + resource_monitors: + - name: "envoy.resource_monitors.global_downstream_max_connections" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig + max_active_downstream_connections: 50000 diff --git a/proto/remoteinfra/service.pb.go b/proto/remoteinfra/service.pb.go index 67e1c1e97b9..a4837c20657 100644 --- a/proto/remoteinfra/service.pb.go +++ b/proto/remoteinfra/service.pb.go @@ -815,8 +815,10 @@ type ResolvedMetricSink struct { ReportCountersAsDeltas bool `protobuf:"varint,5,opt,name=report_counters_as_deltas,json=reportCountersAsDeltas,proto3" json:"report_counters_as_deltas,omitempty"` // report_histograms_as_deltas configures histograms to use delta temporality. ReportHistogramsAsDeltas bool `protobuf:"varint,6,opt,name=report_histograms_as_deltas,json=reportHistogramsAsDeltas,proto3" json:"report_histograms_as_deltas,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // prefix is prepended to emitted stat names, the full name will be `.`. + Prefix string `protobuf:"bytes,7,opt,name=prefix,proto3" json:"prefix,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ResolvedMetricSink) Reset() { @@ -891,6 +893,13 @@ func (x *ResolvedMetricSink) GetReportHistogramsAsDeltas() bool { return false } +func (x *ResolvedMetricSink) GetPrefix() string { + if x != nil { + return x.Prefix + } + return "" +} + // HTTPHeader is a name/value HTTP header. It mirrors the subset of // gwapiv1.HTTPHeader used by ResolvedMetricSink. type HTTPHeader struct { @@ -1529,7 +1538,7 @@ var file_proto_remoteinfra_service_proto_rawDesc = string([]byte{ 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x50, 0x6f, 0x72, 0x74, - 0x22, 0xf8, 0x03, 0x0a, 0x12, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x4d, 0x65, 0x74, + 0x22, 0x90, 0x04, 0x0a, 0x12, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x53, 0x69, 0x6e, 0x6b, 0x12, 0x4c, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x72, 0x65, 0x6d, 0x6f, @@ -1556,104 +1565,106 @@ var file_proto_remoteinfra_service_proto_rawDesc = string([]byte{ 0x68, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x73, 0x5f, 0x61, 0x73, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x18, 0x72, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x73, 0x41, 0x73, 0x44, 0x65, - 0x6c, 0x74, 0x61, 0x73, 0x1a, 0x45, 0x0a, 0x17, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, - 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, - 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x36, 0x0a, 0x0a, 0x48, - 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x22, 0x5c, 0x0a, 0x10, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x44, 0x65, 0x73, 0x74, - 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x48, 0x0a, 0x08, 0x73, 0x65, 0x74, 0x74, 0x69, - 0x6e, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x65, 0x6e, 0x76, 0x6f, - 0x79, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x69, - 0x6e, 0x66, 0x72, 0x61, 0x2e, 0x44, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x52, 0x08, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, - 0x73, 0x22, 0xa0, 0x01, 0x0a, 0x12, 0x44, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x12, 0x4b, 0x0a, 0x09, 0x65, 0x6e, 0x64, 0x70, - 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x65, 0x6e, - 0x76, 0x6f, 0x79, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, - 0x65, 0x69, 0x6e, 0x66, 0x72, 0x61, 0x2e, 0x44, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x09, 0x65, 0x6e, 0x64, 0x70, - 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x12, 0x3d, 0x0a, 0x03, 0x74, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, - 0x79, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x69, 0x6e, 0x66, 0x72, 0x61, 0x2e, 0x54, 0x4c, - 0x53, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, - 0x03, 0x74, 0x6c, 0x73, 0x22, 0x3d, 0x0a, 0x13, 0x44, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x68, - 0x6f, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, - 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x70, - 0x6f, 0x72, 0x74, 0x22, 0xab, 0x01, 0x0a, 0x11, 0x54, 0x4c, 0x53, 0x55, 0x70, 0x73, 0x74, 0x72, - 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x15, 0x0a, 0x03, 0x73, 0x6e, 0x69, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x03, 0x73, 0x6e, 0x69, 0x88, 0x01, 0x01, - 0x12, 0x33, 0x0a, 0x16, 0x75, 0x73, 0x65, 0x5f, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x5f, 0x74, - 0x72, 0x75, 0x73, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x13, 0x75, 0x73, 0x65, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x54, 0x72, 0x75, 0x73, 0x74, - 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x42, 0x0a, 0x0a, 0x74, 0x6c, 0x73, 0x5f, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x65, 0x6e, 0x76, 0x6f, - 0x79, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x69, - 0x6e, 0x66, 0x72, 0x61, 0x2e, 0x54, 0x4c, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x09, - 0x74, 0x6c, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x42, 0x06, 0x0a, 0x04, 0x5f, 0x73, 0x6e, - 0x69, 0x22, 0x5e, 0x0a, 0x09, 0x54, 0x4c, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x51, - 0x0a, 0x0e, 0x63, 0x61, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x67, 0x61, - 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x69, 0x6e, 0x66, 0x72, - 0x61, 0x2e, 0x54, 0x4c, 0x53, 0x43, 0x41, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, - 0x74, 0x65, 0x52, 0x0d, 0x63, 0x61, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, - 0x65, 0x22, 0x34, 0x0a, 0x10, 0x54, 0x4c, 0x53, 0x43, 0x41, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, - 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x63, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, - 0x63, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x65, 0x72, 0x74, - 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x22, 0x25, 0x0a, 0x23, 0x43, 0x72, 0x65, 0x61, 0x74, - 0x65, 0x4f, 0x72, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, - 0x69, 0x74, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x26, - 0x0a, 0x24, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4f, 0x72, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x1d, 0x0a, 0x1b, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, - 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x1e, 0x0a, 0x1c, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, - 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0xe9, 0x04, 0x0a, 0x28, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x47, - 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x6e, 0x66, 0x72, - 0x61, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x75, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, - 0x65, 0x72, 0x12, 0x93, 0x01, 0x0a, 0x18, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4f, 0x72, 0x55, - 0x70, 0x64, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x12, - 0x39, 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x72, - 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x69, 0x6e, 0x66, 0x72, 0x61, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, - 0x65, 0x4f, 0x72, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x49, 0x6e, - 0x66, 0x72, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x3a, 0x2e, 0x65, 0x6e, 0x76, + 0x6c, 0x74, 0x61, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x07, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x1a, 0x45, 0x0a, 0x17, + 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, + 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, + 0x02, 0x38, 0x01, 0x22, 0x36, 0x0a, 0x0a, 0x48, 0x54, 0x54, 0x50, 0x48, 0x65, 0x61, 0x64, 0x65, + 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x5c, 0x0a, 0x10, 0x52, + 0x6f, 0x75, 0x74, 0x65, 0x44, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, + 0x48, 0x0a, 0x08, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x2c, 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, + 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x69, 0x6e, 0x66, 0x72, 0x61, 0x2e, 0x44, 0x65, 0x73, + 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x52, + 0x08, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x22, 0xa0, 0x01, 0x0a, 0x12, 0x44, 0x65, + 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, + 0x12, 0x4b, 0x0a, 0x09, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x67, 0x61, 0x74, 0x65, 0x77, + 0x61, 0x79, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x69, 0x6e, 0x66, 0x72, 0x61, 0x2e, 0x44, + 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, + 0x6e, 0x74, 0x52, 0x09, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x12, 0x3d, 0x0a, + 0x03, 0x74, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, - 0x69, 0x6e, 0x66, 0x72, 0x61, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4f, 0x72, 0x55, 0x70, - 0x64, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x7b, 0x0a, 0x10, 0x44, 0x65, 0x6c, 0x65, - 0x74, 0x65, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x12, 0x31, 0x2e, 0x65, - 0x6e, 0x76, 0x6f, 0x79, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x72, 0x65, 0x6d, 0x6f, - 0x74, 0x65, 0x69, 0x6e, 0x66, 0x72, 0x61, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x50, 0x72, - 0x6f, 0x78, 0x79, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x32, 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x72, - 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x69, 0x6e, 0x66, 0x72, 0x61, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, - 0x65, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x9f, 0x01, 0x0a, 0x1c, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x69, 0x6e, 0x66, 0x72, 0x61, 0x2e, 0x54, 0x4c, 0x53, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, + 0x6d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x03, 0x74, 0x6c, 0x73, 0x22, 0x3d, 0x0a, 0x13, + 0x44, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, + 0x69, 0x6e, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x22, 0xab, 0x01, 0x0a, 0x11, + 0x54, 0x4c, 0x53, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x12, 0x15, 0x0a, 0x03, 0x73, 0x6e, 0x69, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, + 0x52, 0x03, 0x73, 0x6e, 0x69, 0x88, 0x01, 0x01, 0x12, 0x33, 0x0a, 0x16, 0x75, 0x73, 0x65, 0x5f, + 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x5f, 0x74, 0x72, 0x75, 0x73, 0x74, 0x5f, 0x73, 0x74, 0x6f, + 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x75, 0x73, 0x65, 0x53, 0x79, 0x73, + 0x74, 0x65, 0x6d, 0x54, 0x72, 0x75, 0x73, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x42, 0x0a, + 0x0a, 0x74, 0x6c, 0x73, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x23, 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, + 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x69, 0x6e, 0x66, 0x72, 0x61, 0x2e, 0x54, 0x4c, 0x53, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x09, 0x74, 0x6c, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x42, 0x06, 0x0a, 0x04, 0x5f, 0x73, 0x6e, 0x69, 0x22, 0x5e, 0x0a, 0x09, 0x54, 0x4c, 0x53, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x51, 0x0a, 0x0e, 0x63, 0x61, 0x5f, 0x63, 0x65, 0x72, + 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, + 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x72, 0x65, + 0x6d, 0x6f, 0x74, 0x65, 0x69, 0x6e, 0x66, 0x72, 0x61, 0x2e, 0x54, 0x4c, 0x53, 0x43, 0x41, 0x43, + 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x52, 0x0d, 0x63, 0x61, 0x43, 0x65, + 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x22, 0x34, 0x0a, 0x10, 0x54, 0x4c, 0x53, + 0x43, 0x41, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x20, 0x0a, + 0x0b, 0x63, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x22, + 0x25, 0x0a, 0x23, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4f, 0x72, 0x55, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x26, 0x0a, 0x24, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4f, 0x72, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, - 0x74, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x12, 0x3d, 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x67, 0x61, - 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x69, 0x6e, 0x66, 0x72, - 0x61, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4f, 0x72, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x3e, 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x67, 0x61, 0x74, - 0x65, 0x77, 0x61, 0x79, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x69, 0x6e, 0x66, 0x72, 0x61, - 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4f, 0x72, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, - 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x87, 0x01, 0x0a, 0x14, 0x44, 0x65, 0x6c, 0x65, - 0x74, 0x65, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x49, 0x6e, 0x66, 0x72, 0x61, - 0x12, 0x35, 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, - 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x69, 0x6e, 0x66, 0x72, 0x61, 0x2e, 0x44, 0x65, 0x6c, 0x65, - 0x74, 0x65, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x49, 0x6e, 0x66, 0x72, 0x61, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x36, 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x67, + 0x74, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x1d, + 0x0a, 0x1b, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, + 0x74, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x1e, 0x0a, + 0x1c, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, + 0x49, 0x6e, 0x66, 0x72, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0xe9, 0x04, + 0x0a, 0x28, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x52, 0x65, + 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x75, + 0x72, 0x65, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x93, 0x01, 0x0a, 0x18, 0x43, + 0x72, 0x65, 0x61, 0x74, 0x65, 0x4f, 0x72, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, + 0x78, 0x79, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x12, 0x39, 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x69, 0x6e, 0x66, - 0x72, 0x61, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, - 0x69, 0x74, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x42, 0x13, 0x5a, 0x11, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, 0x65, 0x6d, 0x6f, 0x74, - 0x65, 0x69, 0x6e, 0x66, 0x72, 0x61, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x72, 0x61, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4f, 0x72, 0x55, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x3a, 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, + 0x79, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x69, 0x6e, 0x66, 0x72, 0x61, 0x2e, 0x43, 0x72, + 0x65, 0x61, 0x74, 0x65, 0x4f, 0x72, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x78, + 0x79, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x12, 0x7b, 0x0a, 0x10, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x49, + 0x6e, 0x66, 0x72, 0x61, 0x12, 0x31, 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x67, 0x61, 0x74, 0x65, + 0x77, 0x61, 0x79, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x69, 0x6e, 0x66, 0x72, 0x61, 0x2e, + 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x49, 0x6e, 0x66, 0x72, 0x61, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x32, 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x67, + 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x69, 0x6e, 0x66, + 0x72, 0x61, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x49, 0x6e, + 0x66, 0x72, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x9f, 0x01, + 0x0a, 0x1c, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4f, 0x72, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x12, 0x3d, + 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x72, 0x65, + 0x6d, 0x6f, 0x74, 0x65, 0x69, 0x6e, 0x66, 0x72, 0x61, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x4f, 0x72, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, + 0x74, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x3e, 0x2e, + 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x72, 0x65, 0x6d, + 0x6f, 0x74, 0x65, 0x69, 0x6e, 0x66, 0x72, 0x61, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4f, + 0x72, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, + 0x49, 0x6e, 0x66, 0x72, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, + 0x87, 0x01, 0x0a, 0x14, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, + 0x6d, 0x69, 0x74, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x12, 0x35, 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, + 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x69, 0x6e, + 0x66, 0x72, 0x61, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, + 0x6d, 0x69, 0x74, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x36, 0x2e, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x72, + 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x69, 0x6e, 0x66, 0x72, 0x61, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, + 0x65, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x13, 0x5a, 0x11, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2f, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x69, 0x6e, 0x66, 0x72, 0x61, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, }) var ( diff --git a/proto/remoteinfra/service.proto b/proto/remoteinfra/service.proto index bc9a5e05f62..42835e06847 100644 --- a/proto/remoteinfra/service.proto +++ b/proto/remoteinfra/service.proto @@ -158,6 +158,8 @@ message ResolvedMetricSink { bool report_counters_as_deltas = 5; // report_histograms_as_deltas configures histograms to use delta temporality. bool report_histograms_as_deltas = 6; + // prefix is prepended to emitted stat names, the full name will be `.`. + string prefix = 7; } // HTTPHeader is a name/value HTTP header. It mirrors the subset of diff --git a/release-notes/current/new_features/9720-otel-metrics-prefix.md b/release-notes/current/new_features/9720-otel-metrics-prefix.md new file mode 100644 index 00000000000..a013be8f58c --- /dev/null +++ b/release-notes/current/new_features/9720-otel-metrics-prefix.md @@ -0,0 +1 @@ +Added an optional `prefix` field to the OpenTelemetry metrics sink (`EnvoyProxy.spec.telemetry.metrics.sinks[].openTelemetry.prefix`) that is prepended to emitted stat names, so the full stat name becomes `.`. diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index 2d2c7343a86..3c6bcaa1571 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -5257,6 +5257,7 @@ _Appears in:_ | `port` | _integer_ | false | 4317 | Port defines the port the service is exposed on.
Deprecated: Use BackendRefs instead. | | `reportCountersAsDeltas` | _boolean_ | false | | ReportCountersAsDeltas configures the OpenTelemetry sink to report
counters as delta temporality instead of cumulative. | | `reportHistogramsAsDeltas` | _boolean_ | false | | ReportHistogramsAsDeltas configures the OpenTelemetry sink to report
histograms as delta temporality instead of cumulative.
Required for backends like Elastic that drop cumulative histograms. | +| `prefix` | _string_ | false | | Prefix configures the OpenTelemetry sink to prepend the given prefix to emitted stat names,
the full stat name will be `.`. | | `headers` | _[HTTPHeader](#httpheader) array_ | false | | Headers is a list of additional headers to send with OTLP export requests.
These headers are added as gRPC initial metadata for the OTLP gRPC service. | | `resourceAttributes` | _object (keys:string, values:string)_ | false | | ResourceAttributes is a set of labels that describe the source of metrics.
It's recommended to follow semantic conventions: https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/ | diff --git a/test/e2e/testdata/metrics-with-prefix.yaml b/test/e2e/testdata/metrics-with-prefix.yaml new file mode 100644 index 00000000000..bebacd1ff53 --- /dev/null +++ b/test/e2e/testdata/metrics-with-prefix.yaml @@ -0,0 +1,78 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: metric-otel-prefix + namespace: gateway-conformance-infra +spec: + gatewayClassName: "{GATEWAY_CLASS_NAME}" + listeners: + - name: http + port: 80 + protocol: HTTP + allowedRoutes: + namespaces: + from: All + infrastructure: + parametersRef: + group: gateway.envoyproxy.io + kind: EnvoyProxy + name: metric-otel-prefix +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyProxy +metadata: + name: metric-otel-prefix + namespace: gateway-conformance-infra +spec: + ipFamily: IPv4 + logging: + level: + default: debug + telemetry: + metrics: + prometheus: {} + sinks: + - type: OpenTelemetry + openTelemetry: + backendRefs: + - name: otel-collector + namespace: monitoring + port: 4317 + prefix: eg_e2e_prefix + shutdown: + drainTimeout: 5s + minDrainDuration: 1s +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: metric-otel-prefix + namespace: gateway-conformance-infra +spec: + parentRefs: + - name: metric-otel-prefix + rules: + - matches: + - path: + type: PathPrefix + value: /prom-prefix + backendRefs: + - name: infra-backend-v1 + port: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: prefix-metrix-gtw-metrics + namespace: monitoring +spec: + selector: + app.kubernetes.io/instance: eg-addons + app.kubernetes.io/name: opentelemetry-collector + component: standalone-collector + ports: + - name: metrics + protocol: TCP + port: 19001 + targetPort: 19001 + type: LoadBalancer diff --git a/test/e2e/tests/metric.go b/test/e2e/tests/metric.go index 77bfafffaaf..08e21ff08f8 100644 --- a/test/e2e/tests/metric.go +++ b/test/e2e/tests/metric.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/controller-runtime/pkg/client" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" httputils "sigs.k8s.io/gateway-api/conformance/utils/http" "sigs.k8s.io/gateway-api/conformance/utils/kubernetes" @@ -37,6 +38,7 @@ func init() { MetricCompressorGzipTest, MetricCompressorBrotliTest, MetricCompressorZstdTest, + OtelMetricPrefixTest, ) } @@ -180,6 +182,60 @@ var MetricCompressorZstdTest = suite.ConformanceTest{ }, } +var OtelMetricPrefixTest = suite.ConformanceTest{ + ShortName: "OtelMetricPrefix", + Description: "Make sure metrics arrive with configured prefix", + Manifests: []string{"testdata/metrics-with-prefix.yaml"}, + Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { + ns := "gateway-conformance-infra" + routeNN := types.NamespacedName{Name: "metric-otel-prefix", Namespace: ns} + gwNN := types.NamespacedName{Name: "metric-otel-prefix", Namespace: ns} + gwAddr := kubernetes.GatewayAndRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), &gwapiv1.HTTPRoute{}, false, routeNN) + + expectedResponse := httputils.ExpectedResponse{ + Request: httputils.Request{ + Path: "/prom-prefix", + }, + Response: httputils.Response{ + StatusCodes: []int{200}, + }, + Namespace: ns, + } + + httputils.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, expectedResponse) + + err := wait.PollUntilContextTimeout(context.TODO(), 3*time.Second, time.Minute, true, func(_ context.Context) (done bool, err error) { + ok, err := scrapeMetricsHasPrefix(suite.Client, types.NamespacedName{ + Namespace: "monitoring", + Name: "prefix-metrix-gtw-metrics", + }, 19001, "/metrics", "eg_e2e_prefix") + if err != nil { + tlog.Logf(t, "failed to scrape metrics: %v", err) + return false, nil + } + return ok, nil + }) + require.NoError(t, err, "failed to retrieve metrics with expected prefix within timeout.") + }, +} + +func scrapeMetricsHasPrefix(c client.Client, nn types.NamespacedName, port int32, path, prefix string) (bool, error) { + url, err := RetrieveURL(c, nn, port, path) + if err != nil { + return false, err + } + mfs, err := RetrieveMetrics(url, 5*time.Second) + if err != nil { + return false, err + } + for name := range mfs { + if strings.HasPrefix(name, prefix) { + return true, nil + } + } + return false, nil +} + func runMetricCompressorTest(t *testing.T, suite *suite.ConformanceTestSuite, ns string, compressorType egv1a1.CompressorType) { compressor := strings.ToLower(string(compressorType)) // Gzip -> gzip routeName := fmt.Sprintf("%s-route", compressor) diff --git a/test/helm/gateway-crds-helm/all.out.yaml b/test/helm/gateway-crds-helm/all.out.yaml index 197aec2547f..df2fe49713b 100644 --- a/test/helm/gateway-crds-helm/all.out.yaml +++ b/test/helm/gateway-crds-helm/all.out.yaml @@ -51179,6 +51179,11 @@ spec: maximum: 65535 minimum: 0 type: integer + prefix: + description: |- + Prefix configures the OpenTelemetry sink to prepend the given prefix to emitted stat names, + the full stat name will be `.`. + type: string reportCountersAsDeltas: description: |- ReportCountersAsDeltas configures the OpenTelemetry sink to report diff --git a/test/helm/gateway-crds-helm/e2e.out.yaml b/test/helm/gateway-crds-helm/e2e.out.yaml index 32f54c37669..d11ad2e8d72 100644 --- a/test/helm/gateway-crds-helm/e2e.out.yaml +++ b/test/helm/gateway-crds-helm/e2e.out.yaml @@ -27117,6 +27117,11 @@ spec: maximum: 65535 minimum: 0 type: integer + prefix: + description: |- + Prefix configures the OpenTelemetry sink to prepend the given prefix to emitted stat names, + the full stat name will be `.`. + type: string reportCountersAsDeltas: description: |- ReportCountersAsDeltas configures the OpenTelemetry sink to report diff --git a/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml b/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml index db1da4ef9c1..72975eafba9 100644 --- a/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml +++ b/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml @@ -27117,6 +27117,11 @@ spec: maximum: 65535 minimum: 0 type: integer + prefix: + description: |- + Prefix configures the OpenTelemetry sink to prepend the given prefix to emitted stat names, + the full stat name will be `.`. + type: string reportCountersAsDeltas: description: |- ReportCountersAsDeltas configures the OpenTelemetry sink to report