diff --git a/gateway/examples/bedrock-openai-proxy.yaml b/gateway/examples/bedrock-openai-proxy.yaml new file mode 100644 index 0000000000..4d50b7388e --- /dev/null +++ b/gateway/examples/bedrock-openai-proxy.yaml @@ -0,0 +1,54 @@ +# -------------------------------------------------------------------- +# Copyright (c) 2026, WSO2 LLC. (https://www.wso2.com). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# -------------------------------------------------------------------- + +# -------------------------------------------------------------------- +# OpenAI -> AWS Bedrock LLM Proxy (single-provider) +# +# Exposes an OpenAI-shaped /chat/completions endpoint that the +# `openai-to-bedrock-transformer` policy translates into Bedrock's Converse API format +# (path /model/{modelId}/converse[-stream], body rewrite) and forwards to the +# existing `bedrock-provider` upstream. For streaming requests the policy +# decodes Bedrock's binary Amazon event-stream and re-emits OpenAI SSE, so +# OpenAI clients get `data: {...}` chunks terminated by `data: [DONE]`. +# +# Single-provider mode: there is no router in front of the policy, so nothing +# sets metadata["selected_provider"] and the translator runs on every request. +# +# The provider must already be deployed (see bedrock-provider.yaml) with its +# Bearer key on upstream.auth (the policy does NOT inject the Bedrock key). +# The provider example's api-key-auth policy expects the proxy to send the +# issued provider loopback key in X-API-Key. +# -------------------------------------------------------------------- + +apiVersion: gateway.api-platform.wso2.com/v1 +kind: LlmProxy +metadata: + name: openai-to-bedrock +spec: + displayName: OpenAI to Bedrock Proxy + version: v1.0 + context: /openai-bedrock + provider: + id: bedrock-provider + auth: + type: api-key + header: X-API-Key + value: REPLACE_WITH_BEDROCK_PROVIDER_LOOPBACK_KEY + policies: + - name: openai-to-bedrock-transformer + version: v1 + paths: + - path: /chat/completions + methods: [POST] + params: + # Bedrock puts the model id in the URL path; the translator builds + # /model//converse[-stream] from this param. Use an inference + # profile id your key can invoke (e.g. us.* / eu.*). + model: us.amazon.nova-lite-v1:0 diff --git a/gateway/examples/bedrock-provider.yaml b/gateway/examples/bedrock-provider.yaml new file mode 100644 index 0000000000..0413337109 --- /dev/null +++ b/gateway/examples/bedrock-provider.yaml @@ -0,0 +1,73 @@ +# -------------------------------------------------------------------- +# Copyright (c) 2026, WSO2 LLC. (https://www.wso2.com). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# -------------------------------------------------------------------- + +# -------------------------------------------------------------------- +# AWS Bedrock LLM Provider Configuration (native Converse API) +# +# Uses the built-in `awsbedrock` template and points to the region-specific +# Bedrock runtime endpoint. AWS Bedrock now supports Bearer-token auth +# (Amazon Bedrock API keys) in addition to the legacy SigV4 flow -- this +# provider uses the Bearer key. Replace the placeholder with a real key and +# set the URL to the region your key and model access live in. +# +# The model id is carried in the URL path (/model/{modelId}/converse), so the +# awsbedrock template extracts request/response model via location: pathParam +# (a RE2 capture group), for both /converse and /converse-stream. +# +# Auth: upstream.auth is the gateway -> Bedrock vendor auth. The header is +# Authorization and the value is sent verbatim, so the "Bearer " prefix MUST +# be included here (the portal auto-prepends it; raw YAML does not). Proxies in +# front of this provider inherit this vendor auth automatically. +# -------------------------------------------------------------------- + +apiVersion: gateway.api-platform.wso2.com/v1 +kind: LlmProvider +metadata: + name: bedrock-provider +spec: + displayName: AWS Bedrock Provider + version: v1.0 + template: awsbedrock + context: /bedrock + upstream: + # Match the region your key/model access is in. For the no-AWS mock + # (bedrock-mock.py), use: http://host.docker.internal:9010 + url: https://bedrock-runtime.us-east-1.amazonaws.com + auth: + type: api-key + header: Authorization + # Bedrock expects: Authorization: Bearer -- sent verbatim, so the + # "Bearer " prefix MUST be included here. The mock ignores the value. + value: Bearer REPLACE_WITH_BEDROCK_API_KEY + policies: + - name: api-key-auth + version: v1 + paths: + - path: /model/{modelId}/converse + methods: [POST] + params: + key: X-API-Key + in: header + - path: /model/{modelId}/converse-stream + methods: [POST] + params: + key: X-API-Key + in: header + accessControl: + mode: deny_all + exceptions: + - path: /model/{modelId}/converse + methods: [POST] + - path: /model/{modelId}/converse-stream + methods: [POST] + +# The proxy in front of this provider authenticates over the internal loopback +# route with an API key issued for this LlmProvider (see bedrock-testing.md, +# "Issue a consumer key"). Set the issued value as the proxy's provider.auth.value. diff --git a/gateway/examples/openai-multi-provider-proxy.yaml b/gateway/examples/openai-multi-provider-proxy.yaml index e88f189012..93bc619171 100644 --- a/gateway/examples/openai-multi-provider-proxy.yaml +++ b/gateway/examples/openai-multi-provider-proxy.yaml @@ -89,6 +89,16 @@ spec: params: model: gemini-2.5-flash apiVersion: v1beta + - id: bedrock-provider + auth: + type: api-key + header: X-API-Key + value: REPLACE_WITH_BEDROCK_PROVIDER_LOOPBACK_KEY + transformer: + type: openai-to-bedrock-transformer + version: v1 + params: + model: us.amazon.nova-lite-v1:0 policies: - name: api-key-auth version: v1 @@ -117,6 +127,8 @@ spec: provider: mistral-provider - headerValue: gemini provider: gemini-provider + - headerValue: bedrock + provider: bedrock-provider # - name: token-based-ratelimit # version: v1 diff --git a/gateway/gateway-controller/default-llm-provider-templates/awsbedrock-template.yaml b/gateway/gateway-controller/default-llm-provider-templates/awsbedrock-template.yaml index 5b8968f9d5..5abc25b1a5 100644 --- a/gateway/gateway-controller/default-llm-provider-templates/awsbedrock-template.yaml +++ b/gateway/gateway-controller/default-llm-provider-templates/awsbedrock-template.yaml @@ -36,7 +36,7 @@ spec: identifier: $.usage.totalTokens requestModel: location: pathParam - identifier: (?<=model/)[a-zA-Z0-9.:-]+(?=/) + identifier: model/([A-Za-z0-9.:-]+)/ responseModel: location: pathParam - identifier: (?<=model/)[a-zA-Z0-9.:-]+(?=/) + identifier: model/([A-Za-z0-9.:-]+)/ diff --git a/gateway/gateway-controller/pkg/utils/llm_transformer.go b/gateway/gateway-controller/pkg/utils/llm_transformer.go index 131d1d39da..f45eee42ae 100644 --- a/gateway/gateway-controller/pkg/utils/llm_transformer.go +++ b/gateway/gateway-controller/pkg/utils/llm_transformer.go @@ -179,15 +179,22 @@ func (t *LLMProviderTransformer) transformProxy(proxy *api.LLMProxyConfiguration if err != nil { return nil, fmt.Errorf("failed to get context for additional provider '%s': %w", ap.Id, err) } - addURL := fmt.Sprintf("%s://%s:%d%s", - constants.SchemeHTTP, constants.LocalhostIP, t.routerConfig.ListenerPort, addCtx) - defs = append(defs, api.UpstreamDefinition{ + // Named upstream definition URLs are host-only. Keep the provider context + // in basePath so dynamic provider routing does not drop it and send the + // loopback request to the proxy listener root. + addURL := fmt.Sprintf("%s://%s:%d", + constants.SchemeHTTP, constants.LocalhostIP, t.routerConfig.ListenerPort) + def := api.UpstreamDefinition{ Name: name, Upstreams: []struct { Url string `json:"url" yaml:"url"` Weight *int `json:"weight,omitempty" yaml:"weight,omitempty"` }{{Url: addURL}}, - }) + } + if addCtx != "" && addCtx != constants.BASE_PATH { + def.BasePath = &addCtx + } + defs = append(defs, def) } spec.UpstreamDefinitions = &defs } diff --git a/gateway/gateway-controller/pkg/utils/llm_transformer_multiprovider_test.go b/gateway/gateway-controller/pkg/utils/llm_transformer_multiprovider_test.go index 9d558dfaf0..8cb4d9cc58 100644 --- a/gateway/gateway-controller/pkg/utils/llm_transformer_multiprovider_test.go +++ b/gateway/gateway-controller/pkg/utils/llm_transformer_multiprovider_test.go @@ -119,6 +119,10 @@ func TestLLMProviderTransformer_TransformProxy_AdditionalProviderAuthIsCondition require.NotNil(t, result.Spec.UpstreamDefinitions) require.Len(t, *result.Spec.UpstreamDefinitions, 1) assert.Equal(t, "anthropic-provider", (*result.Spec.UpstreamDefinitions)[0].Name) + require.NotNil(t, (*result.Spec.UpstreamDefinitions)[0].BasePath) + assert.Equal(t, "/anthropic-provider", *(*result.Spec.UpstreamDefinitions)[0].BasePath) + require.Len(t, (*result.Spec.UpstreamDefinitions)[0].Upstreams, 1) + assert.Equal(t, "http://127.0.0.1:8080", (*result.Spec.UpstreamDefinitions)[0].Upstreams[0].Url) var chatOp *api.Operation for i := range result.Spec.Operations { diff --git a/gateway/system-policies/analytics/analytics.go b/gateway/system-policies/analytics/analytics.go index 92a51b204b..a3badec79f 100644 --- a/gateway/system-policies/analytics/analytics.go +++ b/gateway/system-policies/analytics/analytics.go @@ -7,9 +7,11 @@ import ( "fmt" "log/slog" "reflect" + "regexp" "sort" "strconv" "strings" + "sync" policy "github.com/wso2/api-platform/sdk/core/policy/v1alpha2" "github.com/wso2/api-platform/sdk/core/utils" @@ -568,7 +570,7 @@ func (a *AnalyticsPolicy) OnResponseBodyChunk(_ context.Context, ctx *policy.Res // Streaming responses are SSE; the last data event carries usage fields. tokenInfo, err := extractLLMProviderAnalyticsInfoFromBytes( template, ctx.RequestHeaders, ctx.ResponseHeaders, - requestBodyBytes, accumulated, + requestBodyBytes, accumulated, ctx.RequestPath, ) if err != nil { slog.Warn("Failed to extract LLM token info from streaming response", "error", err) @@ -662,7 +664,7 @@ func extractLLMProviderAnalyticsInfo(template map[string]interface{}, ctx *polic return extractLLMProviderAnalyticsInfoFromBytes( template, ctx.RequestHeaders, ctx.ResponseHeaders, - requestBodyBytes, responseBodyBytes, + requestBodyBytes, responseBodyBytes, ctx.RequestPath, ) } @@ -676,6 +678,7 @@ func extractLLMProviderAnalyticsInfoFromBytes( template map[string]interface{}, requestHeaders, responseHeaders *policy.Headers, requestBodyBytes, responseBodyBytes []byte, + requestPath string, ) (*LLMProviderAnalyticsInfo, error) { var responseJSON map[string]interface{} if len(responseBodyBytes) > 0 { @@ -693,7 +696,7 @@ func extractLLMProviderAnalyticsInfoFromBytes( _ = json.Unmarshal(requestBodyBytes, &requestJSON) } - return extractLLMAnalyticsFromJSON(template, requestHeaders, responseHeaders, requestJSON, responseJSON) + return extractLLMAnalyticsFromJSON(template, requestHeaders, responseHeaders, requestJSON, responseJSON, requestPath) } // extractLLMAnalyticsFromJSON is the core extraction logic operating on pre-parsed JSON maps. @@ -701,6 +704,7 @@ func extractLLMAnalyticsFromJSON( template map[string]interface{}, requestHeaders, responseHeaders *policy.Headers, requestJSON, responseJSON map[string]interface{}, + requestPath string, ) (*LLMProviderAnalyticsInfo, error) { if template == nil { return nil, fmt.Errorf("template is nil") @@ -754,6 +758,14 @@ func extractLLMAnalyticsFromJSON( } } return nil, fmt.Errorf("header %s not found", identifier) + case "pathparam": + // Model ids for providers like AWS Bedrock and Gemini live in the + // request URL path (e.g. /model/{modelId}/converse), not the body or + // a header. The identifier is a regex whose first capture group (when + // present) is the value; otherwise the whole match is used. The path + // is available for both buffered and streaming responses, so this + // meters the model in either case. + return extractPathParam(requestPath, identifier) default: return nil, fmt.Errorf("unsupported location %s", location) } @@ -806,6 +818,48 @@ func extractLLMAnalyticsFromJSON( return info, nil } +// pathParamRegexCache memoises compiled pathParam identifiers so the response +// data path does not recompile a provider template's regex on every request. +var pathParamRegexCache sync.Map // map[string]*regexp.Regexp + +// extractPathParam applies a pathParam identifier (a regex) to the request URL +// path and returns the first capture group when the pattern defines one, +// otherwise the whole match. AWS Bedrock and Gemini carry the model id in the +// path (e.g. /model/{modelId}/converse), which is why request/response model is +// declared with `location: pathParam`. The path is present for both buffered +// and streaming responses, so the model meters in either case. Go's regexp is +// RE2 — identifiers must use a capture group, not lookaround. +func extractPathParam(requestPath, pattern string) (string, error) { + if requestPath == "" { + return "", fmt.Errorf("request path not available") + } + re, err := compilePathParamRegex(pattern) + if err != nil { + return "", err + } + match := re.FindStringSubmatch(requestPath) + if match == nil { + return "", fmt.Errorf("pathParam identifier did not match request path") + } + if len(match) > 1 { + return match[1], nil + } + return match[0], nil +} + +// compilePathParamRegex compiles (and caches) a pathParam identifier regex. +func compilePathParamRegex(pattern string) (*regexp.Regexp, error) { + if cached, ok := pathParamRegexCache.Load(pattern); ok { + return cached.(*regexp.Regexp), nil + } + re, err := regexp.Compile(pattern) + if err != nil { + return nil, fmt.Errorf("invalid pathParam identifier %q: %w", pattern, err) + } + pathParamRegexCache.Store(pattern, re) + return re, nil +} + // populateTokenAnalyticsMetadata copies LLM token fields into an analytics metadata map. func populateTokenAnalyticsMetadata(analyticsMetadata map[string]any, tokenInfo *LLMProviderAnalyticsInfo) { if tokenInfo.PromptTokens != nil { diff --git a/gateway/system-policies/analytics/analytics_bedrock_test.go b/gateway/system-policies/analytics/analytics_bedrock_test.go new file mode 100644 index 0000000000..f42affa7e7 --- /dev/null +++ b/gateway/system-policies/analytics/analytics_bedrock_test.go @@ -0,0 +1,134 @@ +package analytics + +import "testing" + +// TestExtractPathParam covers the pathParam extraction used to meter the model +// id for providers that carry it in the URL path (AWS Bedrock, Gemini) rather +// than the body or a header. +func TestExtractPathParam(t *testing.T) { + cases := []struct { + name string + path string + pattern string + want string + wantErr bool + }{ + { + name: "bedrock converse model id", + path: "/bedrock/model/us.amazon.nova-lite-v1:0/converse", + pattern: "model/([A-Za-z0-9.:-]+)/", + want: "us.amazon.nova-lite-v1:0", + }, + { + name: "bedrock streaming model id", + path: "/bedrock/model/us.anthropic.claude-3-5-sonnet-20240620-v1:0/converse-stream", + pattern: "model/([A-Za-z0-9.:-]+)/", + want: "us.anthropic.claude-3-5-sonnet-20240620-v1:0", + }, + { + name: "no capture group returns whole match", + path: "/v1beta/models/gemini-1.5-flash:generateContent", + pattern: "models/[A-Za-z0-9.-]+", + want: "models/gemini-1.5-flash", + }, + { + name: "empty path errors", + path: "", + pattern: "model/([A-Za-z0-9.:-]+)/", + wantErr: true, + }, + { + name: "no match errors", + path: "/bedrock/health", + pattern: "model/([A-Za-z0-9.:-]+)/", + wantErr: true, + }, + { + // Go's regexp is RE2 — lookbehind is unsupported. This is exactly the + // class of pattern that must never reach the extractor, which is why + // the awsbedrock template uses a capture group instead. + name: "lookbehind pattern rejected by RE2", + path: "/bedrock/model/x/converse", + pattern: "(?<=model/)[A-Za-z0-9.:-]+(?=/)", + wantErr: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := extractPathParam(tc.path, tc.pattern) + if tc.wantErr { + if err == nil { + t.Fatalf("expected error, got value %q", got) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != tc.want { + t.Fatalf("got %q, want %q", got, tc.want) + } + }) + } +} + +// TestExtractLLMAnalyticsBedrockPathParam verifies the AWS Bedrock template +// shape end-to-end: token counts come from the response payload ($.usage.*) +// while the model id is taken from the request URL path (location: pathParam), +// which is the only place native Bedrock exposes it. The path is present on both +// the buffered and streaming response contexts, so this resolves in either case. +func TestExtractLLMAnalyticsBedrockPathParam(t *testing.T) { + template := map[string]interface{}{ + "metadata": map[string]interface{}{"name": "awsbedrock"}, + "spec": map[string]interface{}{ + "displayName": "AWS Bedrock", + "promptTokens": map[string]interface{}{"location": "payload", "identifier": "$.usage.inputTokens"}, + "completionTokens": map[string]interface{}{"location": "payload", "identifier": "$.usage.outputTokens"}, + "totalTokens": map[string]interface{}{"location": "payload", "identifier": "$.usage.totalTokens"}, + "requestModel": map[string]interface{}{"location": "pathParam", "identifier": "model/([A-Za-z0-9.:-]+)/"}, + "responseModel": map[string]interface{}{"location": "pathParam", "identifier": "model/([A-Za-z0-9.:-]+)/"}, + }, + } + + // Bedrock Converse numbers arrive as JSON numbers → float64 once unmarshalled. + responseJSON := map[string]interface{}{ + "usage": map[string]interface{}{ + "inputTokens": float64(11), + "outputTokens": float64(7), + "totalTokens": float64(18), + }, + } + + const requestPath = "/bedrock/model/us.amazon.nova-lite-v1:0/converse" + + info, err := extractLLMAnalyticsFromJSON(template, nil, nil, nil, responseJSON, requestPath) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if info.PromptTokens == nil || *info.PromptTokens != 11 { + t.Errorf("PromptTokens = %v, want 11", info.PromptTokens) + } + if info.CompletionTokens == nil || *info.CompletionTokens != 7 { + t.Errorf("CompletionTokens = %v, want 7", info.CompletionTokens) + } + if info.TotalTokens == nil || *info.TotalTokens != 18 { + t.Errorf("TotalTokens = %v, want 18", info.TotalTokens) + } + if info.RequestModel == nil || *info.RequestModel != "us.amazon.nova-lite-v1:0" { + t.Errorf("RequestModel = %v, want us.amazon.nova-lite-v1:0", info.RequestModel) + } + if info.ResponseModel == nil || *info.ResponseModel != "us.amazon.nova-lite-v1:0" { + t.Errorf("ResponseModel = %v, want us.amazon.nova-lite-v1:0", info.ResponseModel) + } + + // The metadata that actually reaches analytics must carry the resolved model. + meta := map[string]any{} + populateTokenAnalyticsMetadata(meta, info) + if meta[ModelIDMetadataKey] != "us.amazon.nova-lite-v1:0" { + t.Errorf("%s = %v, want us.amazon.nova-lite-v1:0", ModelIDMetadataKey, meta[ModelIDMetadataKey]) + } + if meta[PromptTokenCountMetadataKey] != "11" { + t.Errorf("%s = %v, want \"11\"", PromptTokenCountMetadataKey, meta[PromptTokenCountMetadataKey]) + } +} diff --git a/platform-api/resources/default-llm-provider-templates/awsbedrock-template.yaml b/platform-api/resources/default-llm-provider-templates/awsbedrock-template.yaml index 6917e16ffa..1cffef5673 100644 --- a/platform-api/resources/default-llm-provider-templates/awsbedrock-template.yaml +++ b/platform-api/resources/default-llm-provider-templates/awsbedrock-template.yaml @@ -24,6 +24,17 @@ spec: managedBy: wso2 version: v1.0 metadata: + # AWS Bedrock now supports Bearer-token auth (Amazon Bedrock API keys) in + # addition to the legacy SigV4 flow. The gateway injects this as + # `Authorization: Bearer `; the portal prepends the `Bearer ` prefix so + # the operator only pastes the raw key. endpointUrl is deliberately omitted + # because the Bedrock runtime host is region-specific + # (https://bedrock-runtime..amazonaws.com) and must be supplied per + # provider. + auth: + type: apiKey + header: Authorization + valuePrefix: "Bearer " logoUrl: https://raw.githubusercontent.com/nomadxd/openapi-connectors/main/openapi/aws.bedrock/icon.png openapiSpecUrl: https://raw.githubusercontent.com/nomadxd/openapi-connectors/main/openapi/aws.bedrock/openapi.yaml promptTokens: