diff --git a/gateway/examples/anthropic-openai-proxy.yaml b/gateway/examples/anthropic-openai-proxy.yaml new file mode 100644 index 000000000..ebccbaa33 --- /dev/null +++ b/gateway/examples/anthropic-openai-proxy.yaml @@ -0,0 +1,52 @@ +# -------------------------------------------------------------------- +# Copyright (c) 2026, WSO2 LLC. (https://www.wso2.com). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# -------------------------------------------------------------------- + +# -------------------------------------------------------------------- +# OpenAI -> Anthropic LLM Proxy (single-provider) +# +# Exposes an OpenAI-shaped /chat/completions endpoint that the +# `openai-to-anthropic` policy translates into Anthropic's Messages API +# format and forwards to the existing `anthropic-provider` upstream +# (api.anthropic.com). +# +# Single-provider mode: there is no router in front of the policy, so +# nothing sets metadata["selected_provider"] and the translator runs on +# every request. "id" is omitted, so it routes to the default upstream +# (the primary provider above). +# +# The provider must already be deployed and must have its x-api-key +# set on upstream.auth (the policy does NOT inject the Anthropic key). +# +# The provider example's api-key-auth policy expects the proxy to send the +# issued provider loopback key in X-API-Key. +# -------------------------------------------------------------------- + +apiVersion: gateway.api-platform.wso2.com/v1 +kind: LlmProxy +metadata: + name: openai-to-anthropic +spec: + displayName: OpenAI to Anthropic Proxy + version: v1.0 + context: /openai-anthropic + provider: + id: anthropic-provider + auth: + type: api-key + header: X-API-Key + value: REPLACE_WITH_ANTHROPIC_PROVIDER_LOOPBACK_KEY + policies: + - name: openai-to-anthropic + version: v1 + paths: + - path: /chat/completions + methods: [POST] + params: + model: claude-sonnet-4-5-20250929 diff --git a/gateway/examples/anthropic-provider.yaml b/gateway/examples/anthropic-provider.yaml new file mode 100644 index 000000000..2a653d354 --- /dev/null +++ b/gateway/examples/anthropic-provider.yaml @@ -0,0 +1,58 @@ +# -------------------------------------------------------------------- +# Copyright (c) 2026, WSO2 LLC. (https://www.wso2.com). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# -------------------------------------------------------------------- + +# -------------------------------------------------------------------- +# Anthropic LLM Provider Configuration +# +# Uses the built-in `anthropic` template and points to the public +# Anthropic API endpoint. Replace the placeholder with a real Anthropic +# API key (sk-ant-...) before deploying. +# +# The openai-to-anthropic policy on the proxy layer translates +# OpenAI-format requests into Anthropic's Messages API wire format +# (path /v1/messages, anthropic-version header, body rewrite). +# +# Auth: upstream.auth is the gateway -> Anthropic vendor auth. Proxies in +# front of this provider inherit this vendor auth automatically. +# -------------------------------------------------------------------- + +apiVersion: gateway.api-platform.wso2.com/v1 +kind: LlmProvider +metadata: + name: anthropic-provider +spec: + displayName: Anthropic Provider + version: v1.0 + template: anthropic + upstream: + url: https://api.anthropic.com + auth: + type: api-key + header: x-api-key + # Anthropic's header is x-api-key with the raw key (NO "Bearer" prefix). + value: sk-ant-REPLACE_WITH_ANTHROPIC_KEY + policies: + - name: api-key-auth + version: v1 + paths: + - path: /v1/messages + methods: [POST] + params: + key: X-API-Key + in: header + accessControl: + mode: deny_all + exceptions: + - path: /v1/messages + methods: [POST] + +# The proxy in front of this provider authenticates over the internal loopback +# route with an API key issued for this LlmProvider. Create one as shown in +# api-key.yaml, then set the issued value as the proxy's provider.auth.value. diff --git a/gateway/examples/azure-openai-provider.yaml b/gateway/examples/azure-openai-provider.yaml new file mode 100644 index 000000000..6e4686e47 --- /dev/null +++ b/gateway/examples/azure-openai-provider.yaml @@ -0,0 +1,79 @@ +# -------------------------------------------------------------------- +# Copyright (c) 2026, WSO2 LLC. (https://www.wso2.com). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# -------------------------------------------------------------------- + +# -------------------------------------------------------------------- +# Azure OpenAI LLM Provider Configuration +# +# Uses the built-in 'azure-openai' template and points to your Azure +# OpenAI endpoint. The openai-to-azure-openai policy on the proxy layer +# translates OpenAI-format requests into the Azure OpenAI wire format +# (path rewriting, api-version query param). +# +# Replace before deploying: +# - url: your Azure OpenAI endpoint, e.g. https://my-resource.openai.azure.com +# - value: your Azure OpenAI API key +# +# Auth: upstream.auth is the gateway -> Azure vendor auth (header: api-key). +# Proxies in front of this provider inherit this vendor auth automatically. +# -------------------------------------------------------------------- + +apiVersion: gateway.api-platform.wso2.com/v1 +kind: LlmProvider +metadata: + name: azure-openai-provider +spec: + displayName: Azure OpenAI Provider + version: v1.0 + template: azure-openai + upstream: + url: https://REPLACE_WITH_AZURE_OPENAI_ENDPOINT + auth: + type: api-key + header: api-key + value: REPLACE_WITH_AZURE_OPENAI_API_KEY + policies: + - name: api-key-auth + version: v1 + paths: + - path: /openai/deployments/{deployment}/chat/completions + methods: [POST] + params: + key: X-API-Key + in: header + - path: /openai/deployments/{deployment}/completions + methods: [POST] + params: + key: X-API-Key + in: header + - path: /openai/deployments/{deployment}/embeddings + methods: [POST] + params: + key: X-API-Key + in: header + - path: /openai/models + methods: [GET] + params: + key: X-API-Key + in: header + accessControl: + mode: deny_all + exceptions: + - path: /openai/deployments/{deployment}/chat/completions + methods: [POST] + - path: /openai/deployments/{deployment}/completions + methods: [POST] + - path: /openai/deployments/{deployment}/embeddings + methods: [POST] + - path: /openai/models + methods: [GET] + +# The proxy in front of this provider authenticates over the internal loopback +# route with an API key issued for this LlmProvider. Create one as shown in +# api-key.yaml, then set the issued value as the proxy's provider.auth.value. diff --git a/gateway/examples/azure-openai-proxy.yaml b/gateway/examples/azure-openai-proxy.yaml new file mode 100644 index 000000000..6c031a564 --- /dev/null +++ b/gateway/examples/azure-openai-proxy.yaml @@ -0,0 +1,51 @@ +# -------------------------------------------------------------------- +# Copyright (c) 2026, WSO2 LLC. (https://www.wso2.com). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# -------------------------------------------------------------------- + +# -------------------------------------------------------------------- +# OpenAI -> Azure OpenAI LLM Proxy (single-provider) +# +# Exposes an OpenAI-shaped /chat/completions endpoint that the +# `openai-to-azure-openai` policy translates into Azure OpenAI's +# /openai/deployments/{deployment}/chat/completions?api-version=... +# format and forwards to the `azure-openai-provider` upstream. +# +# Single-provider mode: the policy omits the "provider" parameter, so +# it runs on every request without needing a setter policy. +# +# Replace before deploying: +# - apiVersion param: a valid Azure OpenAI api-version +# (e.g. 2024-02-15-preview) +# - model param (optional): pin the deployment id here, otherwise +# the request body's "model" field is used as the deployment. +# -------------------------------------------------------------------- + +apiVersion: gateway.api-platform.wso2.com/v1 +kind: LlmProxy +metadata: + name: openai-to-azure-openai +spec: + displayName: OpenAI to Azure OpenAI Proxy + version: v1.0 + context: /azure-openai + provider: + id: azure-openai-provider + auth: + type: api-key + header: X-API-Key + value: REPLACE_WITH_AZURE_OPENAI_PROVIDER_LOOPBACK_KEY + policies: + - name: openai-to-azure-openai + version: v1 + paths: + - path: /chat/completions + methods: [POST] + params: + apiVersion: "2024-02-15-preview" + model: gpt-4o diff --git a/gateway/examples/gemini-provider.yaml b/gateway/examples/gemini-provider.yaml new file mode 100644 index 000000000..d2fb4b424 --- /dev/null +++ b/gateway/examples/gemini-provider.yaml @@ -0,0 +1,60 @@ +# -------------------------------------------------------------------- +# Copyright (c) 2026, WSO2 LLC. (https://www.wso2.com). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# -------------------------------------------------------------------- + +# -------------------------------------------------------------------- +# Gemini LLM Provider Configuration +# +# Uses the built-in `gemini` template and points to Google's Gemini API. +# The openai-to-gemini policy on the proxy layer translates OpenAI-format +# requests into Gemini's generateContent API format. +# +# Auth: upstream.auth is the gateway -> Gemini vendor auth +# (header: x-goog-api-key). Proxies inherit this vendor auth automatically. +# -------------------------------------------------------------------- + +apiVersion: gateway.api-platform.wso2.com/v1 +kind: LlmProvider +metadata: + name: gemini-provider +spec: + displayName: Gemini Provider + version: v1.0 + template: gemini + upstream: + url: https://generativelanguage.googleapis.com + auth: + type: api-key + header: x-goog-api-key + value: REPLACE_WITH_GEMINI_API_KEY + policies: + - name: api-key-auth + version: v1 + paths: + - path: /v1beta/models/{model}:generateContent + methods: [POST] + params: + key: X-API-Key + in: header + - path: /v1beta/models/{model}:streamGenerateContent + methods: [POST] + params: + key: X-API-Key + in: header + accessControl: + mode: deny_all + exceptions: + - path: /v1beta/models/{model}:generateContent + methods: [POST] + - path: /v1beta/models/{model}:streamGenerateContent + methods: [POST] + +# The proxy in front of this provider authenticates over the internal loopback +# route with an API key issued for this LlmProvider. Create one as shown in +# api-key.yaml, then set the issued value as the proxy's provider.auth.value. diff --git a/gateway/examples/mistral-openai-proxy.yaml b/gateway/examples/mistral-openai-proxy.yaml new file mode 100644 index 000000000..26d7d2960 --- /dev/null +++ b/gateway/examples/mistral-openai-proxy.yaml @@ -0,0 +1,53 @@ +# -------------------------------------------------------------------- +# Copyright (c) 2026, WSO2 LLC. (https://www.wso2.com). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# -------------------------------------------------------------------- + +# -------------------------------------------------------------------- +# OpenAI -> Mistral LLM Proxy (single-provider) +# +# Exposes an OpenAI-shaped /chat/completions endpoint that the +# `openai-to-mistral` policy adapts to Mistral's chat completions API +# and forwards to the existing `mistral-provider` upstream +# (api.mistral.ai). +# +# Single-provider mode: there is no router in front of the policy, so +# nothing sets metadata["selected_provider"] and the translator runs on +# every request. "id" is omitted, so it routes to the default upstream +# (the primary provider above). +# +# The provider must already be deployed and must have its Authorization +# bearer token set on upstream.auth (the policy does NOT inject the +# Mistral key). +# +# The provider example's api-key-auth policy expects the proxy to send the +# issued provider loopback key in X-API-Key. +# -------------------------------------------------------------------- + +apiVersion: gateway.api-platform.wso2.com/v1 +kind: LlmProxy +metadata: + name: openai-to-mistral +spec: + displayName: OpenAI to Mistral Proxy + version: v1.0 + context: /openai-mistral + provider: + id: mistral-provider + auth: + type: api-key + header: X-API-Key + value: REPLACE_WITH_MISTRAL_PROVIDER_LOOPBACK_KEY + policies: + - name: openai-to-mistral + version: v1 + paths: + - path: /chat/completions + methods: [POST] + params: + model: mistral-large-latest diff --git a/gateway/examples/mistral-provider.yaml b/gateway/examples/mistral-provider.yaml new file mode 100644 index 000000000..7f55201aa --- /dev/null +++ b/gateway/examples/mistral-provider.yaml @@ -0,0 +1,70 @@ +# -------------------------------------------------------------------- +# Copyright (c) 2026, WSO2 LLC. (https://www.wso2.com). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# -------------------------------------------------------------------- + +# -------------------------------------------------------------------- +# Mistral LLM Provider Configuration +# +# Uses the built-in `mistralai` template and points to the public +# Mistral API endpoint. Replace the placeholder with a real Mistral key. +# +# The openai-to-mistral policy on the proxy layer translates +# OpenAI-format requests into Mistral's wire format (path +# /v1/chat/completions, model override, drop unsupported fields). +# +# Auth: upstream.auth is the gateway -> Mistral vendor auth. Proxies in +# front of this provider inherit this vendor auth automatically. +# -------------------------------------------------------------------- + +apiVersion: gateway.api-platform.wso2.com/v1 +kind: LlmProvider +metadata: + name: mistral-provider +spec: + displayName: Mistral Provider + version: v1.0 + template: mistralai + upstream: + url: https://api.mistral.ai + auth: + type: api-key + header: Authorization + value: Bearer REPLACE_WITH_MISTRAL_API_KEY + policies: + - name: api-key-auth + version: v1 + paths: + - path: /v1/chat/completions + methods: [POST] + params: + key: X-API-Key + in: header + - path: /v1/embeddings + methods: [POST] + params: + key: X-API-Key + in: header + - path: /v1/models + methods: [GET] + params: + key: X-API-Key + in: header + accessControl: + mode: deny_all + exceptions: + - path: /v1/chat/completions + methods: [POST] + - path: /v1/embeddings + methods: [POST] + - path: /v1/models + methods: [GET] + +# The proxy in front of this provider authenticates over the internal loopback +# route with an API key issued for this LlmProvider. Create one as shown in +# api-key.yaml, then set the issued value as the proxy's provider.auth.value. diff --git a/gateway/examples/openai-multi-provider-proxy.yaml b/gateway/examples/openai-multi-provider-proxy.yaml new file mode 100644 index 000000000..e88f18901 --- /dev/null +++ b/gateway/examples/openai-multi-provider-proxy.yaml @@ -0,0 +1,148 @@ +# -------------------------------------------------------------------- +# Copyright (c) 2026, WSO2 LLC. (https://www.wso2.com). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# -------------------------------------------------------------------- + +# -------------------------------------------------------------------- +# OpenAI Multi-Provider Proxy +# +# Exposes one OpenAI-shaped /chat/completions endpoint and fans out to +# several LLM providers. Two modes are supported by the translators: +# +# * Single-provider mode — attach exactly one translator with no +# router in front of it. With no "selected_provider" in the request +# metadata the translator runs unconditionally. +# +# * Multi-provider mode — put openai-header-router first. It writes +# the chosen provider into metadata["selected_provider"]. Each +# translator then runs only when that selection equals its own "id". +# The "id" doubles as the upstream cluster name, so it must match an +# entry in additionalProviders (id or as). +# +# -------------------------------------------------------------------- + +apiVersion: gateway.api-platform.wso2.com/v1 +kind: LlmProxy +metadata: + name: openai-multi +spec: + displayName: OpenAI Multi-Provider Proxy + version: v1.0 + context: /openai-multi + provider: + id: openai-provider + auth: + type: api-key + header: X-API-Key + value: REPLACE_WITH_OPENAI_PROVIDER_LOOPBACK_KEY + additionalProviders: + - id: anthropic-provider + # Optional alias, for when the provider id is not readable enough to use + # in a routing policy. When set, "as" -- not "id" -- becomes the upstream + # name that the router selects and the translator matches on, so the + # mapping below would have to send x-provider: anthropic to + # "anthropic-upstream" instead. Defaults to "id" when omitted. + # as: anthropic-upstream + auth: + type: api-key + header: X-API-Key + value: REPLACE_WITH_ANTHROPIC_PROVIDER_LOOPBACK_KEY + transformer: + type: openai-to-anthropic + version: v1 + params: + model: claude-sonnet-4-5-20250929 + - id: azure-openai-provider + auth: + type: api-key + header: X-API-Key + value: REPLACE_WITH_AZURE_OPENAI_PROVIDER_LOOPBACK_KEY + transformer: + type: openai-to-azure-openai + version: v1 + params: + model: gpt-4o + apiVersion: "2024-02-15-preview" + - id: mistral-provider + auth: + type: api-key + header: X-API-Key + value: REPLACE_WITH_MISTRAL_PROVIDER_LOOPBACK_KEY + transformer: + type: openai-to-mistral + version: v1 + params: + model: mistral-large-latest + - id: gemini-provider + auth: + type: api-key + header: X-API-Key + value: REPLACE_WITH_GEMINI_PROVIDER_LOOPBACK_KEY + transformer: + type: openai-to-gemini + version: v1 + params: + model: gemini-2.5-flash + apiVersion: v1beta + policies: + - name: api-key-auth + version: v1 + paths: + - path: /chat/completions + methods: [POST] + params: + key: X-API-Key + in: header + - name: openai-header-router + version: v1 + paths: + - path: /chat/completions + methods: [POST] + params: + headerName: x-provider + defaultProvider: openai-provider + mappings: + - headerValue: openai + provider: openai-provider + - headerValue: anthropic + provider: anthropic-provider + - headerValue: azure-openai + provider: azure-openai-provider + - headerValue: mistral + provider: mistral-provider + - headerValue: gemini + provider: gemini-provider + + # - name: token-based-ratelimit + # version: v1 + # paths: + # - path: /chat/completions + # methods: [POST] + # params: + # totalTokenLimits: + # - count: 40 + # duration: "1m" + # algorithm: fixed-window + # backend: memory +--- +# Client API key for this proxy. The api-key-auth policy above validates +# the X-API-Key header against the key the gateway issues for this resource. +# apiKey is omitted here, so the gateway generates one. Read it from the +# create response (POST .../llm-proxies/openai-multi/api-keys). +apiVersion: gateway.api-platform.wso2.com/v1 +kind: ApiKey +metadata: + name: openai-multi-consumer-key +spec: + parentRef: + kind: LlmProxy + name: openai-multi + displayName: OpenAI multi-proxy consumer key + expiresIn: + duration: 30 + unit: days diff --git a/gateway/examples/openai-provider.yaml b/gateway/examples/openai-provider.yaml new file mode 100644 index 000000000..569b45bb4 --- /dev/null +++ b/gateway/examples/openai-provider.yaml @@ -0,0 +1,84 @@ +# -------------------------------------------------------------------- +# Copyright (c) 2026, WSO2 LLC. (https://www.wso2.com). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# -------------------------------------------------------------------- + +# -------------------------------------------------------------------- +# OpenAI LLM Provider Configuration +# +# Uses the built-in `openai` template and points to the OpenAI public +# endpoint. The auth value below is a SAMPLE / placeholder token -- +# replace it with a real OpenAI API key (Bearer sk-...) before deploying. +# +# Auth: upstream.auth is the gateway -> OpenAI vendor auth. This is the +# only auth this provider needs; proxies in front of it route freely and +# inherit this vendor auth automatically. +# -------------------------------------------------------------------- + +apiVersion: gateway.api-platform.wso2.com/v1 +kind: LlmProvider +metadata: + name: openai-provider +spec: + displayName: OpenAI Provider + version: v1.0 + template: openai + upstream: + url: https://api.openai.com/v1 + auth: + type: api-key + header: Authorization + # OpenAI expects: Authorization: Bearer -- sent verbatim, so the + # "Bearer " prefix MUST be included here. + value: Bearer sk-REPLACE_WITH_OPENAI_KEY + policies: + - name: api-key-auth + version: v1 + paths: + - path: /chat/completions + methods: [POST] + params: + key: X-API-Key + in: header + - path: /completions + methods: [POST] + params: + key: X-API-Key + in: header + - path: /embeddings + methods: [POST] + params: + key: X-API-Key + in: header + - path: /models + methods: [GET] + params: + key: X-API-Key + in: header + - path: /models/{modelId} + methods: [GET] + params: + key: X-API-Key + in: header + accessControl: + mode: deny_all + exceptions: + - path: /chat/completions + methods: [POST] + - path: /completions + methods: [POST] + - path: /embeddings + methods: [POST] + - path: /models + methods: [GET] + - path: /models/{modelId} + methods: [GET] + +# The proxy in front of this provider authenticates over the internal loopback +# route with an API key issued for this LlmProvider. Create one as shown in +# api-key.yaml, then set the issued value as the proxy's provider.auth.value. diff --git a/gateway/gateway-controller/api/management-openapi.yaml b/gateway/gateway-controller/api/management-openapi.yaml index b1589ec6a..3f256d95a 100644 --- a/gateway/gateway-controller/api/management-openapi.yaml +++ b/gateway/gateway-controller/api/management-openapi.yaml @@ -5978,6 +5978,64 @@ components: auth: $ref: '#/components/schemas/LLMUpstreamAuth' + LLMProxyAdditionalProvider: + type: object + required: + - id + description: > + Additional LLM provider attached to this proxy as a selectable upstream. + Policies route to it by referring to the `as` name (defaults to `id`). + Optional auth config is used by the proxy when calling a protected + LlmProvider over the internal loopback route. + properties: + id: + type: string + description: Unique id of a deployed llm provider + example: anthropic-provider + as: + type: string + description: > + Logical LLM Provider name used by policies to select this provider. Must + be unique within the proxy. Defaults to `id` when omitted. + pattern: '^[a-zA-Z0-9\-_]+$' + minLength: 1 + maxLength: 100 + example: anthropic-upstream + auth: + $ref: '#/components/schemas/LLMUpstreamAuth' + transformer: + $ref: '#/components/schemas/LLMProxyTransformer' + + LLMProxyTransformer: + type: object + required: + - type + - version + description: > + Request/response translator applied when this provider is the selected + upstream. The proxy injects the translator as a conditional policy whose + execution condition matches this provider, so it runs only when the + provider is selected. The provider's `as` name (defaults to `id`) is + passed to the translator as its target upstream. + properties: + type: + type: string + description: Translator policy name (for example openai-to-anthropic). + minLength: 1 + example: openai-to-anthropic + version: + type: string + description: > + Major-only translator policy version (for example v1). The Gateway + Controller resolves it to the installed full version. + pattern: '^v\d+$' + minLength: 1 + example: v1 + params: + type: object + description: Translator-specific parameters (for example model, apiVersion). + additionalProperties: true + LLMAccessControl: type: object required: @@ -6214,6 +6272,15 @@ components: description: Operation-level policies scoped to specific paths/methods, evaluated after global policies. items: $ref: "#/components/schemas/OperationPolicy" + additionalProviders: + type: array + description: > + Optional list of additional LLM providers attached to this proxy as + selectable upstreams. Policies (e.g. an OpenAI translator) can route + requests to any of these by setting the upstream name. The primary + `provider` field above remains the default upstream and the FK target. + items: + $ref: '#/components/schemas/LLMProxyAdditionalProvider' policies: deprecated: true type: array diff --git a/gateway/gateway-controller/pkg/api/management/generated.go b/gateway/gateway-controller/pkg/api/management/generated.go index 8353ccc2a..5eaf63020 100644 --- a/gateway/gateway-controller/pkg/api/management/generated.go +++ b/gateway/gateway-controller/pkg/api/management/generated.go @@ -976,8 +976,24 @@ type LLMProviderTemplateResourceMappings struct { Resources *[]LLMProviderTemplateResourceMapping `json:"resources,omitempty" yaml:"resources,omitempty"` } +// LLMProxyAdditionalProvider Additional LLM provider attached to this proxy as a selectable upstream. Policies route to it by referring to the `as` name (defaults to `id`). Optional auth config is used by the proxy when calling a protected LlmProvider over the internal loopback route. +type LLMProxyAdditionalProvider struct { + // As Logical upstream name used by policies to select this provider. Must be unique within the proxy. Defaults to `id` when omitted. + As *string `json:"as,omitempty" yaml:"as,omitempty"` + Auth *LLMUpstreamAuth `json:"auth,omitempty" yaml:"auth,omitempty"` + + // Id Unique id of a deployed llm provider + Id string `json:"id" yaml:"id"` + + // Transformer Request/response translator applied when this provider is the selected upstream. The proxy injects the translator as a conditional policy whose execution condition matches this provider, so it runs only when the provider is selected. The provider's `as` name (defaults to `id`) is passed to the translator as its target upstream. + Transformer *LLMProxyTransformer `json:"transformer,omitempty" yaml:"transformer,omitempty"` +} + // LLMProxyConfigData defines model for LLMProxyConfigData. type LLMProxyConfigData struct { + // AdditionalProviders Optional list of additional LLM providers attached to this proxy as selectable upstreams. Policies (e.g. an OpenAI translator) can route requests to any of these by setting the upstream name. The primary `provider` field above remains the default upstream and the FK target. + AdditionalProviders *[]LLMProxyAdditionalProvider `json:"additionalProviders,omitempty" yaml:"additionalProviders,omitempty"` + // Context Base path for all API routes (must start with /, no trailing slash) Context *string `json:"context,omitempty" yaml:"context,omitempty"` @@ -1056,6 +1072,18 @@ type LLMProxyProvider struct { Id string `json:"id" yaml:"id"` } +// LLMProxyTransformer Request/response translator applied when this provider is the selected upstream. The proxy injects the translator as a conditional policy whose execution condition matches this provider, so it runs only when the provider is selected. The provider's `as` name (defaults to `id`) is passed to the translator as its target upstream. +type LLMProxyTransformer struct { + // Params Translator-specific parameters (for example model, apiVersion). + Params *map[string]interface{} `json:"params,omitempty" yaml:"params,omitempty"` + + // Type Translator policy name (for example openai-to-anthropic). + Type string `json:"type" yaml:"type"` + + // Version Major-only translator policy version (for example v1). The Gateway Controller resolves it to the installed full version. + Version string `json:"version" yaml:"version"` +} + // LLMUpstreamAuth defines model for LLMUpstreamAuth. type LLMUpstreamAuth struct { Header *string `json:"header,omitempty" yaml:"header,omitempty"` diff --git a/gateway/gateway-controller/pkg/config/llm_validator.go b/gateway/gateway-controller/pkg/config/llm_validator.go index a33c90952..dbd7720f6 100644 --- a/gateway/gateway-controller/pkg/config/llm_validator.go +++ b/gateway/gateway-controller/pkg/config/llm_validator.go @@ -613,6 +613,55 @@ func (v *LLMValidator) validateProxyData(spec *api.LLMProxyConfigData) []Validat Message: "spec.provider.id must consist of lowercase alphanumeric characters, hyphens, or dots, and must start and end with an alphanumeric character", }) } + if spec.Provider.Auth != nil { + errors = append(errors, v.validateLLMUpstreamAuth("spec.provider.auth", spec.Provider.Auth)...) + } + + if spec.AdditionalProviders != nil { + seen := map[string]bool{spec.Provider.Id: true} + for i, provider := range *spec.AdditionalProviders { + fieldPrefix := fmt.Sprintf("spec.additionalProviders[%d]", i) + if provider.Id == "" { + errors = append(errors, ValidationError{ + Field: fieldPrefix + ".id", + Message: "Provider is required", + }) + } else if !v.metadataNameRegex.MatchString(provider.Id) { + errors = append(errors, ValidationError{ + Field: fieldPrefix + ".id", + Message: fieldPrefix + ".id must consist of lowercase alphanumeric characters, hyphens, or dots, and must start and end with an alphanumeric character", + }) + } + + upstreamName := provider.Id + if provider.As != nil && *provider.As != "" { + upstreamName = *provider.As + if !regexp.MustCompile(`^[a-zA-Z0-9\-_]+$`).MatchString(upstreamName) { + errors = append(errors, ValidationError{ + Field: fieldPrefix + ".as", + Message: fieldPrefix + ".as must contain only letters, numbers, hyphens, or underscores", + }) + } + } + if upstreamName != "" { + if seen[upstreamName] { + errors = append(errors, ValidationError{ + Field: fieldPrefix, + Message: fmt.Sprintf("duplicate upstream name '%s' in additionalProviders", upstreamName), + }) + } + seen[upstreamName] = true + } + + if provider.Auth != nil { + errors = append(errors, v.validateLLMUpstreamAuth(fieldPrefix+".auth", provider.Auth)...) + } + + if provider.Transformer != nil { + errors = append(errors, v.validateLLMProxyTransformer(fieldPrefix+".transformer", provider.Transformer)...) + } + } + } // The deprecated `policies` list must not coexist with the new policy lists errors = append(errors, v.validatePolicyListExclusivity(spec.GlobalPolicies, spec.OperationPolicies, spec.Policies)...) @@ -624,6 +673,58 @@ func (v *LLMValidator) validateProxyData(spec *api.LLMProxyConfigData) []Validat return errors } +func (v *LLMValidator) validateLLMProxyTransformer(fieldPrefix string, transformer *api.LLMProxyTransformer) []ValidationError { + var errors []ValidationError + if transformer.Type == "" { + errors = append(errors, ValidationError{ + Field: fieldPrefix + ".type", + Message: "Transformer type is required", + }) + } + if transformer.Version == "" { + errors = append(errors, ValidationError{ + Field: fieldPrefix + ".version", + Message: "Transformer version is required", + }) + } else if !majorVersionPattern.MatchString(transformer.Version) { + errors = append(errors, ValidationError{ + Field: fieldPrefix + ".version", + Message: "Transformer version must be major-only (e.g. v1)", + }) + } + return errors +} + +func (v *LLMValidator) validateLLMUpstreamAuth(fieldPrefix string, auth *api.LLMUpstreamAuth) []ValidationError { + var errors []ValidationError + if auth.Type == "" { + errors = append(errors, ValidationError{ + Field: fieldPrefix + ".type", + Message: "Auth type is required", + }) + } else if auth.Type != api.LLMUpstreamAuthTypeApiKey { + errors = append(errors, ValidationError{ + Field: fieldPrefix + ".type", + Message: "Auth type must be 'api-key'", + }) + } + if auth.Type == api.LLMUpstreamAuthTypeApiKey { + if auth.Header == nil || *auth.Header == "" { + errors = append(errors, ValidationError{ + Field: fieldPrefix + ".header", + Message: "Auth header is required when api-key auth type is set", + }) + } + if auth.Value == nil || *auth.Value == "" { + errors = append(errors, ValidationError{ + Field: fieldPrefix + ".value", + Message: "Auth value is required when api-key auth type is set", + }) + } + } + return errors +} + // validateAccessControl validates access control configuration func (v *LLMValidator) validateAccessControl(fieldPrefix string, ac *api.LLMAccessControl) []ValidationError { var errors []ValidationError diff --git a/gateway/gateway-controller/pkg/utils/llm_transformer.go b/gateway/gateway-controller/pkg/utils/llm_transformer.go index b318632cc..131d1d39d 100644 --- a/gateway/gateway-controller/pkg/utils/llm_transformer.go +++ b/gateway/gateway-controller/pkg/utils/llm_transformer.go @@ -148,6 +148,50 @@ func (t *LLMProviderTransformer) transformProxy(proxy *api.LLMProxyConfiguration spec.Upstream.Main = api.Upstream{ Url: &upstream, } + + // Step 3.1: Resolve additional providers (multi-provider proxies). Each is + // exposed as a named UpstreamDefinition so policies can route to it via + // the loopback context. The primary provider above remains the default. + if proxy.Spec.AdditionalProviders != nil && len(*proxy.Spec.AdditionalProviders) > 0 { + seen := map[string]bool{proxy.Spec.Provider.Id: true} + var defs []api.UpstreamDefinition + for _, ap := range *proxy.Spec.AdditionalProviders { + if ap.Id == "" { + return nil, fmt.Errorf("additionalProviders entry must have a non-empty id") + } + name := ap.Id + if ap.As != nil && *ap.As != "" { + name = *ap.As + } + if seen[name] { + return nil, fmt.Errorf("duplicate upstream name '%s' in additionalProviders (must be unique within the proxy and not collide with the primary provider id)", name) + } + seen[name] = true + + addCfg, err := t.db.GetConfigByKindAndHandle(string(api.LLMProviderConfigurationKindLlmProvider), ap.Id) + if err != nil { + return nil, fmt.Errorf("failed to look up additional provider '%s': %w", ap.Id, err) + } + if addCfg == nil { + return nil, fmt.Errorf("additional provider '%s' not found", ap.Id) + } + addCtx, err := addCfg.GetContext() + if err != nil { + return nil, fmt.Errorf("failed to get context for additional provider '%s': %w", ap.Id, err) + } + addURL := fmt.Sprintf("%s://%s:%d%s", + constants.SchemeHTTP, constants.LocalhostIP, t.routerConfig.ListenerPort, addCtx) + defs = append(defs, api.UpstreamDefinition{ + Name: name, + Upstreams: []struct { + Url string `json:"url" yaml:"url"` + Weight *int `json:"weight,omitempty" yaml:"weight,omitempty"` + }{{Url: addURL}}, + }) + } + spec.UpstreamDefinitions = &defs + } + // If provider has vhost configured add a host adding policy if providerConfig.Spec.Vhost != nil && *providerConfig.Spec.Vhost != "" { providerVhost := *providerConfig.Spec.Vhost @@ -182,34 +226,43 @@ func (t *LLMProviderTransformer) transformProxy(proxy *api.LLMProxyConfiguration } // Step 3.5: Apply proxy-level provider auth for proxy->provider loopback upstream - var upstreamAuthPolicy *api.Policy + // and inline translators declared per additional provider. Both are attached as + // conditional policies so they run only when their provider is selected. + var upstreamAuthPolicies []api.Policy + var transformerPolicies []api.Policy if proxy.Spec.Provider.Auth != nil { - auth := proxy.Spec.Provider.Auth - switch auth.Type { - case api.LLMUpstreamAuthTypeApiKey: - if auth.Value == nil || *auth.Value == "" { - return nil, fmt.Errorf("provider.auth.value is required") - } - header := "" - if auth.Header != nil { - header = *auth.Header - } - params, err := GetUpstreamAuthApikeyPolicyParams(header, *auth.Value) - if err != nil { - return nil, fmt.Errorf("failed to build upstream auth params: %w", err) + pol, err := t.proxyUpstreamAuthPolicy(proxy.Spec.Provider.Auth, "provider.auth") + if err != nil { + return nil, err + } + condition := selectedProviderExecutionCondition(proxy.Spec.Provider.Id, true) + pol.ExecutionCondition = &condition + upstreamAuthPolicies = append(upstreamAuthPolicies, *pol) + } + if proxy.Spec.AdditionalProviders != nil { + for _, ap := range *proxy.Spec.AdditionalProviders { + name := ap.Id + if ap.As != nil && *ap.As != "" { + name = *ap.As } - policyVersion, err := t.resolvePolicyVersion(constants.UPSTREAM_AUTH_APIKEY_POLICY_NAME) - if err != nil { - return nil, err + + if ap.Auth != nil { + pol, err := t.proxyUpstreamAuthPolicy(ap.Auth, fmt.Sprintf("additionalProviders[%s].auth", name)) + if err != nil { + return nil, err + } + condition := selectedProviderExecutionCondition(name, false) + pol.ExecutionCondition = &condition + upstreamAuthPolicies = append(upstreamAuthPolicies, *pol) } - mh := api.Policy{ - Name: constants.UPSTREAM_AUTH_APIKEY_POLICY_NAME, - Version: policyVersion, - Params: ¶ms, + + if ap.Transformer != nil { + pol, err := t.proxyTransformerPolicy(ap.Transformer, name, fmt.Sprintf("additionalProviders[%s].transformer", name)) + if err != nil { + return nil, err + } + transformerPolicies = append(transformerPolicies, *pol) } - upstreamAuthPolicy = &mh - default: - return nil, fmt.Errorf("unsupported upstream auth type: %s", auth.Type) } } @@ -284,14 +337,19 @@ func (t *LLMProviderTransformer) transformProxy(proxy *api.LLMProxyConfiguration ops = append(ops, *op) } ops = sortOperationsBySpecificity(ops) - if upstreamAuthPolicy != nil { + // Translators must run before upstream auth so the request is rewritten into + // the selected provider's shape before the upstream key is added. + if len(transformerPolicies) > 0 { for i := range ops { - if ops[i].Policies == nil { - ops[i].Policies = &[]api.Policy{*upstreamAuthPolicy} - } else { - existing := *ops[i].Policies - existing = append(existing, *upstreamAuthPolicy) - ops[i].Policies = &existing + for _, transformerPolicy := range transformerPolicies { + appendOperationPolicy(&ops[i], transformerPolicy) + } + } + } + if len(upstreamAuthPolicies) > 0 { + for i := range ops { + for _, upstreamAuthPolicy := range upstreamAuthPolicies { + appendOperationPolicy(&ops[i], upstreamAuthPolicy) } } } @@ -709,6 +767,76 @@ func GetUpstreamAuthApikeyPolicyParams(header, value string) (map[string]interfa return m, nil } +func (t *LLMProviderTransformer) proxyUpstreamAuthPolicy(auth *api.LLMUpstreamAuth, field string) (*api.Policy, error) { + if auth == nil { + return nil, nil + } + switch auth.Type { + case api.LLMUpstreamAuthTypeApiKey: + if auth.Header == nil || *auth.Header == "" { + return nil, fmt.Errorf("%s.header is required", field) + } + if auth.Value == nil || *auth.Value == "" { + return nil, fmt.Errorf("%s.value is required", field) + } + params, err := GetUpstreamAuthApikeyPolicyParams(*auth.Header, *auth.Value) + if err != nil { + return nil, fmt.Errorf("failed to build upstream auth params: %w", err) + } + policyVersion, err := t.resolvePolicyVersion(constants.UPSTREAM_AUTH_APIKEY_POLICY_NAME) + if err != nil { + return nil, err + } + return &api.Policy{ + Name: constants.UPSTREAM_AUTH_APIKEY_POLICY_NAME, + Version: policyVersion, + Params: ¶ms, + }, nil + default: + return nil, fmt.Errorf("unsupported upstream auth type: %s", auth.Type) + } +} + +// proxyTransformerPolicy builds a translator policy for an additional provider's +// inline transformer. The provider's upstream name is passed to the translator as +// its "id" param so it targets the correct upstream, and gates execution so the +// translator runs only when this provider is the selected upstream. +func (t *LLMProviderTransformer) proxyTransformerPolicy(transformer *api.LLMProxyTransformer, name, field string) (*api.Policy, error) { + if transformer == nil { + return nil, nil + } + if transformer.Type == "" { + return nil, fmt.Errorf("%s.type is required", field) + } + if transformer.Version == "" { + return nil, fmt.Errorf("%s.version is required", field) + } + + params := map[string]interface{}{} + if transformer.Params != nil { + for k, v := range *transformer.Params { + params[k] = v + } + } + params["id"] = name + + condition := selectedProviderExecutionCondition(name, false) + return &api.Policy{ + Name: transformer.Type, + Version: transformer.Version, + Params: ¶ms, + ExecutionCondition: &condition, + }, nil +} + +func selectedProviderExecutionCondition(providerName string, includeDefault bool) string { + selectedExpr := fmt.Sprintf("request.Metadata['selected_provider'] == '%s'", providerName) + if includeDefault { + return fmt.Sprintf("!('selected_provider' in request.Metadata) || %s", selectedExpr) + } + return fmt.Sprintf("'selected_provider' in request.Metadata && %s", selectedExpr) +} + // GetHostAdditionPolicyParams renders the policy params with given host value (host-rewrite) func GetHostAdditionPolicyParams(value string) (map[string]interface{}, error) { rendered := fmt.Sprintf(constants.PROXY_HOST__HEADER_POLICY_PARAMS, value) diff --git a/gateway/gateway-controller/pkg/utils/llm_transformer_multiprovider_test.go b/gateway/gateway-controller/pkg/utils/llm_transformer_multiprovider_test.go new file mode 100644 index 000000000..9d558dfaf --- /dev/null +++ b/gateway/gateway-controller/pkg/utils/llm_transformer_multiprovider_test.go @@ -0,0 +1,258 @@ +/* + * Copyright (c) 2025, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package utils + +import ( + "io" + "log/slog" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + api "github.com/wso2/api-platform/gateway/gateway-controller/pkg/api/management" + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/config" + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/constants" + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/models" + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/storage" +) + +func TestLLMProviderTransformer_TransformProxy_AdditionalProviderAuthIsConditional(t *testing.T) { + store := storage.NewConfigStore() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + db := newTestSQLiteStorage(t, logger) + + template := &models.StoredLLMProviderTemplate{ + UUID: "0000-db-template-id-0000-000000000002", + Configuration: api.LLMProviderTemplate{ + ApiVersion: api.LLMProviderTemplateApiVersionGatewayApiPlatformWso2Comv1, + Kind: api.LLMProviderTemplateKindLlmProviderTemplate, + Metadata: api.Metadata{Name: "openai"}, + Spec: api.LLMProviderTemplateData{DisplayName: "openai"}, + }, + } + require.NoError(t, db.SaveLLMProviderTemplate(template)) + + saveProvider := func(name, context string) { + providerSourceConfig := api.LLMProviderConfiguration{ + ApiVersion: api.LLMProviderConfigurationApiVersionGatewayApiPlatformWso2Comv1, + Kind: api.LLMProviderConfigurationKindLlmProvider, + Metadata: api.Metadata{Name: name}, + Spec: api.LLMProviderConfigData{ + DisplayName: name, + Version: "v1.0", + Context: stringPtr(context), + Template: "openai", + Upstream: api.LLMProviderConfigData_Upstream{Url: stringPtr("https://example.com")}, + AccessControl: api.LLMAccessControl{Mode: api.AllowAll}, + }, + } + require.NoError(t, db.SaveConfig(&models.StoredConfig{ + UUID: name + "-uuid", + Kind: string(api.LLMProviderConfigurationKindLlmProvider), + Handle: name, + DisplayName: name, + Version: "v1.0", + SourceConfiguration: providerSourceConfig, + DesiredState: models.StateDeployed, + })) + } + saveProvider("openai-provider", "/openai-provider") + saveProvider("anthropic-provider", "/anthropic-provider") + + transformer := NewLLMProviderTransformer(store, db, &config.RouterConfig{ListenerPort: 8080}, newTestPolicyVersionResolver()) + + proxy := &api.LLMProxyConfiguration{ + ApiVersion: api.LLMProxyConfigurationApiVersionGatewayApiPlatformWso2Comv1, + Kind: api.LLMProxyConfigurationKindLlmProxy, + Metadata: api.Metadata{Name: "openai-multi"}, + Spec: api.LLMProxyConfigData{ + DisplayName: "openai-multi", + Version: "v1.0", + Provider: api.LLMProxyProvider{ + Id: "openai-provider", + Auth: &api.LLMUpstreamAuth{ + Type: api.LLMUpstreamAuthTypeApiKey, + Header: stringPtr("Authorization"), + Value: stringPtr("Bearer primary"), + }, + }, + AdditionalProviders: &[]api.LLMProxyAdditionalProvider{{ + Id: "anthropic-provider", + Auth: &api.LLMUpstreamAuth{ + Type: api.LLMUpstreamAuthTypeApiKey, + Header: stringPtr("X-Provider-Key"), + Value: stringPtr("anthropic-loopback"), + }, + }}, + Policies: &[]api.LLMPolicy{{ + Name: "openai-header-router", + Version: "v1", + Paths: []api.LLMPolicyPath{{ + Path: "/chat/completions", + Methods: []api.LLMPolicyPathMethods{"POST"}, + Params: map[string]interface{}{ + "defaultProvider": "openai-provider", + }, + }}, + }}, + }, + } + + result, err := transformer.Transform(proxy, &api.RestAPI{}) + require.NoError(t, err) + require.NotNil(t, result.Spec.UpstreamDefinitions) + require.Len(t, *result.Spec.UpstreamDefinitions, 1) + assert.Equal(t, "anthropic-provider", (*result.Spec.UpstreamDefinitions)[0].Name) + + var chatOp *api.Operation + for i := range result.Spec.Operations { + if result.Spec.Operations[i].Path != nil && *result.Spec.Operations[i].Path == "/chat/completions" && + result.Spec.Operations[i].Method != nil && *result.Spec.Operations[i].Method == api.OperationMethod("POST") { + chatOp = &result.Spec.Operations[i] + break + } + } + require.NotNil(t, chatOp) + require.NotNil(t, chatOp.Policies) + + var authPolicies []api.Policy + for _, pol := range *chatOp.Policies { + if pol.Name == constants.UPSTREAM_AUTH_APIKEY_POLICY_NAME { + authPolicies = append(authPolicies, pol) + } + } + require.Len(t, authPolicies, 2) + require.NotNil(t, authPolicies[0].ExecutionCondition) + require.NotNil(t, authPolicies[1].ExecutionCondition) + assert.Contains(t, *authPolicies[0].ExecutionCondition, "openai-provider") + assert.Contains(t, *authPolicies[1].ExecutionCondition, "anthropic-provider") + assert.Equal(t, "Bearer primary", firstRequestHeaderValue(t, authPolicies[0].Params)) + assert.Equal(t, "anthropic-loopback", firstRequestHeaderValue(t, authPolicies[1].Params)) +} + +func TestLLMProviderTransformer_TransformProxy_AdditionalProviderTransformerIsConditional(t *testing.T) { + store := storage.NewConfigStore() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + db := newTestSQLiteStorage(t, logger) + + template := &models.StoredLLMProviderTemplate{ + UUID: "0000-db-template-id-0000-000000000003", + Configuration: api.LLMProviderTemplate{ + ApiVersion: api.LLMProviderTemplateApiVersionGatewayApiPlatformWso2Comv1, + Kind: api.LLMProviderTemplateKindLlmProviderTemplate, + Metadata: api.Metadata{Name: "openai"}, + Spec: api.LLMProviderTemplateData{DisplayName: "openai"}, + }, + } + require.NoError(t, db.SaveLLMProviderTemplate(template)) + + saveProvider := func(name, context string) { + providerSourceConfig := api.LLMProviderConfiguration{ + ApiVersion: api.LLMProviderConfigurationApiVersionGatewayApiPlatformWso2Comv1, + Kind: api.LLMProviderConfigurationKindLlmProvider, + Metadata: api.Metadata{Name: name}, + Spec: api.LLMProviderConfigData{ + DisplayName: name, + Version: "v1.0", + Context: stringPtr(context), + Template: "openai", + Upstream: api.LLMProviderConfigData_Upstream{Url: stringPtr("https://example.com")}, + AccessControl: api.LLMAccessControl{Mode: api.AllowAll}, + }, + } + require.NoError(t, db.SaveConfig(&models.StoredConfig{ + UUID: name + "-uuid", + Kind: string(api.LLMProviderConfigurationKindLlmProvider), + Handle: name, + DisplayName: name, + Version: "v1.0", + SourceConfiguration: providerSourceConfig, + DesiredState: models.StateDeployed, + })) + } + saveProvider("openai-provider", "/openai-provider") + saveProvider("anthropic-provider", "/anthropic-provider") + + transformer := NewLLMProviderTransformer(store, db, &config.RouterConfig{ListenerPort: 8080}, newTestPolicyVersionResolver()) + + proxy := &api.LLMProxyConfiguration{ + ApiVersion: api.LLMProxyConfigurationApiVersionGatewayApiPlatformWso2Comv1, + Kind: api.LLMProxyConfigurationKindLlmProxy, + Metadata: api.Metadata{Name: "openai-multi"}, + Spec: api.LLMProxyConfigData{ + DisplayName: "openai-multi", + Version: "v1.0", + Provider: api.LLMProxyProvider{Id: "openai-provider"}, + AdditionalProviders: &[]api.LLMProxyAdditionalProvider{{ + Id: "anthropic-provider", + Transformer: &api.LLMProxyTransformer{ + Type: "openai-to-anthropic", + Version: "v1", + Params: &map[string]interface{}{ + "model": "claude-sonnet-4-5-20250929", + }, + }, + }}, + }, + } + + result, err := transformer.Transform(proxy, &api.RestAPI{}) + require.NoError(t, err) + + // The translator is attached conditionally to every operation, so locate it + // wherever it lands rather than assuming a specific route. + var transformerPolicy *api.Policy + for i := range result.Spec.Operations { + op := result.Spec.Operations[i] + if op.Policies == nil { + continue + } + for j := range *op.Policies { + if (*op.Policies)[j].Name == "openai-to-anthropic" { + transformerPolicy = &(*op.Policies)[j] + break + } + } + if transformerPolicy != nil { + break + } + } + require.NotNil(t, transformerPolicy) + assert.Equal(t, "v1", transformerPolicy.Version) + require.NotNil(t, transformerPolicy.ExecutionCondition) + assert.Contains(t, *transformerPolicy.ExecutionCondition, "anthropic-provider") + require.NotNil(t, transformerPolicy.Params) + assert.Equal(t, "anthropic-provider", (*transformerPolicy.Params)["id"]) + assert.Equal(t, "claude-sonnet-4-5-20250929", (*transformerPolicy.Params)["model"]) +} + +func firstRequestHeaderValue(t *testing.T, params *map[string]interface{}) string { + t.Helper() + require.NotNil(t, params) + request, ok := (*params)["request"].(map[string]interface{}) + require.True(t, ok) + headers, ok := request["headers"].([]interface{}) + require.True(t, ok) + require.NotEmpty(t, headers) + header, ok := headers[0].(map[string]interface{}) + require.True(t, ok) + value, ok := header["value"].(string) + require.True(t, ok) + return value +} diff --git a/kubernetes/gateway-operator/api/v1alpha1/llmproxy_types.go b/kubernetes/gateway-operator/api/v1alpha1/llmproxy_types.go index 62aac873c..82b1fc7aa 100644 --- a/kubernetes/gateway-operator/api/v1alpha1/llmproxy_types.go +++ b/kubernetes/gateway-operator/api/v1alpha1/llmproxy_types.go @@ -18,6 +18,7 @@ package v1alpha1 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" ) // LLMProxyProvider references a deployed LlmProvider that this proxy fronts. @@ -32,6 +33,51 @@ type LLMProxyProvider struct { Auth *LLMUpstreamAuth `json:"auth,omitempty"` } +// LLMProxyAdditionalProvider references an additional LlmProvider that this +// proxy can route to by policy-selected upstream name. +type LLMProxyAdditionalProvider struct { + // Id is the LlmProvider handle (metadata.name). + // +kubebuilder:validation:Required + Id string `json:"id"` + + // As is the logical upstream name used by policies. Defaults to Id. + // +optional + As *string `json:"as,omitempty"` + + // Auth optionally configures credentials for proxy-to-provider loopback + // calls when the referenced provider is protected by an auth policy. + // +optional + Auth *LLMUpstreamAuth `json:"auth,omitempty"` + + // Transformer optionally applies a request/response translator when this + // provider is the selected upstream. The proxy injects it as a conditional + // policy that runs only when this provider is selected. + // +optional + Transformer *LLMProxyTransformer `json:"transformer,omitempty"` +} + +// LLMProxyTransformer is a request/response translator applied when its owning +// provider is the selected upstream; mirrors the management-API +// LLMProxyTransformer payload. +type LLMProxyTransformer struct { + // Type is the translator policy name (for example openai-to-anthropic). + // +kubebuilder:validation:Required + Type string `json:"type"` + + // Version is the major-only translator policy version (for example v1). + // The Gateway Controller resolves it to the installed full version. + // +kubebuilder:validation:Required + // +kubebuilder:validation:Pattern=`^v\d+$` + Version string `json:"version"` + + // Params carries translator-specific parameters (for example model, + // apiVersion). + // +optional + // +kubebuilder:pruning:PreserveUnknownFields + // +kubebuilder:validation:Schemaless + Params *runtime.RawExtension `json:"params,omitempty"` +} + // LLMProxyConfigData mirrors the management-API LLMProxyConfigData payload. type LLMProxyConfigData struct { // DisplayName is a human-readable LLM proxy name. @@ -47,6 +93,11 @@ type LLMProxyConfigData struct { // +kubebuilder:validation:Required Provider LLMProxyProvider `json:"provider"` + // AdditionalProviders are extra LLM providers attached as selectable + // upstreams for multi-provider routing. + // +optional + AdditionalProviders []LLMProxyAdditionalProvider `json:"additionalProviders,omitempty"` + // Context is the base path for routes (must start with /). // +optional // +kubebuilder:validation:Pattern=`^/[a-zA-Z0-9\-._~!$&'()*+,;=:@%/]*[^/]$` diff --git a/kubernetes/gateway-operator/api/v1alpha1/zz_generated.deepcopy.go b/kubernetes/gateway-operator/api/v1alpha1/zz_generated.deepcopy.go index 23be55a2d..41b7e7c26 100644 --- a/kubernetes/gateway-operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/kubernetes/gateway-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -985,10 +985,47 @@ func (in *LLMProviderUpstream) DeepCopy() *LLMProviderUpstream { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LLMProxyAdditionalProvider) DeepCopyInto(out *LLMProxyAdditionalProvider) { + *out = *in + if in.As != nil { + in, out := &in.As, &out.As + *out = new(string) + **out = **in + } + if in.Auth != nil { + in, out := &in.Auth, &out.Auth + *out = new(LLMUpstreamAuth) + (*in).DeepCopyInto(*out) + } + if in.Transformer != nil { + in, out := &in.Transformer, &out.Transformer + *out = new(LLMProxyTransformer) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LLMProxyAdditionalProvider. +func (in *LLMProxyAdditionalProvider) DeepCopy() *LLMProxyAdditionalProvider { + if in == nil { + return nil + } + out := new(LLMProxyAdditionalProvider) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LLMProxyConfigData) DeepCopyInto(out *LLMProxyConfigData) { *out = *in in.Provider.DeepCopyInto(&out.Provider) + if in.AdditionalProviders != nil { + in, out := &in.AdditionalProviders, &out.AdditionalProviders + *out = make([]LLMProxyAdditionalProvider, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.Context != nil { in, out := &in.Context, &out.Context *out = new(string) @@ -1048,6 +1085,26 @@ func (in *LLMProxyProvider) DeepCopy() *LLMProxyProvider { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LLMProxyTransformer) DeepCopyInto(out *LLMProxyTransformer) { + *out = *in + if in.Params != nil { + in, out := &in.Params, &out.Params + *out = new(runtime.RawExtension) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LLMProxyTransformer. +func (in *LLMProxyTransformer) DeepCopy() *LLMProxyTransformer { + if in == nil { + return nil + } + out := new(LLMProxyTransformer) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LLMUpstreamAuth) DeepCopyInto(out *LLMUpstreamAuth) { *out = *in diff --git a/kubernetes/gateway-operator/config/crd/bases/gateway.api-platform.wso2.com_llmproxies.yaml b/kubernetes/gateway-operator/config/crd/bases/gateway.api-platform.wso2.com_llmproxies.yaml index 81da02ebd..abb9ccb13 100644 --- a/kubernetes/gateway-operator/config/crd/bases/gateway.api-platform.wso2.com_llmproxies.yaml +++ b/kubernetes/gateway-operator/config/crd/bases/gateway.api-platform.wso2.com_llmproxies.yaml @@ -41,6 +41,111 @@ spec: spec: description: Spec is the LLM proxy configuration payload. properties: + additionalProviders: + description: |- + AdditionalProviders are extra LLM providers attached as selectable + upstreams for multi-provider routing. + items: + description: |- + LLMProxyAdditionalProvider references an additional LlmProvider that this + proxy can route to by policy-selected upstream name. + properties: + as: + description: As is the logical upstream name used by policies. + Defaults to Id. + type: string + auth: + description: |- + Auth optionally configures credentials for proxy-to-provider loopback + calls when the referenced provider is protected by an auth policy. + properties: + header: + description: |- + Header is the HTTP header to set on outbound requests when the auth + type uses headers (e.g. api-key). + type: string + type: + description: Type identifies the auth scheme. Only api-key + is currently supported. + enum: + - api-key + type: string + value: + description: |- + Value sources the credential. Exactly one of value or valueFrom must + be set. + properties: + value: + description: |- + Value is the inline plaintext value. Avoid for production use; prefer + ValueFrom so the secret is stored in a Kubernetes Secret. + type: string + valueFrom: + description: |- + ValueFrom selects a key from a Kubernetes Secret in the same namespace + as the owning CR. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + x-kubernetes-validations: + - message: exactly one of value or valueFrom must be set + rule: has(self.value) != has(self.valueFrom) + required: + - type + - value + type: object + id: + description: Id is the LlmProvider handle (metadata.name). + type: string + transformer: + description: |- + Transformer optionally applies a request/response translator when this + provider is the selected upstream. The proxy injects it as a conditional + policy that runs only when this provider is selected. + properties: + params: + description: |- + Params carries translator-specific parameters (for example model, + apiVersion). + x-kubernetes-preserve-unknown-fields: true + type: + description: Type is the translator policy name (for example + openai-to-anthropic). + type: string + version: + description: |- + Version is the major-only translator policy version (for example v1). + The Gateway Controller resolves it to the installed full version. + pattern: ^v\d+$ + type: string + required: + - type + - version + type: object + required: + - id + type: object + type: array context: description: Context is the base path for routes (must start with /). diff --git a/kubernetes/gateway-operator/internal/controller/llmproxy_controller.go b/kubernetes/gateway-operator/internal/controller/llmproxy_controller.go index 96405afa0..99eb1f24b 100644 --- a/kubernetes/gateway-operator/internal/controller/llmproxy_controller.go +++ b/kubernetes/gateway-operator/internal/controller/llmproxy_controller.go @@ -159,20 +159,38 @@ func (a *llmProxyAdapter) Deploy(ctx context.Context, k8sClient client.Client, g } specPayload := interface{}(spec) - if spec.Provider.Auth != nil { - resolved, err := resolveLLMProviderUpstreamAuth(ctx, k8sClient, cr.Namespace, spec.Provider.Auth, "spec.provider.auth.value") - if err != nil { - return DeployResult{}, err - } - if resolved == nil { - return DeployResult{}, &gatewayclient.NonRetryableError{Err: fmt.Errorf("internal: provider.auth resolution produced nil")} - } + if spec.Provider.Auth != nil || len(spec.AdditionalProviders) > 0 { m, err := specToJSONMap(spec) if err != nil { return DeployResult{}, &gatewayclient.NonRetryableError{Err: err} } - if err := flattenUpstreamAuthCredentialValue(m, "provider", *resolved); err != nil { - return DeployResult{}, &gatewayclient.NonRetryableError{Err: err} + if spec.Provider.Auth != nil { + resolved, err := resolveLLMProviderUpstreamAuth(ctx, k8sClient, cr.Namespace, spec.Provider.Auth, "spec.provider.auth.value") + if err != nil { + return DeployResult{}, err + } + if resolved == nil { + return DeployResult{}, &gatewayclient.NonRetryableError{Err: fmt.Errorf("internal: provider.auth resolution produced nil")} + } + if err := flattenUpstreamAuthCredentialValue(m, "provider", *resolved); err != nil { + return DeployResult{}, &gatewayclient.NonRetryableError{Err: err} + } + } + for i := range spec.AdditionalProviders { + if spec.AdditionalProviders[i].Auth == nil { + continue + } + fieldPath := fmt.Sprintf("spec.additionalProviders[%d].auth.value", i) + resolved, err := resolveLLMProviderUpstreamAuth(ctx, k8sClient, cr.Namespace, spec.AdditionalProviders[i].Auth, fieldPath) + if err != nil { + return DeployResult{}, err + } + if resolved == nil { + return DeployResult{}, &gatewayclient.NonRetryableError{Err: fmt.Errorf("internal: additionalProviders[%d].auth resolution produced nil", i)} + } + if err := flattenAdditionalProviderAuthCredentialValue(m, i, *resolved); err != nil { + return DeployResult{}, &gatewayclient.NonRetryableError{Err: err} + } } specPayload = m } diff --git a/kubernetes/gateway-operator/internal/controller/llmproxy_controller_test.go b/kubernetes/gateway-operator/internal/controller/llmproxy_controller_test.go index 2a017a92e..f6cec1c13 100644 --- a/kubernetes/gateway-operator/internal/controller/llmproxy_controller_test.go +++ b/kubernetes/gateway-operator/internal/controller/llmproxy_controller_test.go @@ -92,3 +92,86 @@ func TestLlmProxyDeploy_ResolvesPolicyParamsValueFrom(t *testing.T) { require.NotContains(t, got, "valueFrom") require.True(t, strings.Contains(got, "name: api-key-auth")) } + +func TestLlmProxyDeploy_ResolvesAdditionalProviderAuthValueFrom(t *testing.T) { + scheme := runtime.NewScheme() + utilruntime.Must(corev1.AddToScheme(scheme)) + utilruntime.Must(apiv1.AddToScheme(scheme)) + + sec := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "apigateway-demo", Name: "provider-secrets"}, + Data: map[string][]byte{ + "primary": []byte("Bearer primary-key"), + "additional": []byte("Bearer additional-key"), + }, + } + k8sClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(sec).Build() + + header := "Authorization" + authType := "api-key" + cr := &apiv1.LlmProxy{ + ObjectMeta: metav1.ObjectMeta{Name: "demo-llm-proxy", Namespace: "apigateway-demo"}, + Spec: apiv1.LLMProxyConfigData{ + DisplayName: "Lifecycle LLM Proxy", + Version: "v1.0", + Provider: apiv1.LLMProxyProvider{ + Id: "openai-provider", + Auth: &apiv1.LLMUpstreamAuth{ + Type: authType, + Header: &header, + Value: apiv1.SecretValueSource{ValueFrom: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "provider-secrets"}, + Key: "primary", + }}, + }, + }, + AdditionalProviders: []apiv1.LLMProxyAdditionalProvider{{ + Id: "anthropic-provider", + Auth: &apiv1.LLMUpstreamAuth{ + Type: authType, + Header: &header, + Value: apiv1.SecretValueSource{ValueFrom: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "provider-secrets"}, + Key: "additional", + }}, + }, + }}, + }, + } + + var ( + mu sync.Mutex + payload string + ) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte("not found")) + return + } + if r.Method == http.MethodPost { + b, _ := io.ReadAll(r.Body) + mu.Lock() + payload = string(b) + mu.Unlock() + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte("{}")) + return + } + w.WriteHeader(http.StatusMethodNotAllowed) + })) + defer srv.Close() + + adapter := &llmProxyAdapter{} + _, err := adapter.Deploy(context.Background(), k8sClient, srv.URL, cr, nil) + require.NoError(t, err) + + mu.Lock() + got := payload + mu.Unlock() + require.Contains(t, got, "value: Bearer primary-key") + require.Contains(t, got, "value: Bearer additional-key") + require.Contains(t, got, "additionalProviders:") + require.NotContains(t, got, "secretKeyRef") + require.NotContains(t, got, "valueFrom") +} diff --git a/kubernetes/gateway-operator/internal/controller/management_upstream_auth_payload.go b/kubernetes/gateway-operator/internal/controller/management_upstream_auth_payload.go index e32c01e77..ffe5f609f 100644 --- a/kubernetes/gateway-operator/internal/controller/management_upstream_auth_payload.go +++ b/kubernetes/gateway-operator/internal/controller/management_upstream_auth_payload.go @@ -42,3 +42,23 @@ func flattenUpstreamAuthCredentialValue(specMap map[string]interface{}, parentKe auth["value"] = plaintext return nil } + +func flattenAdditionalProviderAuthCredentialValue(specMap map[string]interface{}, index int, plaintext string) error { + additionalProviders, ok := specMap["additionalProviders"].([]interface{}) + if !ok { + return fmt.Errorf("spec.additionalProviders must be an array") + } + if index < 0 || index >= len(additionalProviders) { + return fmt.Errorf("spec.additionalProviders[%d] is out of range", index) + } + parent, ok := additionalProviders[index].(map[string]interface{}) + if !ok { + return fmt.Errorf("spec.additionalProviders[%d] must be an object", index) + } + auth, ok := parent["auth"].(map[string]interface{}) + if !ok { + return fmt.Errorf("spec.additionalProviders[%d].auth must be an object", index) + } + auth["value"] = plaintext + return nil +} diff --git a/kubernetes/gateway-operator/internal/controller/management_valuefrom_enqueue.go b/kubernetes/gateway-operator/internal/controller/management_valuefrom_enqueue.go index 5d32bbd55..7760063f4 100644 --- a/kubernetes/gateway-operator/internal/controller/management_valuefrom_enqueue.go +++ b/kubernetes/gateway-operator/internal/controller/management_valuefrom_enqueue.go @@ -218,6 +218,14 @@ func llmProxyReferencesSecret(cr *apiv1.LlmProxy, secretNS, secretName string) b return true } } + for i := range cr.Spec.AdditionalProviders { + auth := cr.Spec.AdditionalProviders[i].Auth + if auth != nil && auth.Value.ValueFrom != nil { + if cr.Namespace == secretNS && strings.TrimSpace(auth.Value.ValueFrom.Name) == secretName { + return true + } + } + } return llmProxyReferencesValueFromKind(cr, secretKeyRefKey, secretNS, secretName) } diff --git a/kubernetes/gateway-operator/internal/controller/management_valuefrom_fingerprint.go b/kubernetes/gateway-operator/internal/controller/management_valuefrom_fingerprint.go index 4857a40a7..16bd4a42d 100644 --- a/kubernetes/gateway-operator/internal/controller/management_valuefrom_fingerprint.go +++ b/kubernetes/gateway-operator/internal/controller/management_valuefrom_fingerprint.go @@ -110,6 +110,11 @@ func llmProxyExternalDepsFingerprint(ctx context.Context, c client.Client, cr *a if cr.Spec.Provider.Auth != nil { accumulateBackingFromSecretValueSource(cr.Spec.Provider.Auth.Value, cr.Namespace, backing) } + for i := range cr.Spec.AdditionalProviders { + if cr.Spec.AdditionalProviders[i].Auth != nil { + accumulateBackingFromSecretValueSource(cr.Spec.AdditionalProviders[i].Auth.Value, cr.Namespace, backing) + } + } for i := range cr.Spec.Policies { for j := range cr.Spec.Policies[i].Paths { if p := cr.Spec.Policies[i].Paths[j].Params; p != nil { diff --git a/kubernetes/helm/operator-helm-chart/crds/gateway.api-platform.wso2.com_llmproxies.yaml b/kubernetes/helm/operator-helm-chart/crds/gateway.api-platform.wso2.com_llmproxies.yaml index 81da02ebd..abb9ccb13 100644 --- a/kubernetes/helm/operator-helm-chart/crds/gateway.api-platform.wso2.com_llmproxies.yaml +++ b/kubernetes/helm/operator-helm-chart/crds/gateway.api-platform.wso2.com_llmproxies.yaml @@ -41,6 +41,111 @@ spec: spec: description: Spec is the LLM proxy configuration payload. properties: + additionalProviders: + description: |- + AdditionalProviders are extra LLM providers attached as selectable + upstreams for multi-provider routing. + items: + description: |- + LLMProxyAdditionalProvider references an additional LlmProvider that this + proxy can route to by policy-selected upstream name. + properties: + as: + description: As is the logical upstream name used by policies. + Defaults to Id. + type: string + auth: + description: |- + Auth optionally configures credentials for proxy-to-provider loopback + calls when the referenced provider is protected by an auth policy. + properties: + header: + description: |- + Header is the HTTP header to set on outbound requests when the auth + type uses headers (e.g. api-key). + type: string + type: + description: Type identifies the auth scheme. Only api-key + is currently supported. + enum: + - api-key + type: string + value: + description: |- + Value sources the credential. Exactly one of value or valueFrom must + be set. + properties: + value: + description: |- + Value is the inline plaintext value. Avoid for production use; prefer + ValueFrom so the secret is stored in a Kubernetes Secret. + type: string + valueFrom: + description: |- + ValueFrom selects a key from a Kubernetes Secret in the same namespace + as the owning CR. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + x-kubernetes-validations: + - message: exactly one of value or valueFrom must be set + rule: has(self.value) != has(self.valueFrom) + required: + - type + - value + type: object + id: + description: Id is the LlmProvider handle (metadata.name). + type: string + transformer: + description: |- + Transformer optionally applies a request/response translator when this + provider is the selected upstream. The proxy injects it as a conditional + policy that runs only when this provider is selected. + properties: + params: + description: |- + Params carries translator-specific parameters (for example model, + apiVersion). + x-kubernetes-preserve-unknown-fields: true + type: + description: Type is the translator policy name (for example + openai-to-anthropic). + type: string + version: + description: |- + Version is the major-only translator policy version (for example v1). + The Gateway Controller resolves it to the installed full version. + pattern: ^v\d+$ + type: string + required: + - type + - version + type: object + required: + - id + type: object + type: array context: description: Context is the base path for routes (must start with /). diff --git a/platform-api/api/generated.go b/platform-api/api/generated.go index 96fd0f52f..66520fad2 100644 --- a/platform-api/api/generated.go +++ b/platform-api/api/generated.go @@ -1559,6 +1559,9 @@ type LLMProviderTemplateResourceMappings struct { // LLMProxy defines model for LLMProxy. type LLMProxy struct { + // AdditionalProviders Optional list of additional LLM providers attached to this proxy as selectable upstreams. Policies route requests to any of these by setting the upstream name. The primary `provider` field above remains the default upstream and the FK target. + AdditionalProviders *[]LLMProxyAdditionalProvider `json:"additionalProviders,omitempty" yaml:"additionalProviders,omitempty"` + // AssociatedGateways Optional list of gateways this LLM proxy can be deployed to, along with per-gateway configuration overrides. This field is optional; omitting it does not change existing behaviour. AssociatedGateways *[]AssociatedGateway `json:"associatedGateways,omitempty" yaml:"associatedGateways,omitempty"` @@ -1626,6 +1629,18 @@ type LLMProxyAPIKeyListResponse struct { Pagination Pagination `json:"pagination" yaml:"pagination"` } +// LLMProxyAdditionalProvider Additional LLM provider attached to this proxy as a selectable upstream. Policies route to it by referring to the `as` name (defaults to `id`). +type LLMProxyAdditionalProvider struct { + // As Logical LLM Provider name used by policies to select this provider. Must be unique within the proxy. Defaults to `id` when omitted. + As *string `json:"as,omitempty" yaml:"as,omitempty"` + + // Id Unique id of a deployed llm provider + Id string `binding:"required" json:"id" yaml:"id"` + + // Transformer Request/response translator applied when this provider is the selected upstream. The proxy injects the translator as a conditional policy whose execution condition matches this provider, so it runs only when the provider is selected. The provider's `as` name (defaults to `id`) is passed to the translator as its target upstream. + Transformer *LLMProxyTransformer `json:"transformer,omitempty" yaml:"transformer,omitempty"` +} + // LLMProxyListItem defines model for LLMProxyListItem. type LLMProxyListItem struct { // Context Context path where the proxy is exposed @@ -1672,6 +1687,18 @@ type LLMProxyProvider struct { Id string `binding:"required" json:"id" yaml:"id"` } +// LLMProxyTransformer Request/response translator applied when this provider is the selected upstream. The proxy injects the translator as a conditional policy whose execution condition matches this provider, so it runs only when the provider is selected. The provider's `as` name (defaults to `id`) is passed to the translator as its target upstream. +type LLMProxyTransformer struct { + // Params Translator-specific parameters (for example model, apiVersion). + Params *map[string]interface{} `json:"params,omitempty" yaml:"params,omitempty"` + + // Type Translator policy name (for example openai-to-anthropic). + Type string `binding:"required" json:"type" yaml:"type"` + + // Version Major-only translator policy version (for example v1). The Gateway Controller resolves it to the installed full version. + Version string `binding:"required" json:"version" yaml:"version"` +} + // LLMRateLimitingConfig Rate limiting configuration for an LLM provider at provider and consumer levels. type LLMRateLimitingConfig struct { // ConsumerLevel Rate limiting configuration for a scope (provider or consumer). Either global or resource-wise limits can be defined. diff --git a/platform-api/internal/dto/llm_deployment.go b/platform-api/internal/dto/llm_deployment.go index a8ac0b70d..ff17c565b 100644 --- a/platform-api/internal/dto/llm_deployment.go +++ b/platform-api/internal/dto/llm_deployment.go @@ -61,17 +61,24 @@ type LLMProxyDeploymentYAML struct { // LLMProxyDeploymentSpec represents the spec section for LLM proxy deployments type LLMProxyDeploymentSpec struct { - DisplayName string `yaml:"displayName"` - Version string `yaml:"version"` - Context string `yaml:"context,omitempty"` - VHost string `yaml:"vhost,omitempty"` - Provider LLMProxyDeploymentProvider `yaml:"provider"` - GlobalPolicies []api.Policy `yaml:"globalPolicies,omitempty"` - OperationPolicies []api.OperationPolicy `yaml:"operationPolicies,omitempty"` - Policies []api.LLMPolicy `yaml:"policies,omitempty"` + DisplayName string `yaml:"displayName"` + Version string `yaml:"version"` + Context string `yaml:"context,omitempty"` + VHost string `yaml:"vhost,omitempty"` + Provider LLMProxyDeploymentProvider `yaml:"provider"` + AdditionalProviders []LLMProxyDeploymentAdditionalProvider `yaml:"additionalProviders,omitempty"` + GlobalPolicies []api.Policy `yaml:"globalPolicies,omitempty"` + OperationPolicies []api.OperationPolicy `yaml:"operationPolicies,omitempty"` + Policies []api.LLMPolicy `yaml:"policies,omitempty"` } type LLMProxyDeploymentProvider struct { ID string `yaml:"id"` Auth *api.UpstreamAuth `yaml:"auth,omitempty"` } + +type LLMProxyDeploymentAdditionalProvider struct { + ID string `yaml:"id"` + As string `yaml:"as,omitempty"` + Transformer *api.LLMProxyTransformer `yaml:"transformer,omitempty"` +} diff --git a/platform-api/internal/model/llm.go b/platform-api/internal/model/llm.go index 1dcd57b19..ced7be1aa 100644 --- a/platform-api/internal/model/llm.go +++ b/platform-api/internal/model/llm.go @@ -169,25 +169,25 @@ type AssociatedGatewayMapping struct { } type LLMProviderTemplate struct { - UUID string `json:"uuid" db:"uuid"` - OrganizationUUID string `json:"organizationId" db:"organization_uuid"` - ID string `json:"id" db:"handle"` - GroupID string `json:"groupId,omitempty" db:"group_id"` - Name string `json:"displayName" db:"display_name"` - Description string `json:"description,omitempty" db:"description"` - ManagedBy string `json:"managedBy,omitempty" db:"managed_by"` - CreatedBy string `json:"createdBy,omitempty" db:"created_by"` - UpdatedBy string `json:"updatedBy,omitempty" db:"updated_by"` - Version string `json:"version" db:"version"` - IsLatest bool `json:"isLatest" db:"is_latest"` - Enabled bool `json:"enabled" db:"enabled"` - Metadata *LLMProviderTemplateMetadata `json:"metadata,omitempty" db:"-"` - PromptTokens *ExtractionIdentifier `json:"promptTokens,omitempty" db:"-"` - CompletionTokens *ExtractionIdentifier `json:"completionTokens,omitempty" db:"-"` - TotalTokens *ExtractionIdentifier `json:"totalTokens,omitempty" db:"-"` - RemainingTokens *ExtractionIdentifier `json:"remainingTokens,omitempty" db:"-"` - RequestModel *ExtractionIdentifier `json:"requestModel,omitempty" db:"-"` - ResponseModel *ExtractionIdentifier `json:"responseModel,omitempty" db:"-"` + UUID string `json:"uuid" db:"uuid"` + OrganizationUUID string `json:"organizationId" db:"organization_uuid"` + ID string `json:"id" db:"handle"` + GroupID string `json:"groupId,omitempty" db:"group_id"` + Name string `json:"displayName" db:"display_name"` + Description string `json:"description,omitempty" db:"description"` + ManagedBy string `json:"managedBy,omitempty" db:"managed_by"` + CreatedBy string `json:"createdBy,omitempty" db:"created_by"` + UpdatedBy string `json:"updatedBy,omitempty" db:"updated_by"` + Version string `json:"version" db:"version"` + IsLatest bool `json:"isLatest" db:"is_latest"` + Enabled bool `json:"enabled" db:"enabled"` + Metadata *LLMProviderTemplateMetadata `json:"metadata,omitempty" db:"-"` + PromptTokens *ExtractionIdentifier `json:"promptTokens,omitempty" db:"-"` + CompletionTokens *ExtractionIdentifier `json:"completionTokens,omitempty" db:"-"` + TotalTokens *ExtractionIdentifier `json:"totalTokens,omitempty" db:"-"` + RemainingTokens *ExtractionIdentifier `json:"remainingTokens,omitempty" db:"-"` + RequestModel *ExtractionIdentifier `json:"requestModel,omitempty" db:"-"` + ResponseModel *ExtractionIdentifier `json:"responseModel,omitempty" db:"-"` ResourceMappings *LLMProviderTemplateResourceMappings `json:"resourceMappings,omitempty" db:"-"` OpenAPISpec string `json:"openapi,omitempty" db:"openapi_spec"` Origin string `json:"origin,omitempty" db:"origin"` @@ -197,23 +197,23 @@ type LLMProviderTemplate struct { // LLMProvider represents an LLM provider entity type LLMProvider struct { - UUID string `json:"uuid" db:"uuid"` - OrganizationUUID string `json:"organizationId" db:"organization_uuid"` - ID string `json:"id" db:"handle"` - Name string `json:"displayName" db:"display_name"` - Description string `json:"description,omitempty" db:"description"` - CreatedBy string `json:"createdBy,omitempty" db:"created_by"` - UpdatedBy string `json:"updatedBy,omitempty" db:"updated_by"` - Version string `json:"version" db:"version"` - TemplateUUID string `json:"templateUuid" db:"template_uuid"` - OpenAPISpec string `json:"openapi,omitempty" db:"openapi_spec"` - ModelProviders []LLMModelProvider `json:"modelProviders,omitempty" db:"-"` - CreatedAt time.Time `json:"createdAt" db:"created_at"` - UpdatedAt time.Time `json:"updatedAt" db:"updated_at"` - Configuration LLMProviderConfig `json:"configuration" db:"configuration"` - Origin string `json:"origin,omitempty" db:"origin"` - AssociatedGateways []AssociatedGatewayMapping `json:"-" db:"-"` - ReplaceAssociatedGateways bool `json:"-" db:"-"` + UUID string `json:"uuid" db:"uuid"` + OrganizationUUID string `json:"organizationId" db:"organization_uuid"` + ID string `json:"id" db:"handle"` + Name string `json:"displayName" db:"display_name"` + Description string `json:"description,omitempty" db:"description"` + CreatedBy string `json:"createdBy,omitempty" db:"created_by"` + UpdatedBy string `json:"updatedBy,omitempty" db:"updated_by"` + Version string `json:"version" db:"version"` + TemplateUUID string `json:"templateUuid" db:"template_uuid"` + OpenAPISpec string `json:"openapi,omitempty" db:"openapi_spec"` + ModelProviders []LLMModelProvider `json:"modelProviders,omitempty" db:"-"` + CreatedAt time.Time `json:"createdAt" db:"created_at"` + UpdatedAt time.Time `json:"updatedAt" db:"updated_at"` + Configuration LLMProviderConfig `json:"configuration" db:"configuration"` + Origin string `json:"origin,omitempty" db:"origin"` + AssociatedGateways []AssociatedGatewayMapping `json:"-" db:"-"` + ReplaceAssociatedGateways bool `json:"-" db:"-"` } type LLMProviderConfig struct { @@ -233,36 +233,56 @@ type LLMProviderConfig struct { // LLMProxy represents an LLM proxy entity type LLMProxy struct { - UUID string `json:"uuid" db:"uuid"` - OrganizationUUID string `json:"organizationId" db:"organization_uuid"` - ID string `json:"id" db:"handle"` - Name string `json:"displayName" db:"display_name"` - ProjectUUID string `json:"projectId" db:"project_uuid"` - Description string `json:"description,omitempty" db:"description"` - CreatedBy string `json:"createdBy,omitempty" db:"created_by"` - UpdatedBy string `json:"updatedBy,omitempty" db:"updated_by"` - Version string `json:"version" db:"version"` - ProviderUUID string `json:"providerUuid" db:"provider_uuid"` - OpenAPISpec string `json:"openapi,omitempty" db:"openapi_spec"` - CreatedAt time.Time `json:"createdAt" db:"created_at"` - UpdatedAt time.Time `json:"updatedAt" db:"updated_at"` - Configuration LLMProxyConfig `json:"configuration" db:"configuration"` - Origin string `json:"origin,omitempty" db:"origin"` - AssociatedGateways []AssociatedGatewayMapping `json:"-" db:"-"` - ReplaceAssociatedGateways bool `json:"-" db:"-"` + UUID string `json:"uuid" db:"uuid"` + OrganizationUUID string `json:"organizationId" db:"organization_uuid"` + ID string `json:"id" db:"handle"` + Name string `json:"displayName" db:"display_name"` + ProjectUUID string `json:"projectId" db:"project_uuid"` + Description string `json:"description,omitempty" db:"description"` + CreatedBy string `json:"createdBy,omitempty" db:"created_by"` + UpdatedBy string `json:"updatedBy,omitempty" db:"updated_by"` + Version string `json:"version" db:"version"` + ProviderUUID string `json:"providerUuid" db:"provider_uuid"` + OpenAPISpec string `json:"openapi,omitempty" db:"openapi_spec"` + CreatedAt time.Time `json:"createdAt" db:"created_at"` + UpdatedAt time.Time `json:"updatedAt" db:"updated_at"` + Configuration LLMProxyConfig `json:"configuration" db:"configuration"` + Origin string `json:"origin,omitempty" db:"origin"` + AssociatedGateways []AssociatedGatewayMapping `json:"-" db:"-"` + ReplaceAssociatedGateways bool `json:"-" db:"-"` } type LLMProxyConfig struct { - Name string `json:"name,omitempty" db:"-"` - Version string `json:"version,omitempty" db:"-"` - Context *string `json:"context,omitempty" db:"-"` - Vhost *string `json:"vhost,omitempty" db:"-"` - Provider string `json:"provider,omitempty" db:"-"` - UpstreamAuth *UpstreamAuth `json:"upstreamAuth,omitempty" db:"-"` - GlobalPolicies []GlobalPolicy `json:"globalPolicies,omitempty" db:"-"` - OperationPolicies []OperationPolicy `json:"operationPolicies,omitempty" db:"-"` - Policies []LLMPolicy `json:"policies,omitempty" db:"-"` - Security *SecurityConfig `json:"security,omitempty" db:"-"` + Name string `json:"name,omitempty" db:"-"` + Version string `json:"version,omitempty" db:"-"` + Context *string `json:"context,omitempty" db:"-"` + Vhost *string `json:"vhost,omitempty" db:"-"` + Provider string `json:"provider,omitempty" db:"-"` + UpstreamAuth *UpstreamAuth `json:"upstreamAuth,omitempty" db:"-"` + AdditionalProviders []LLMProxyAdditionalProvider `json:"additionalProviders,omitempty" db:"-"` + GlobalPolicies []GlobalPolicy `json:"globalPolicies,omitempty" db:"-"` + OperationPolicies []OperationPolicy `json:"operationPolicies,omitempty" db:"-"` + Policies []LLMPolicy `json:"policies,omitempty" db:"-"` + Security *SecurityConfig `json:"security,omitempty" db:"-"` +} + +// LLMProxyAdditionalProvider is an additional LLM provider attached to a proxy +// as a selectable upstream. Policies route to it by the `As` name (which +// defaults to `ID` when empty). Provider upstream auth is taken from the +// referenced LlmProvider's own configuration. +type LLMProxyAdditionalProvider struct { + ID string `json:"id" db:"-"` + As string `json:"as,omitempty" db:"-"` + Transformer *LLMProxyTransformer `json:"transformer,omitempty" db:"-"` +} + +// LLMProxyTransformer is a request/response translator applied when its +// provider is the selected upstream. The gateway-controller injects it as a +// conditional policy gated on that selection. +type LLMProxyTransformer struct { + Type string `json:"type" db:"-"` + Version string `json:"version" db:"-"` + Params map[string]interface{} `json:"params,omitempty" db:"-"` } type SecurityConfig struct { diff --git a/platform-api/internal/service/llm.go b/platform-api/internal/service/llm.go index 151fb72a6..80119a8d3 100644 --- a/platform-api/internal/service/llm.go +++ b/platform-api/internal/service/llm.go @@ -1248,6 +1248,38 @@ func (s *LLMProviderService) Delete(orgUUID, handle, deletedBy string) error { return nil } +// validateAdditionalProviders eagerly validates a proxy's additional providers +// so Create/Update surface an immediate, actionable API error instead of a +// confusing deployment-time failure. It mirrors the checks the gateway performs +// at transform time (see llm_transformer.go): every referenced provider must +// exist, and each upstream name (the `as` alias, or the provider id when no +// alias is set) must be unique within the proxy and must not collide with the +// primary provider id. +func (s *LLMProxyService) validateAdditionalProviders(orgUUID, primaryProviderID string, additionalProviders *[]api.LLMProxyAdditionalProvider) error { + if additionalProviders == nil { + return nil + } + seen := map[string]bool{primaryProviderID: true} + for _, ap := range *additionalProviders { + prov, err := s.providerRepo.GetByID(ap.Id, orgUUID) + if err != nil { + return fmt.Errorf("failed to validate additional provider %q: %w", ap.Id, err) + } + if prov == nil { + return constants.ErrLLMProviderNotFound + } + name := ap.Id + if ap.As != nil && *ap.As != "" { + name = *ap.As + } + if seen[name] { + return constants.ErrInvalidInput + } + seen[name] = true + } + return nil +} + func (s *LLMProxyService) Create(orgUUID, createdBy string, req *api.LLMProxy) (*api.LLMProxy, error) { if req == nil { return nil, constants.ErrInvalidInput @@ -1287,6 +1319,11 @@ func (s *LLMProxyService) Create(orgUUID, createdBy string, req *api.LLMProxy) ( return nil, constants.ErrLLMProviderNotFound } + // Validate additional providers exist and have unique upstream names + if err := s.validateAdditionalProviders(orgUUID, req.Provider.Id, req.AdditionalProviders); err != nil { + return nil, err + } + // Determine handle: use provided id or auto-generate from displayName var handle string if req.Id != nil && *req.Id != "" { @@ -1351,14 +1388,15 @@ func (s *LLMProxyService) Create(orgUUID, createdBy string, req *api.LLMProxy) ( ProviderUUID: prov.UUID, OpenAPISpec: utils.ValueOrEmpty(req.Openapi), Configuration: model.LLMProxyConfig{ - Context: &contextValue, - Vhost: req.Vhost, - Provider: req.Provider.Id, - UpstreamAuth: mapUpstreamAuthAPIToModel(req.Provider.Auth), - GlobalPolicies: mapGlobalPoliciesAPIToModel(req.GlobalPolicies), - OperationPolicies: mapOperationPoliciesAPIToModel(req.OperationPolicies), - Policies: mapPoliciesAPIToModel(req.Policies), - Security: mapSecurityAPIToModel(req.Security), + Context: &contextValue, + Vhost: req.Vhost, + Provider: req.Provider.Id, + UpstreamAuth: mapUpstreamAuthAPIToModel(req.Provider.Auth), + AdditionalProviders: mapAdditionalProvidersAPIToModel(req.AdditionalProviders), + GlobalPolicies: mapGlobalPoliciesAPIToModel(req.GlobalPolicies), + OperationPolicies: mapOperationPoliciesAPIToModel(req.OperationPolicies), + Policies: mapPoliciesAPIToModel(req.Policies), + Security: mapSecurityAPIToModel(req.Security), }, Origin: constants.OriginCP, AssociatedGateways: associatedGateways, @@ -1589,6 +1627,11 @@ func (s *LLMProxyService) Update(orgUUID, handle, updatedBy string, req *api.LLM } } + // Validate additional providers exist and have unique upstream names + if err := s.validateAdditionalProviders(orgUUID, req.Provider.Id, req.AdditionalProviders); err != nil { + return nil, err + } + contextValue := utils.DefaultStringPtr(req.Context, "/") m := &model.LLMProxy{ OrganizationUUID: orgUUID, @@ -1600,14 +1643,15 @@ func (s *LLMProxyService) Update(orgUUID, handle, updatedBy string, req *api.LLM ProviderUUID: prov.UUID, OpenAPISpec: utils.ValueOrEmpty(req.Openapi), Configuration: model.LLMProxyConfig{ - Context: &contextValue, - Vhost: req.Vhost, - Provider: req.Provider.Id, - UpstreamAuth: mapUpstreamAuthAPIToModel(req.Provider.Auth), - GlobalPolicies: mapGlobalPoliciesAPIToModel(req.GlobalPolicies), - OperationPolicies: mapOperationPoliciesAPIToModel(req.OperationPolicies), - Policies: mapPoliciesAPIToModel(req.Policies), - Security: mapSecurityAPIToModel(req.Security), + Context: &contextValue, + Vhost: req.Vhost, + Provider: req.Provider.Id, + UpstreamAuth: mapUpstreamAuthAPIToModel(req.Provider.Auth), + AdditionalProviders: mapAdditionalProvidersAPIToModel(req.AdditionalProviders), + GlobalPolicies: mapGlobalPoliciesAPIToModel(req.GlobalPolicies), + OperationPolicies: mapOperationPoliciesAPIToModel(req.OperationPolicies), + Policies: mapPoliciesAPIToModel(req.Policies), + Security: mapSecurityAPIToModel(req.Security), }, } migrateLegacyProxyPoliciesInPlace(&m.Configuration) @@ -2092,6 +2136,56 @@ func mapUpstreamAuthAPIToModel(in *api.UpstreamAuth) *model.UpstreamAuth { } } +func mapAdditionalProvidersAPIToModel(in *[]api.LLMProxyAdditionalProvider) []model.LLMProxyAdditionalProvider { + if in == nil || len(*in) == 0 { + return nil + } + out := make([]model.LLMProxyAdditionalProvider, 0, len(*in)) + for _, p := range *in { + entry := model.LLMProxyAdditionalProvider{ + ID: p.Id, + As: utils.ValueOrEmpty(p.As), + } + if p.Transformer != nil { + entry.Transformer = &model.LLMProxyTransformer{ + Type: p.Transformer.Type, + Version: p.Transformer.Version, + } + if p.Transformer.Params != nil { + entry.Transformer.Params = *p.Transformer.Params + } + } + out = append(out, entry) + } + return out +} + +func mapAdditionalProvidersModelToAPI(in []model.LLMProxyAdditionalProvider) *[]api.LLMProxyAdditionalProvider { + if len(in) == 0 { + return nil + } + out := make([]api.LLMProxyAdditionalProvider, 0, len(in)) + for _, p := range in { + entry := api.LLMProxyAdditionalProvider{Id: p.ID} + if p.As != "" { + as := p.As + entry.As = &as + } + if p.Transformer != nil { + entry.Transformer = &api.LLMProxyTransformer{ + Type: p.Transformer.Type, + Version: p.Transformer.Version, + } + if len(p.Transformer.Params) > 0 { + params := p.Transformer.Params + entry.Transformer.Params = ¶ms + } + } + out = append(out, entry) + } + return &out +} + func normalizeUpstreamAuthType(authType string) string { normalized := strings.TrimSpace(authType) if normalized == "" { @@ -3003,6 +3097,9 @@ func mapProxyModelToAPI(m *model.LLMProxy) *api.LLMProxy { Value: nil, // Redact auth credential value } } + if extra := mapAdditionalProvidersModelToAPI(m.Configuration.AdditionalProviders); extra != nil { + out.AdditionalProviders = extra + } out.GlobalPolicies = globalPoliciesProxy out.OperationPolicies = operationPoliciesProxy out.Policies = nil diff --git a/platform-api/internal/service/llm_deployment.go b/platform-api/internal/service/llm_deployment.go index b64f2cc96..dcff53ac9 100644 --- a/platform-api/internal/service/llm_deployment.go +++ b/platform-api/internal/service/llm_deployment.go @@ -1885,6 +1885,30 @@ func generateLLMProxyDeploymentYAML(proxy *model.LLMProxy) (dto.LLMProxyDeployme proxyDeployment.Spec.Provider.Auth = mapModelUpstreamAuthToAPI(proxy.Configuration.UpstreamAuth) } + // Carry additional providers (multi-provider proxies) into the deployment + // artifact so the gateway-controller can expose each as a selectable upstream. + if len(proxy.Configuration.AdditionalProviders) > 0 { + additional := make([]dto.LLMProxyDeploymentAdditionalProvider, 0, len(proxy.Configuration.AdditionalProviders)) + for _, ap := range proxy.Configuration.AdditionalProviders { + entry := dto.LLMProxyDeploymentAdditionalProvider{ + ID: ap.ID, + As: ap.As, + } + if ap.Transformer != nil { + entry.Transformer = &api.LLMProxyTransformer{ + Type: ap.Transformer.Type, + Version: ap.Transformer.Version, + } + if len(ap.Transformer.Params) > 0 { + params := ap.Transformer.Params + entry.Transformer.Params = ¶ms + } + } + additional = append(additional, entry) + } + proxyDeployment.Spec.AdditionalProviders = additional + } + // Promote any legacy policies assembled by the generator into operationPolicies. for _, p := range proxyDeployment.Spec.Policies { paths := make([]api.OperationPolicyPath, 0, len(p.Paths)) diff --git a/platform-api/internal/service/llm_test.go b/platform-api/internal/service/llm_test.go index c16de920e..0d2349af9 100644 --- a/platform-api/internal/service/llm_test.go +++ b/platform-api/internal/service/llm_test.go @@ -1746,10 +1746,10 @@ func TestLLMProxyServiceUpdate_CleansUpRotatedSecret(t *testing.T) { func validProviderRequest(template string) *api.LLMProvider { return &api.LLMProvider{ - Id: strPointer("provider-1"), - DisplayName: "Test Provider", - Version: "v1.0", - Template: template, + Id: strPointer("provider-1"), + DisplayName: "Test Provider", + Version: "v1.0", + Template: template, Upstream: api.Upstream{ Main: api.UpstreamDefinition{Url: stringPtr("https://example.com/openai/v1")}, }, @@ -1759,10 +1759,10 @@ func validProviderRequest(template string) *api.LLMProvider { func validProxyRequest(providerID, projectID string) *api.LLMProxy { return &api.LLMProxy{ - Id: strPointer("proxy-1"), - DisplayName: "Test Proxy", - Version: "v1.0", - ProjectId: projectID, + Id: strPointer("proxy-1"), + DisplayName: "Test Proxy", + Version: "v1.0", + ProjectId: projectID, Provider: api.LLMProxyProvider{ Id: providerID, }, @@ -1777,3 +1777,119 @@ func upstreamAuthTypePtr(v string) *api.UpstreamAuthType { t := api.UpstreamAuthType(v) return &t } + +func TestLLMProxyServiceCreateFailsWhenAdditionalProviderNotFound(t *testing.T) { + proxyRepo := &mockLLMProxyRepo{} + providerRepo := &mockLLMProviderRepo{ + getByIDFunc: func(providerID, orgUUID string) (*model.LLMProvider, error) { + if providerID == "provider-1" { + return &model.LLMProvider{UUID: "provider-uuid", ID: providerID}, nil + } + return nil, nil + }, + } + service := NewLLMProxyService(proxyRepo, providerRepo, nil, nil, nil, nil, slog.Default(), &noopAuditRepo{}, &config.Server{}, newTestIdentityService()) + + req := validProxyRequest("provider-1", "project-1") + req.AdditionalProviders = &[]api.LLMProxyAdditionalProvider{{Id: "missing-provider"}} + + if _, err := service.Create("org-1", "alice", req); err != constants.ErrLLMProviderNotFound { + t.Fatalf("expected ErrLLMProviderNotFound, got: %v", err) + } +} + +func TestLLMProxyServiceCreateFailsWhenAdditionalProviderNameCollides(t *testing.T) { + proxyRepo := &mockLLMProxyRepo{} + providerRepo := &mockLLMProviderRepo{ + getByIDFunc: func(providerID, orgUUID string) (*model.LLMProvider, error) { + return &model.LLMProvider{UUID: "provider-uuid", ID: providerID}, nil + }, + } + service := NewLLMProxyService(proxyRepo, providerRepo, nil, nil, nil, nil, slog.Default(), &noopAuditRepo{}, &config.Server{}, newTestIdentityService()) + + req := validProxyRequest("provider-1", "project-1") + // The additional provider exists, but its upstream `as` name collides with + // the primary provider id, which the gateway rejects at transform time. + req.AdditionalProviders = &[]api.LLMProxyAdditionalProvider{ + {Id: "provider-2", As: stringPtr("provider-1")}, + } + + if _, err := service.Create("org-1", "alice", req); err != constants.ErrInvalidInput { + t.Fatalf("expected ErrInvalidInput, got: %v", err) + } +} + +func TestLLMProxyServiceUpdateFailsWhenAdditionalProviderNotFound(t *testing.T) { + now := time.Now() + proxyRepo := &mockLLMProxyRepo{} + proxyRepo.getByIDFunc = func(proxyID, orgUUID string) (*model.LLMProxy, error) { + return &model.LLMProxy{ + UUID: "proxy-uuid", + ID: proxyID, + Name: "Old Proxy", + Version: "v1.0", + ProjectUUID: "project-1", + ProviderUUID: "provider-uuid", + CreatedAt: now, + UpdatedAt: now, + Configuration: model.LLMProxyConfig{Provider: "provider-1"}, + }, nil + } + providerRepo := &mockLLMProviderRepo{ + getByIDFunc: func(providerID, orgUUID string) (*model.LLMProvider, error) { + if providerID == "provider-1" { + return &model.LLMProvider{UUID: "provider-uuid", ID: providerID}, nil + } + return nil, nil + }, + } + service := NewLLMProxyService(proxyRepo, providerRepo, nil, nil, nil, nil, slog.Default(), &noopAuditRepo{}, &config.Server{}, newTestIdentityService()) + + req := validProxyRequest("provider-1", "project-1") + req.AdditionalProviders = &[]api.LLMProxyAdditionalProvider{{Id: "missing-provider"}} + + if _, err := service.Update("org-1", "proxy-1", "test-user", req); err != constants.ErrLLMProviderNotFound { + t.Fatalf("expected ErrLLMProviderNotFound, got: %v", err) + } + if proxyRepo.updated != nil { + t.Fatalf("expected update to be rejected before persisting, but proxy was updated") + } +} + +func TestLLMProxyServiceUpdateFailsWhenAdditionalProviderNameCollides(t *testing.T) { + now := time.Now() + proxyRepo := &mockLLMProxyRepo{} + proxyRepo.getByIDFunc = func(proxyID, orgUUID string) (*model.LLMProxy, error) { + return &model.LLMProxy{ + UUID: "proxy-uuid", + ID: proxyID, + Name: "Old Proxy", + Version: "v1.0", + ProjectUUID: "project-1", + ProviderUUID: "provider-uuid", + CreatedAt: now, + UpdatedAt: now, + Configuration: model.LLMProxyConfig{Provider: "provider-1"}, + }, nil + } + providerRepo := &mockLLMProviderRepo{ + getByIDFunc: func(providerID, orgUUID string) (*model.LLMProvider, error) { + return &model.LLMProvider{UUID: "provider-uuid", ID: providerID}, nil + }, + } + service := NewLLMProxyService(proxyRepo, providerRepo, nil, nil, nil, nil, slog.Default(), &noopAuditRepo{}, &config.Server{}, newTestIdentityService()) + + req := validProxyRequest("provider-1", "project-1") + // Two additional providers resolve to the same upstream `as` name. + req.AdditionalProviders = &[]api.LLMProxyAdditionalProvider{ + {Id: "provider-2", As: stringPtr("shared")}, + {Id: "provider-3", As: stringPtr("shared")}, + } + + if _, err := service.Update("org-1", "proxy-1", "test-user", req); err != constants.ErrInvalidInput { + t.Fatalf("expected ErrInvalidInput, got: %v", err) + } + if proxyRepo.updated != nil { + t.Fatalf("expected update to be rejected before persisting, but proxy was updated") + } +} diff --git a/platform-api/resources/openapi.yaml b/platform-api/resources/openapi.yaml index 6b702d19e..a46bf0ee2 100644 --- a/platform-api/resources/openapi.yaml +++ b/platform-api/resources/openapi.yaml @@ -7713,6 +7713,15 @@ components: example: "api.openai" provider: $ref: '#/components/schemas/LLMProxyProvider' + additionalProviders: + type: array + description: > + Optional list of additional LLM providers attached to this proxy as + selectable upstreams. Policies route requests to any of these by + setting the upstream name. The primary `provider` field above + remains the default upstream and the FK target. + items: + $ref: '#/components/schemas/LLMProxyAdditionalProvider' openapi: type: string description: OpenAPI specification (JSON or YAML) for the proxy endpoint @@ -7854,6 +7863,59 @@ components: auth: $ref: '#/components/schemas/UpstreamAuth' + LLMProxyAdditionalProvider: + type: object + required: + - id + description: > + Additional LLM provider attached to this proxy as a selectable upstream. + Policies route to it by referring to the `as` name (defaults to `id`). + properties: + id: + type: string + description: Unique id of a deployed llm provider + example: anthropic-provider + as: + type: string + description: > + Logical LLM Provider name used by policies to select this provider. Must + be unique within the proxy. Defaults to `id` when omitted. + pattern: '^[a-zA-Z0-9\-_]+$' + minLength: 1 + maxLength: 100 + example: anthropic-upstream + transformer: + $ref: '#/components/schemas/LLMProxyTransformer' + + LLMProxyTransformer: + type: object + required: + - type + - version + description: > + Request/response translator applied when this provider is the selected + upstream. The proxy injects the translator as a conditional policy whose + execution condition matches this provider, so it runs only when the + provider is selected. The provider's `as` name (defaults to `id`) is + passed to the translator as its target upstream. + properties: + type: + type: string + description: Translator policy name (for example openai-to-anthropic). + minLength: 1 + example: openai-to-anthropic + version: + type: string + description: > + Major-only translator policy version (for example v1). The Gateway + Controller resolves it to the installed full version. + minLength: 1 + example: v1 + params: + type: object + description: Translator-specific parameters (for example model, apiVersion). + additionalProperties: true + CreateLLMProviderAPIKeyRequest: type: object required: