Skip to content

Commit ced79c5

Browse files
sihyeonnsihyeonn
committed
feat(ai-proxy): add defaults field for fallback model options
Separate options and defaults behavior: - options: always override user request values - defaults: apply only when not set in user request This allows more flexible configuration where administrators can enforce certain values (via options) while providing sensible defaults for optional parameters. Priority order: options > client request > defaults Closes #13149 Signed-off-by: Sihyeon Jang <sihyeon.jang@navercorp.com>
1 parent da45cbe commit ced79c5

8 files changed

Lines changed: 204 additions & 1 deletion

File tree

‎apisix/plugins/ai-proxy/base.lua‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,15 @@ function _M.before_proxy(conf, ctx, on_error)
127127
auth = ai_instance.auth,
128128
}
129129

130+
-- defaults: apply fallback values before routing
131+
if ai_instance.defaults then
132+
for opt, val in pairs(ai_instance.defaults) do
133+
if request_body[opt] == nil then
134+
request_body[opt] = val
135+
end
136+
end
137+
end
138+
130139
-- Step 1: Route client protocol to driver capability
131140
local client_protocol = ctx.ai_client_protocol
132141
local client_proto = protocols.get(client_protocol)
@@ -164,7 +173,9 @@ function _M.before_proxy(conf, ctx, on_error)
164173
if request_model then
165174
ctx.var.request_llm_model = request_model
166175
end
167-
local model = ai_instance.options and ai_instance.options.model or request_model
176+
local model = ai_instance.options and ai_instance.options.model
177+
or request_model
178+
or ai_instance.defaults and ai_instance.defaults.model
168179
if model then
169180
ctx.var.llm_model = model
170181
end

‎apisix/plugins/ai-proxy/schema.lua‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,12 @@ local provider_vertex_ai_schema = {
8787
required = { "project_id", "region" },
8888
}
8989

90+
local model_defaults_schema = {
91+
description = "Default values applied only when not set in request",
92+
type = "object",
93+
additionalProperties = true,
94+
}
95+
9096
local ai_instance_schema = {
9197
type = "array",
9298
minItems = 1,
@@ -115,6 +121,7 @@ local ai_instance_schema = {
115121
},
116122
auth = auth_schema,
117123
options = model_options_schema,
124+
defaults = model_defaults_schema,
118125
override = {
119126
type = "object",
120127
properties = {
@@ -176,6 +183,7 @@ _M.ai_proxy_schema = {
176183
logging = logging_schema,
177184
auth = auth_schema,
178185
options = model_options_schema,
186+
defaults = model_defaults_schema,
179187
timeout = {
180188
type = "integer",
181189
minimum = 1,

‎docs/en/latest/plugins/ai-proxy-multi.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ In addition, the Plugin also supports logging LLM request information in the acc
7575
| instances.options.model | string | False | | | Name of the LLM model, such as `gpt-4` or `gpt-3.5`. See your LLM provider's API documentation for more available models. |
7676
| instances.override | object | False | | | Override setting. |
7777
| instances.override.endpoint | string | False | | | LLM provider endpoint to replace the default endpoint with. If not configured, the Plugin uses the default OpenAI endpoint `https://api.openai.com/v1/chat/completions`. |
78+
| instances.defaults | object | False | | | Default model configurations applied only when not set in the user request. Unlike `options` which always overrides user values, `defaults` provides fallback values. For example, you can set default `max_tokens` or `temperature` that users can override in their requests. |
7879
| logging | object | False | | | Logging configurations. Does not affect `error.log`. |
7980
| logging.summaries | boolean | False | false | | If true, logs request LLM model, duration, request, and response tokens. |
8081
| logging.payloads | boolean | False | false | | If true, logs request and response payload. |

‎docs/en/latest/plugins/ai-proxy.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ In addition, the Plugin also supports logging LLM request information in the acc
6464
| auth.gcp.expire_early_secs | integer | False | 60 | minimum = 0 | Seconds to expire the access token before its actual expiration time to avoid edge cases. |
6565
| options | object | False | | | Model configurations. In addition to `model`, you can configure additional parameters and they will be forwarded to the upstream LLM service in the request body. For instance, if you are working with OpenAI, you can configure additional parameters such as `temperature`, `top_p`, and `stream`. See your LLM provider's API documentation for more available options. |
6666
| options.model | string | False | | | Name of the LLM model, such as `gpt-4` or `gpt-3.5`. Refer to the LLM provider's API documentation for available models. |
67+
| defaults | object | False | | | Default model configurations applied only when not set in the user request. Unlike `options` which always overrides user values, `defaults` provides fallback values. For example, you can set default `max_tokens` or `temperature` that users can override in their requests. |
6768
| override | object | False | | | Override setting. |
6869
| override.endpoint | string | False | | | Custom LLM provider endpoint, required when `provider` is `openai-compatible`. |
6970
| logging | object | False | | | Logging configurations. Does not affect `error.log`. |

‎docs/zh/latest/plugins/ai-proxy-multi.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ description: ai-proxy-multi 插件通过负载均衡、重试、故障转移和
7575
| instances.options.model | string | 否 | | | LLM 模型的名称，如 `gpt-4` 或 `gpt-3.5`。有关更多可用模型，请参阅您的 LLM 提供商的 API 文档。 |
7676
| instances.override | object | 否 | | | 覆盖设置。 |
7777
| instances.override.endpoint | string | 否 | | | 用于替换默认端点的 LLM 提供商端点。如果未配置，插件使用默认的 OpenAI 端点 `https://api.openai.com/v1/chat/completions`。 |
78+
| instances.defaults | object | 否 | | | 仅在用户请求中未设置时应用的默认模型配置。与始终覆盖用户值的 `options` 不同，`defaults` 提供回退值。例如，您可以设置默认的 `max_tokens` 或 `temperature`，用户可以在其请求中覆盖这些值。 |
7879
| logging | object | 否 | | | 日志配置。不影响 `error.log`。 |
7980
| logging.summaries | boolean | 否 | false | | 如果为 true，记录请求 LLM 模型、持续时间、请求和响应令牌。 |
8081
| logging.payloads | boolean | 否 | false | | 如果为 true，记录请求和响应负载。 |

‎docs/zh/latest/plugins/ai-proxy.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ description: ai-proxy 插件通过将插件配置转换为所需的请求格式
6464
| auth.gcp.expire_early_secs | integer | 否 | 60 | minimum = 0 | 在访问令牌实际过期时间之前使其过期的秒数，以避免边缘情况。 |
6565
| options | object | 否 | | | 模型配置。除了 `model` 之外，您还可以配置其他参数，它们将在请求体中转发到上游 LLM 服务。例如，如果您使用 OpenAI，可以配置其他参数，如 `temperature`、`top_p` 和 `stream`。有关更多可用选项，请参阅您的 LLM 提供商的 API 文档。 |
6666
| options.model | string | 否 | | | LLM 模型的名称，如 `gpt-4` 或 `gpt-3.5`。请参阅 LLM 提供商的 API 文档以了解可用模型。 |
67+
| defaults | object | 否 | | | 仅在用户请求中未设置时应用的默认模型配置。与始终覆盖用户值的 `options` 不同，`defaults` 提供回退值。例如，您可以设置默认的 `max_tokens` 或 `temperature`，用户可以在其请求中覆盖这些值。 |
6768
| override | object | 否 | | | 覆盖设置。 |
6869
| override.endpoint | string | 否 | | | 自定义 LLM 提供商端点，当 `provider` 为 `openai-compatible` 时必需。 |
6970
| logging | object | 否 | | | 日志配置。不影响 `error.log`。 |

‎t/plugin/ai-proxy-multi.openai-compatible.t‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,16 @@ _EOC_
7777
return
7878
end
7979
80+
if test_type == "defaults" then
81+
ngx.status = 200
82+
ngx.say(json.encode({
83+
model = body.model,
84+
max_tokens = body.max_tokens,
85+
temperature = body.temperature
86+
}))
87+
return
88+
end
89+
8090
local header_auth = ngx.req.get_headers()["authorization"]
8191
local query_auth = ngx.req.get_uri_args()["apikey"]
8292
@@ -296,3 +306,86 @@ passed
296306
}
297307
--- response_body_like eval
298308
qr/6data: \[DONE\]\n\n/
309+
310+
311+
312+
=== TEST 5: set route with defaults field
313+
--- config
314+
location /t {
315+
content_by_lua_block {
316+
local t = require("lib.test_admin").test
317+
local code, body = t('/apisix/admin/routes/1',
318+
ngx.HTTP_PUT,
319+
[[{
320+
"uri": "/anything",
321+
"plugins": {
322+
"ai-proxy-multi": {
323+
"instances": [
324+
{
325+
"name": "self-hosted",
326+
"provider": "openai-compatible",
327+
"weight": 1,
328+
"auth": {
329+
"header": {
330+
"Authorization": "Bearer token"
331+
}
332+
},
333+
"options": {
334+
"model": "server-model"
335+
},
336+
"defaults": {
337+
"max_tokens": 512,
338+
"temperature": 0.7
339+
},
340+
"override": {
341+
"endpoint": "http://localhost:6724/v1/chat/completions"
342+
}
343+
}
344+
],
345+
"ssl_verify": false
346+
}
347+
}
348+
}]]
349+
)
350+
351+
if code >= 300 then
352+
ngx.status = code
353+
end
354+
ngx.say(body)
355+
}
356+
}
357+
--- response_body
358+
passed
359+
360+
361+
362+
=== TEST 6: defaults applied when not set in request
363+
--- request
364+
POST /anything
365+
{ "messages": [ { "role": "user", "content": "hello" } ] }
366+
--- more_headers
367+
test-type: defaults
368+
--- response_body_like eval
369+
qr/(?=.*"max_tokens":512)(?=.*"model":"server-model")(?=.*"temperature":0\.7)/
370+
371+
372+
373+
=== TEST 7: user value takes precedence over defaults
374+
--- request
375+
POST /anything
376+
{ "messages": [ { "role": "user", "content": "hello" } ], "max_tokens": 100, "temperature": 0.5 }
377+
--- more_headers
378+
test-type: defaults
379+
--- response_body_like eval
380+
qr/(?=.*"max_tokens":100)(?=.*"model":"server-model")(?=.*"temperature":0\.5)/
381+
382+
383+
384+
=== TEST 8: options always override user value
385+
--- request
386+
POST /anything
387+
{ "messages": [ { "role": "user", "content": "hello" } ], "model": "user-model", "max_tokens": 100 }
388+
--- more_headers
389+
test-type: defaults
390+
--- response_body_like eval
391+
qr/(?=.*"max_tokens":100)(?=.*"model":"server-model")(?=.*"temperature":0\.7)/

‎t/plugin/ai-proxy.openai-compatible.t‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,16 @@ add_block_preprocessor(sub {
7070
return
7171
end
7272
73+
if test_type == "defaults" then
74+
ngx.status = 200
75+
ngx.say(json.encode({
76+
model = body.model,
77+
max_tokens = body.max_tokens,
78+
temperature = body.temperature
79+
}))
80+
return
81+
end
82+
7383
local header_auth = ngx.req.get_headers()["authorization"]
7484
local query_auth = ngx.req.get_uri_args()["apikey"]
7585
@@ -338,3 +348,80 @@ passed
338348
}
339349
--- response_body_eval
340350
qr/6data: \[DONE\]\n\n/
351+
352+
353+
354+
=== TEST 6: set route with defaults field
355+
--- config
356+
location /t {
357+
content_by_lua_block {
358+
local t = require("lib.test_admin").test
359+
local code, body = t('/apisix/admin/routes/1',
360+
ngx.HTTP_PUT,
361+
[[{
362+
"uri": "/anything",
363+
"plugins": {
364+
"ai-proxy": {
365+
"provider": "openai-compatible",
366+
"auth": {
367+
"header": {
368+
"Authorization": "Bearer token"
369+
}
370+
},
371+
"options": {
372+
"model": "server-model"
373+
},
374+
"defaults": {
375+
"max_tokens": 512,
376+
"temperature": 0.7
377+
},
378+
"override": {
379+
"endpoint": "http://localhost:6724/v1/chat/completions"
380+
},
381+
"ssl_verify": false
382+
}
383+
}
384+
}]]
385+
)
386+
387+
if code >= 300 then
388+
ngx.status = code
389+
end
390+
ngx.say(body)
391+
}
392+
}
393+
--- response_body
394+
passed
395+
396+
397+
398+
=== TEST 7: defaults applied when not set in request
399+
--- request
400+
POST /anything
401+
{ "messages": [ { "role": "user", "content": "hello" } ] }
402+
--- more_headers
403+
test-type: defaults
404+
--- response_body_like eval
405+
qr/(?=.*"max_tokens":512)(?=.*"model":"server-model")(?=.*"temperature":0\.7)/
406+
407+
408+
409+
=== TEST 8: user value takes precedence over defaults
410+
--- request
411+
POST /anything
412+
{ "messages": [ { "role": "user", "content": "hello" } ], "max_tokens": 100, "temperature": 0.5 }
413+
--- more_headers
414+
test-type: defaults
415+
--- response_body_like eval
416+
qr/(?=.*"max_tokens":100)(?=.*"model":"server-model")(?=.*"temperature":0\.5)/
417+
418+
419+
420+
=== TEST 9: options always override user value
421+
--- request
422+
POST /anything
423+
{ "messages": [ { "role": "user", "content": "hello" } ], "model": "user-model", "max_tokens": 100 }
424+
--- more_headers
425+
test-type: defaults
426+
--- response_body_like eval
427+
qr/(?=.*"max_tokens":100)(?=.*"model":"server-model")(?=.*"temperature":0\.7)/

0 commit comments

Comments
 (0)