Skip to content

Commit 7ea42d4

Browse files
feat(ai-proxy): send LLM requests through ngx_http_ffi_client (#13778)
1 parent a4bacbd commit 7ea42d4

9 files changed

Lines changed: 899 additions & 6 deletions

File tree

.requirements

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,5 +17,5 @@
1717

1818
APISIX_PACKAGE_NAME=apisix
1919

20-
APISIX_RUNTIME=1.3.14
20+
APISIX_RUNTIME=1.3.16
2121
APISIX_DASHBOARD_COMMIT=c8d3466d3c36386d3888efbc8250cd8183c77298

apisix/cli/config.lua

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -309,6 +309,9 @@ local _M = {
309309
},
310310
stream_plugins = { "ip-restriction", "limit-conn", "mqtt-proxy", "syslog", "traffic-split" },
311311
plugin_attr = {
312+
["ai-proxy"] = {
313+
http_client = "ngx_http_ffi_client"
314+
},
312315
["log-rotate"] = {
313316
timeout = 10000,
314317
interval = 3600,

apisix/plugins/ai-transport/http.lua

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
-- Provides HTTP client lifecycle management for AI provider requests.
2020

2121
local core = require("apisix.core")
22-
local http = require("resty.http")
22+
local http_client = require("apisix.utils.http")
2323
local ngx_now = ngx.now
2424
local pairs = pairs
2525
local ipairs = ipairs
@@ -28,8 +28,41 @@ local type = type
2828
local str_lower = string.lower
2929
local tostring = tostring
3030

31+
local attr_schema = {
32+
type = "object",
33+
properties = {
34+
http_client = http_client.client_schema,
35+
},
36+
}
37+
3138
local _M = {}
3239

40+
local client_name
41+
42+
43+
--- Which client this transport should use.
44+
-- `plugin_attr.ai-proxy.http_client` names it; the shared module owns the
45+
-- names, validation and loading. Read on first request, because local_conf is
46+
-- not readable while this module is still loading.
47+
local function resolve_client_name()
48+
if client_name then
49+
return client_name
50+
end
51+
52+
local local_conf = core.config.local_conf()
53+
local attr = core.table.try_read_attr(local_conf, "plugin_attr", "ai-proxy") or {}
54+
55+
local ok, err = core.schema.check(attr_schema, attr)
56+
if not ok then
57+
core.log.error("invalid plugin_attr.ai-proxy: ", err)
58+
return nil, "invalid plugin_attr.ai-proxy: " .. err
59+
end
60+
61+
client_name = attr.http_client or http_client.DEFAULT_CLIENT
62+
63+
return client_name
64+
end
65+
3366

3467
--- Map network errors to HTTP status codes.
3568
-- Cosocket timers report "timeout"; OS errno (ETIMEDOUT) and the resolver
@@ -100,7 +133,12 @@ end
100133
-- @return string|nil Error message
101134
-- @return table|nil Upstream metadata on failure (for recording failed attempts)
102135
function _M.request(params, timeout)
103-
local httpc, err = http.new()
136+
local name, name_err = resolve_client_name()
137+
if not name then
138+
return nil, "failed to create http client: " .. name_err
139+
end
140+
141+
local httpc, err = http_client.new(name)
104142
if not httpc then
105143
return nil, "failed to create http client: " .. (err or "unknown")
106144
end

apisix/utils/http.lua

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
--
2+
-- Licensed to the Apache Software Foundation (ASF) under one or more
3+
-- contributor license agreements. See the NOTICE file distributed with
4+
-- this work for additional information regarding copyright ownership.
5+
-- The ASF licenses this file to You under the Apache License, Version 2.0
6+
-- (the "License"); you may not use this file except in compliance with
7+
-- the License. You may obtain a copy of the License at
8+
--
9+
-- http://www.apache.org/licenses/LICENSE-2.0
10+
--
11+
-- Unless required by applicable law or agreed to in writing, software
12+
-- distributed under the License is distributed on an "AS IS" BASIS,
13+
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
-- See the License for the specific language governing permissions and
15+
-- limitations under the License.
16+
--
17+
18+
--- Outbound HTTP client selection.
19+
-- Shared by any module that makes outbound HTTP calls, so the choice between
20+
-- `ngx_http_ffi_client` and `lua-resty-http` is made in one place rather than
21+
-- per plugin. The caller decides where the preference comes from (its own
22+
-- config key) and passes the name in.
23+
24+
local core = require("apisix.core")
25+
local pcall = pcall
26+
local require = require
27+
local tostring = tostring
28+
local type = type
29+
30+
local FFI_CLIENT = "ngx_http_ffi_client"
31+
local LUA_RESTY_HTTP = "lua-resty-http"
32+
33+
-- the client name used in configuration is not the module name
34+
local CLIENT_MODULES = {
35+
[FFI_CLIENT] = "resty.ngx_http_ffi_client",
36+
[LUA_RESTY_HTTP] = "resty.http",
37+
}
38+
39+
local loaded = {}
40+
41+
42+
local _M = {
43+
version = 0.1,
44+
FFI_CLIENT = FFI_CLIENT,
45+
LUA_RESTY_HTTP = LUA_RESTY_HTTP,
46+
DEFAULT_CLIENT = FFI_CLIENT,
47+
}
48+
49+
50+
--- Schema fragment for a client-name config field.
51+
-- Callers embed this in their own attribute schema so every module validates
52+
-- the name the same way.
53+
_M.client_schema = {
54+
type = "string",
55+
enum = {FFI_CLIENT, LUA_RESTY_HTTP},
56+
default = FFI_CLIENT,
57+
}
58+
59+
60+
--- Load the module for a client name.
61+
-- `ngx_http_ffi_client` is a C client with the same object API as
62+
-- lua-resty-http and around half its outbound CPU cost, and it exists only
63+
-- when the gateway runtime was built with the module. A name that cannot be
64+
-- loaded is an error, never a silent switch to the other client.
65+
-- Cached per name once loaded, so a failure is retried rather than remembered.
66+
local function load_client(name)
67+
local cached = loaded[name]
68+
if cached then
69+
return cached
70+
end
71+
72+
local module_name = CLIENT_MODULES[name]
73+
if not module_name then
74+
return nil, "unknown http client: " .. tostring(name)
75+
end
76+
77+
local ok, mod = pcall(require, module_name)
78+
if not ok or type(mod) ~= "table" then
79+
core.log.error(module_name, " is not available: ", mod)
80+
return nil, module_name .. " is not available: " .. tostring(mod)
81+
end
82+
83+
-- Cosockets have their names resolved by apisix/patch.lua, which routes
84+
-- them through core.resolver and so honours dns_resolver, /etc/hosts and
85+
-- the search domains. The C client dials from C and never touches a
86+
-- cosocket, so without this it would see only nginx's `resolver`. Handing
87+
-- it the same resolver keeps every outbound name on one set of rules.
88+
-- A client too old to take one is an error rather than a client quietly
89+
-- resolving names by different rules than the rest of the gateway.
90+
if name == FFI_CLIENT then
91+
if type(mod.set_resolver) ~= "function" then
92+
core.log.error(module_name, " does not support set_resolver, ",
93+
"the runtime is older than the pinned one")
94+
return nil, module_name .. " does not support set_resolver, "
95+
.. "the runtime is older than the pinned one"
96+
end
97+
98+
mod.set_resolver(core.resolver.parse_domain)
99+
end
100+
101+
loaded[name] = mod
102+
103+
return mod
104+
end
105+
106+
107+
--- Create an HTTP client.
108+
-- @tparam string|nil name client name; defaults to DEFAULT_CLIENT
109+
-- @treturn table|nil the client
110+
-- @treturn string|nil error message
111+
function _M.new(name)
112+
name = name or _M.DEFAULT_CLIENT
113+
114+
local mod, err = load_client(name)
115+
if not mod then
116+
return nil, err
117+
end
118+
119+
-- The Lua half of `ngx_http_ffi_client` loads even when the C module is
120+
-- not compiled into the runtime; new() is what reports that.
121+
return mod.new()
122+
end
123+
124+
125+
return _M

ci/linux-install-openresty.sh

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -61,19 +61,19 @@ else
6161
sudo apt-get -y update --fix-missing
6262
sudo apt-get install -y build-essential gcc g++ cpanminus libxml2-dev libxslt-dev
6363

64-
if [ "$APISIX_RUNTIME" != "1.3.14" ]; then
64+
if [ "$APISIX_RUNTIME" != "1.3.16" ]; then
6565
echo "Please update the apisix-runtime-debug checksum for APISIX_RUNTIME=$APISIX_RUNTIME" >&2
6666
exit 1
6767
fi
6868

6969
case "$ARCH" in
7070
x86_64|amd64)
7171
DEB_ARCH="amd64"
72-
EXPECTED_SHA256="2d2350347c982e4467ff9326b5b93fcb9af2089b33b02bcd426e84a5adacf6f2"
72+
EXPECTED_SHA256="a56f0adc9bf6f6a491f7548df4f8e45fa3df3dd5e209d4a2ba5341b66eb7e060"
7373
;;
7474
arm64|aarch64)
7575
DEB_ARCH="arm64"
76-
EXPECTED_SHA256="495320e6377b96ab8d8a80a980a845e346c88160e2798ba46113a4da9042af4d"
76+
EXPECTED_SHA256="b645ee4f5ea36d26aaacb1b1c8278d89756b0b8448701ec561ab982b4768204a"
7777
;;
7878
*)
7979
echo "Unsupported architecture: $ARCH" >&2

conf/config.yaml.example

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -659,6 +659,9 @@ stream_plugins: # stream plugin list (sorted by priority)
659659
# protocols:
660660
# - name: pingpong
661661
plugin_attr: # Plugin attributes
662+
ai-proxy: # Plugin: ai-proxy, ai-proxy-multi
663+
http_client: ngx_http_ffi_client # HTTP client the AI plugins use to reach the
664+
# LLM upstream: ngx_http_ffi_client or lua-resty-http.
662665
log-rotate: # Plugin: log-rotate
663666
timeout: 10000 # maximum wait time for a log rotation(unit: millisecond)
664667
interval: 3600 # Set the log rotate interval in seconds.

docs/en/latest/plugins/ai-proxy-multi.md

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,23 @@ By default, `ai-proxy-multi` forwards the incoming client request headers to the
152152

153153
Because the LLM upstream is often a third-party service, be aware that any header the client sends (for example `Authorization`, `Cookie`, or internal application headers) is forwarded to that provider unless it is overridden by `auth.header`. If the client should not expose certain headers to the LLM provider, strip them before the request reaches `ai-proxy-multi`, for example with the [`proxy-rewrite`](./proxy-rewrite.md) plugin.
154154

155+
## Upstream HTTP Client
156+
157+
Requests to the LLM upstream go through `ngx_http_ffi_client`, a C HTTP client that costs around half the outbound CPU time of `lua-resty-http`. Both clients behave the same on the wire.
158+
159+
`plugin_attr.ai-proxy.http_client` in `config.yaml` names the client:
160+
161+
```yaml
162+
plugin_attr:
163+
ai-proxy:
164+
http_client: ngx_http_ffi_client # or lua-resty-http
165+
```
166+
167+
- `ngx_http_ffi_client` (default): the C client. It requires an APISIX runtime built with the module, which the runtime pinned in `.requirements` is. On a runtime without it, the request fails and the error names the missing module; the plugin never silently switches clients.
168+
- `lua-resty-http`: the Lua client, on every runtime.
169+
170+
The setting covers `ai-proxy`, `ai-proxy-multi`, and `ai-request-rewrite`, which share the same transport.
171+
155172
## Upstream Error Responses
156173

157174
When the selected LLM upstream returns a `429` or `5xx` status, `ai-proxy-multi` reads the upstream error body before deciding whether to fall back:

docs/en/latest/plugins/ai-proxy.md

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,23 @@ By default, `ai-proxy` forwards the incoming client request headers to the confi
145145

146146
Because the LLM upstream is often a third-party service, be aware that any header the client sends (for example `Authorization`, `Cookie`, or internal application headers) is forwarded to that provider unless it is overridden by `auth.header`. If the client should not expose certain headers to the LLM provider, strip them before the request reaches `ai-proxy`, for example with the [`proxy-rewrite`](./proxy-rewrite.md) plugin.
147147

148+
## Upstream HTTP Client
149+
150+
Requests to the LLM upstream go through `ngx_http_ffi_client`, a C HTTP client that costs around half the outbound CPU time of `lua-resty-http`. Both clients behave the same on the wire.
151+
152+
`plugin_attr.ai-proxy.http_client` in `config.yaml` names the client:
153+
154+
```yaml
155+
plugin_attr:
156+
ai-proxy:
157+
http_client: ngx_http_ffi_client # or lua-resty-http
158+
```
159+
160+
- `ngx_http_ffi_client` (default): the C client. It requires an APISIX runtime built with the module, which the runtime pinned in `.requirements` is. On a runtime without it, the request fails and the error names the missing module; the plugin never silently switches clients.
161+
- `lua-resty-http`: the Lua client, on every runtime.
162+
163+
The setting covers `ai-proxy`, `ai-proxy-multi`, and `ai-request-rewrite`, which share the same transport.
164+
148165
## Upstream Error Responses
149166

150167
When the LLM upstream returns a `429` or `5xx` status, `ai-proxy` reads the upstream error body and returns it to the client together with the upstream status code and `Content-Type`, so provider-side error details (such as rate-limit information or validation errors) are not discarded.

0 commit comments

Comments
 (0)