feat(ai-proxy): send LLM requests through ngx_http_ffi_client

.requirements

@@ -17,5 +17,5 @@
 
 APISIX_PACKAGE_NAME=apisix
 
-APISIX_RUNTIME=1.3.11
+APISIX_RUNTIME=1.3.16
 APISIX_DASHBOARD_COMMIT=c8d3466d3c36386d3888efbc8250cd8183c77298

apisix/cli/config.lua

@@ -308,6 +308,9 @@ local _M = {
   },
   stream_plugins = { "ip-restriction", "limit-conn", "mqtt-proxy", "syslog", "traffic-split" },
   plugin_attr = {
+    ["ai-proxy"] = {
+      http_client = "ngx_http_ffi_client"
+    },
     ["log-rotate"] = {
       timeout = 10000,
       interval = 3600,

apisix/plugins/ai-transport/http.lua

@@ -19,7 +19,7 @@
 -- Provides HTTP client lifecycle management for AI provider requests.
 
 local core = require("apisix.core")
-local http = require("resty.http")
+local http_client = require("apisix.utils.http")
 local ngx_now = ngx.now
 local pairs = pairs
 local ipairs = ipairs
@@ -28,8 +28,41 @@ local type = type
 local str_lower = string.lower
 local tostring = tostring
 
+local attr_schema = {
+    type = "object",
+    properties = {
+        http_client = http_client.client_schema,
+    },
+}
+
 local _M = {}
 
+local client_name
+
+
+--- Which client this transport should use.
+-- `plugin_attr.ai-proxy.http_client` names it; the shared module owns the
+-- names, validation and loading. Read on first request, because local_conf is
+-- not readable while this module is still loading.
+local function resolve_client_name()
+    if client_name then
+        return client_name
+    end
+
+    local local_conf = core.config.local_conf()
+    local attr = core.table.try_read_attr(local_conf, "plugin_attr", "ai-proxy") or {}
+
+    local ok, err = core.schema.check(attr_schema, attr)
+    if not ok then
+        core.log.error("invalid plugin_attr.ai-proxy: ", err)
+        return nil, "invalid plugin_attr.ai-proxy: " .. err
+    end
+
+    client_name = attr.http_client or http_client.DEFAULT_CLIENT
+
+    return client_name
+end
+
 
 --- Map network errors to HTTP status codes.
 -- Cosocket timers report "timeout"; OS errno (ETIMEDOUT) and the resolver
@@ -100,7 +133,12 @@ end
 -- @return string|nil Error message
 -- @return table|nil Upstream metadata on failure (for recording failed attempts)
 function _M.request(params, timeout)
-    local httpc, err = http.new()
+    local name, name_err = resolve_client_name()
+    if not name then
+        return nil, "failed to create http client: " .. name_err
+    end
+
+    local httpc, err = http_client.new(name)
     if not httpc then
         return nil, "failed to create http client: " .. (err or "unknown")
     end
@@ -111,6 +149,19 @@ function _M.request(params, timeout)
     local upstream_scheme = params.scheme or "http"
     local t0 = ngx_now()
 
+    if http_client.needs_resolve(name) then
+        local resolved, rerr = http_client.resolve_upstream_host(params)
+        if not resolved then
+            return nil, "connect: " .. rerr, {
+                upstream_addr = upstream_addr,
+                upstream_host = upstream_host,
+                upstream_scheme = upstream_scheme,
+                upstream_uri = params.path,
+                t0 = t0,
+            }
+        end
+    end
+
     local ok, err = httpc:connect(params)
     if not ok then
         return nil, "connect: " .. (err or "unknown"), {

apisix/utils/http.lua

@@ -0,0 +1,155 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements.  See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+--- Outbound HTTP client selection.
+-- Shared by any module that makes outbound HTTP calls, so the choice between
+-- `ngx_http_ffi_client` and `lua-resty-http` is made in one place rather than
+-- per plugin. The caller decides where the preference comes from (its own
+-- config key) and passes the name in.
+
+local core = require("apisix.core")
+local pcall = pcall
+local require = require
+local tonumber = tonumber
+local tostring = tostring
+local type = type
+
+local FFI_CLIENT = "ngx_http_ffi_client"
+local LUA_RESTY_HTTP = "lua-resty-http"
+
+-- the client name used in configuration is not the module name
+local CLIENT_MODULES = {
+    [FFI_CLIENT] = "resty.ngx_http_ffi_client",
+    [LUA_RESTY_HTTP] = "resty.http",
+}
+
+local loaded = {}
+
+
+local _M = {
+    version = 0.1,
+    FFI_CLIENT = FFI_CLIENT,
+    LUA_RESTY_HTTP = LUA_RESTY_HTTP,
+    DEFAULT_CLIENT = FFI_CLIENT,
+}
+
+
+--- Schema fragment for a client-name config field.
+-- Callers embed this in their own attribute schema so every module validates
+-- the name the same way.
+_M.client_schema = {
+    type = "string",
+    enum = {FFI_CLIENT, LUA_RESTY_HTTP},
+    default = FFI_CLIENT,
+}
+
+
+--- Load the module for a client name.
+-- `ngx_http_ffi_client` is a C client with the same object API as
+-- lua-resty-http and around half its outbound CPU cost, and it exists only
+-- when the gateway runtime was built with the module. A name that cannot be
+-- loaded is an error, never a silent switch to the other client.
+-- Cached per name once loaded, so a failure is retried rather than remembered.
+local function load_client(name)
+    local cached = loaded[name]
+    if cached then
+        return cached
+    end
+
+    local module_name = CLIENT_MODULES[name]
+    if not module_name then
+        return nil, "unknown http client: " .. tostring(name)
+    end
+
+    local ok, mod = pcall(require, module_name)
+    if not ok or type(mod) ~= "table" then
+        core.log.error(module_name, " is not available: ", mod)
+        return nil, module_name .. " is not available: " .. tostring(mod)
+    end
+
+    loaded[name] = mod
+
+    return mod
+end
+
+
+--- Create an HTTP client.
+-- @tparam string|nil name client name; defaults to DEFAULT_CLIENT
+-- @treturn table|nil the client
+-- @treturn string|nil error message
+function _M.new(name)
+    name = name or _M.DEFAULT_CLIENT
+
+    local mod, err = load_client(name)
+    if not mod then
+        return nil, err
+    end
+
+    -- The Lua half of `ngx_http_ffi_client` loads even when the C module is
+    -- not compiled into the runtime; new() is what reports that.
+    return mod.new()
+end
+
+
+--- Whether a client name needs resolve_upstream_host() before connecting.
+function _M.needs_resolve(name)
+    return (name or _M.DEFAULT_CLIENT) == FFI_CLIENT
+end
+
+
+--- Resolve the upstream name the way every other socket in the gateway does.
+-- Cosockets are patched (apisix/patch.lua) to run names through core.resolver,
+-- which honours dns_resolver, /etc/hosts and the search domains. A client that
+-- dials from C never touches a cosocket and only sees nginx's `resolver`, so
+-- the name is resolved here and kept for the Host header and the SNI.
+-- Mutates `params` in place.
+-- @treturn boolean|nil true on success
+-- @treturn string|nil error message
+function _M.resolve_upstream_host(params)
+    local host = params.host
+    if not host
+       or core.utils.parse_ipv4(host)
+       or core.utils.parse_ipv6(host)
+    then
+        return true
+    end
+
+    local ip, err = core.resolver.parse_domain(host)
+    if not ip then
+        return nil, "failed to parse domain: " .. (err or "unknown")
+    end
+
+    params.ssl_server_name = params.ssl_server_name or host
+
+    local headers = params.headers or {}
+    if not headers["Host"] and not headers["host"] then
+        local default_port = params.scheme == "https" and 443 or 80
+        if params.port and tonumber(params.port) ~= default_port then
+            headers["Host"] = host .. ":" .. params.port
+        else
+            headers["Host"] = host
+        end
+    end
+    params.headers = headers
+
+    params.host = ip
+
+    return true
+end
+
+
+return _M

ci/linux-install-openresty.sh

@@ -61,19 +61,19 @@ else
     sudo apt-get -y update --fix-missing
     sudo apt-get install -y build-essential gcc g++ cpanminus libxml2-dev libxslt-dev
 
-    if [ "$APISIX_RUNTIME" != "1.3.11" ]; then
+    if [ "$APISIX_RUNTIME" != "1.3.16" ]; then
         echo "Please update the apisix-runtime-debug checksum for APISIX_RUNTIME=$APISIX_RUNTIME" >&2
         exit 1
     fi
 
     case "$ARCH" in
         x86_64|amd64)
             DEB_ARCH="amd64"
-            EXPECTED_SHA256="6c03f0a47a80e84c595c7e067f7d05fc69890237f9191af55108a284b356c4ee"
+            EXPECTED_SHA256="a56f0adc9bf6f6a491f7548df4f8e45fa3df3dd5e209d4a2ba5341b66eb7e060"
             ;;
         arm64|aarch64)
             DEB_ARCH="arm64"
-            EXPECTED_SHA256="cdc124262a1acb2de170f12a2180cdc357ba867d6447cd08a9ba1639994d4e50"
+            EXPECTED_SHA256="b645ee4f5ea36d26aaacb1b1c8278d89756b0b8448701ec561ab982b4768204a"
             ;;
         *)
             echo "Unsupported architecture: $ARCH" >&2

conf/config.yaml.example

@@ -642,6 +642,9 @@ stream_plugins:                    # stream plugin list (sorted by priority)
 #   protocols:
 #     - name: pingpong
 plugin_attr:          # Plugin attributes
+  ai-proxy:           # Plugin: ai-proxy, ai-proxy-multi
+    http_client: ngx_http_ffi_client # HTTP client the AI plugins use to reach the
+                      # LLM upstream: ngx_http_ffi_client or lua-resty-http.
   log-rotate:         # Plugin: log-rotate
     timeout: 10000    # maximum wait time for a log rotation(unit: millisecond)
     interval: 3600    # Set the log rotate interval in seconds.

docs/en/latest/plugins/ai-proxy-multi.md

@@ -152,6 +152,23 @@ By default, `ai-proxy-multi` forwards the incoming client request headers to the
 
 Because the LLM upstream is often a third-party service, be aware that any header the client sends (for example `Authorization`, `Cookie`, or internal application headers) is forwarded to that provider unless it is overridden by `auth.header`. If the client should not expose certain headers to the LLM provider, strip them before the request reaches `ai-proxy-multi`, for example with the [`proxy-rewrite`](./proxy-rewrite.md) plugin.
 
+## Upstream HTTP Client
+
+Requests to the LLM upstream go through `ngx_http_ffi_client`, a C HTTP client that costs around half the outbound CPU time of `lua-resty-http`. Both clients behave the same on the wire.
+
+`plugin_attr.ai-proxy.http_client` in `config.yaml` names the client:
+
+```yaml
+plugin_attr:
+  ai-proxy:
+    http_client: ngx_http_ffi_client # or lua-resty-http
+```
+
+- `ngx_http_ffi_client` (default): the C client. It requires an APISIX runtime built with the module, which the runtime pinned in `.requirements` is. On a runtime without it, the request fails and the error names the missing module; the plugin never silently switches clients.
+- `lua-resty-http`: the Lua client, on every runtime.
+
+The setting covers `ai-proxy`, `ai-proxy-multi`, and `ai-request-rewrite`, which share the same transport.
+
 ## Upstream Error Responses
 
 When the selected LLM upstream returns a `429` or `5xx` status, `ai-proxy-multi` reads the upstream error body before deciding whether to fall back:

docs/en/latest/plugins/ai-proxy.md

@@ -145,6 +145,23 @@ By default, `ai-proxy` forwards the incoming client request headers to the confi
 
 Because the LLM upstream is often a third-party service, be aware that any header the client sends (for example `Authorization`, `Cookie`, or internal application headers) is forwarded to that provider unless it is overridden by `auth.header`. If the client should not expose certain headers to the LLM provider, strip them before the request reaches `ai-proxy`, for example with the [`proxy-rewrite`](./proxy-rewrite.md) plugin.
 
+## Upstream HTTP Client
+
+Requests to the LLM upstream go through `ngx_http_ffi_client`, a C HTTP client that costs around half the outbound CPU time of `lua-resty-http`. Both clients behave the same on the wire.
+
+`plugin_attr.ai-proxy.http_client` in `config.yaml` names the client:
+
+```yaml
+plugin_attr:
+  ai-proxy:
+    http_client: ngx_http_ffi_client # or lua-resty-http
+```
+
+- `ngx_http_ffi_client` (default): the C client. It requires an APISIX runtime built with the module, which the runtime pinned in `.requirements` is. On a runtime without it, the request fails and the error names the missing module; the plugin never silently switches clients.
+- `lua-resty-http`: the Lua client, on every runtime.
+
+The setting covers `ai-proxy`, `ai-proxy-multi`, and `ai-request-rewrite`, which share the same transport.
+
 ## Upstream Error Responses
 
 When the LLM upstream returns a `429` or `5xx` status, `ai-proxy` reads the upstream error body and returns it to the client together with the upstream status code and `Content-Type`, so provider-side error details (such as rate-limit information or validation errors) are not discarded.