fix(ai-proxy-multi): preserve request_llm_model across fallback retries [FaaFyfxR9WAQrL7FcAgEHJvztd8cVMxvjHRS55rw1nwH]

apisix/plugins/ai-proxy-multi.lua

@@ -956,7 +956,8 @@ local function retry_on_error(ctx, conf, code, body)
         -- The failed attempt's body never reaches the client (a later attempt
         -- responds instead), so surface the upstream error here for diagnostics.
         core.log.warn("ai instance ", failed_instance, " returned status ", code,
-                      ", falling back to ", name, ". upstream error body: ",
+                      ", falling back to ", name, ", request_llm_model=",
+                      ctx.var.request_llm_model, ". upstream error body: ",
                       body or "")
         ctx.balancer_ip = name
         ctx.picked_ai_instance_name = name

apisix/plugins/ai-proxy/base.lua

@@ -242,8 +242,14 @@ function _M.before_proxy(conf, ctx, on_error)
 
         -- Step 2: Extract model from request
         local request_model = request_body.model
-
-        if request_model then
+        -- request_body is the cached table from get_json_request_body_table.
+        -- On a fallback retry, build_body has already overwritten body.model with
+        -- the picked instance's model, so re-reading it here would clobber
+        -- $request_llm_model with the retry instance's model instead of keeping
+        -- the client's original. Capture the client model only on the first
+        -- attempt and preserve it across retries.
+        if request_model and not ctx.ai_request_llm_captured then
+            ctx.ai_request_llm_captured = true
             ctx.var.request_llm_model = request_model
         end
         local model = ai_instance.options and ai_instance.options.model or request_model

t/plugin/ai-proxy-multi-retry.t

@@ -217,3 +217,48 @@ POST /anything
 --- response_body_like: slow internal error
 --- error_log
 exceeding retry_on_failure_within_ms 200
+
+
+
+=== TEST 7: fallback preserves $request_llm_model as the client's original model
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/routes/1',
+                 ngx.HTTP_PUT,
+                 [[{
+                    "uri": "/anything",
+                    "plugins": {
+                        "ai-proxy-multi": {
+                            "fallback_strategy": ["http_5xx"],
+                            "instances": [
+                                {"name":"err-1","provider":"openai-compatible","weight":1,"priority":10,"auth":{"header":{"Authorization":"Bearer token"}},"options":{"model":"gpt-4"},"override":{"endpoint":"http://127.0.0.1:6731"}},
+                                {"name":"success","provider":"openai-compatible","weight":1,"priority":0,"auth":{"header":{"Authorization":"Bearer token"}},"options":{"model":"gpt-4-turbo"},"override":{"endpoint":"http://127.0.0.1:6733"}}
+                            ],
+                            "ssl_verify": false
+                        }
+                    }
+                }]]
+            )
+            if code >= 300 then
+                ngx.status = code
+            end
+            ngx.say(body)
+        }
+    }
+--- response_body
+passed
+
+
+=== TEST 8: request_llm_model retains the client's original model across fallback
+--- request
+POST /anything
+{ "model": "model_test", "messages": [ { "role": "user", "content": "What is 1+1?"} ] }
+--- response_body chomp
+success
+--- error_code: 200
+--- error_log
+request_llm_model=model_test
+--- no_error_log
+request_llm_model=gpt-4