From 7aeec962c40ccb8afa2938c52b86819bff1b11fa Mon Sep 17 00:00:00 2001 From: sirouk <8901571+sirouk@users.noreply.github.com> Date: Fri, 24 Apr 2026 10:57:26 -0400 Subject: [PATCH 1/4] strip thinking, users can control from chat kwargs --- lua/e2ee_handler.lua | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lua/e2ee_handler.lua b/lua/e2ee_handler.lua index 71938e5..6d60fb6 100644 --- a/lua/e2ee_handler.lua +++ b/lua/e2ee_handler.lua @@ -110,6 +110,19 @@ end function _M.e2ee_round_trip(api_key, model, body_json, is_streaming, e2e_path, on_chunk) local err + -- Strip :THINKING suffix and enable thinking mode in chat_template_kwargs. + -- Users can also control thinking directly via chat_template_kwargs in + -- the request body; this is just the :THINKING shorthand from chutes-api. + if model:sub(-9) == ":THINKING" then + model = model:match("^(.-):THINKING") + local payload = cjson.decode(body_json) + payload.model = model + payload.chat_template_kwargs = payload.chat_template_kwargs or {} + payload.chat_template_kwargs.thinking = true + payload.chat_template_kwargs.enable_thinking = true + body_json = cjson.encode(payload) + end + -- Resolve model -> chute_id local chute_id chute_id, err = discovery.resolve_chute_id(model, api_key) From bf119ca39f644eadaabe9be4571aded4b82a47f7 Mon Sep 17 00:00:00 2001 From: sirouk <8901571+sirouk@users.noreply.github.com> Date: Tue, 5 May 2026 13:07:57 -0400 Subject: [PATCH 2/4] update for thinking in chat template norm --- lua/e2ee_handler.lua | 53 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/lua/e2ee_handler.lua b/lua/e2ee_handler.lua index 6d60fb6..0ad40f2 100644 --- a/lua/e2ee_handler.lua +++ b/lua/e2ee_handler.lua @@ -110,16 +110,49 @@ end function _M.e2ee_round_trip(api_key, model, body_json, is_streaming, e2e_path, on_chunk) local err - -- Strip :THINKING suffix and enable thinking mode in chat_template_kwargs. - -- Users can also control thinking directly via chat_template_kwargs in - -- the request body; this is just the :THINKING shorthand from chutes-api. - if model:sub(-9) == ":THINKING" then - model = model:match("^(.-):THINKING") - local payload = cjson.decode(body_json) - payload.model = model - payload.chat_template_kwargs = payload.chat_template_kwargs or {} - payload.chat_template_kwargs.thinking = true - payload.chat_template_kwargs.enable_thinking = true + local payload = cjson.decode(body_json) + local changed = false + if payload then + local thinking = nil + if model:sub(-9) == ":THINKING" then + model = model:match("^(.-):THINKING") + payload.model = model + thinking = true + changed = true + end + + local headers = ngx.req.get_headers() + local thinking_header = headers["X-Enable-Thinking"] or headers["x-enable-thinking"] + if type(thinking_header) == "table" then + thinking_header = thinking_header[1] + end + thinking_header = type(thinking_header) == "string" and thinking_header:lower() or thinking_header + if thinking_header == true or thinking_header == false + or thinking_header == "true" or thinking_header == "false" then + thinking = thinking_header == true or thinking_header == "true" + changed = true + end + + local kwargs = type(payload.chat_template_kwargs) == "table" and payload.chat_template_kwargs or nil + if thinking ~= nil then + kwargs = kwargs or {} + payload.chat_template_kwargs = kwargs + kwargs.thinking = thinking + kwargs.enable_thinking = thinking + end + if kwargs then + if kwargs.thinking ~= nil and kwargs.enable_thinking == nil then + kwargs.enable_thinking = kwargs.thinking + changed = true + end + if kwargs.enable_thinking ~= nil and kwargs.thinking == nil then + kwargs.thinking = kwargs.enable_thinking + changed = true + end + end + end + + if changed then body_json = cjson.encode(payload) end From fe0c080a59538b7242f3481ef08e5d62f7c98147 Mon Sep 17 00:00:00 2001 From: sirouk <8901571+sirouk@users.noreply.github.com> Date: Fri, 29 May 2026 16:07:26 -0400 Subject: [PATCH 3/4] Support thinking header normalization --- lua/claude_handler.lua | 5 ++- lua/e2ee_handler.lua | 90 ++++++++++++++++++++++----------------- lua/responses_handler.lua | 5 ++- 3 files changed, 58 insertions(+), 42 deletions(-) diff --git a/lua/claude_handler.lua b/lua/claude_handler.lua index 8531a1c..9538cb4 100644 --- a/lua/claude_handler.lua +++ b/lua/claude_handler.lua @@ -54,7 +54,7 @@ function _M.handle() if not is_streaming then -- Non-streaming local decrypted, round_err = e2ee.e2ee_round_trip( - api_key, model, oai_body, false, "/v1/chat/completions" + api_key, model, oai_body, false, "/v1/chat/completions", nil, oai_request ) if not decrypted then @@ -101,7 +101,8 @@ function _M.handle() ngx.flush(true) end end - end) + end, + oai_request) if round_err then if round_err.raw then diff --git a/lua/e2ee_handler.lua b/lua/e2ee_handler.lua index 0ad40f2..df4f64c 100644 --- a/lua/e2ee_handler.lua +++ b/lua/e2ee_handler.lua @@ -107,53 +107,64 @@ end --- E2EE round-trip: encrypt request, send, decrypt response -- For non-streaming: returns (decrypted_json_string, nil) or (nil, {status=N, message=S}) -- For streaming: calls on_chunk(line) for each decrypted SSE data line, on_chunk(nil) at end -function _M.e2ee_round_trip(api_key, model, body_json, is_streaming, e2e_path, on_chunk) +function _M.e2ee_round_trip(api_key, model, body_json, is_streaming, e2e_path, on_chunk, request_payload) local err - local payload = cjson.decode(body_json) + local thinking + local base_model = model:match("^(.-):THINKING$") local changed = false - if payload then - local thinking = nil - if model:sub(-9) == ":THINKING" then - model = model:match("^(.-):THINKING") - payload.model = model - thinking = true - changed = true - end - local headers = ngx.req.get_headers() - local thinking_header = headers["X-Enable-Thinking"] or headers["x-enable-thinking"] - if type(thinking_header) == "table" then - thinking_header = thinking_header[1] - end - thinking_header = type(thinking_header) == "string" and thinking_header:lower() or thinking_header - if thinking_header == true or thinking_header == false - or thinking_header == "true" or thinking_header == "false" then - thinking = thinking_header == true or thinking_header == "true" - changed = true - end + if base_model then + model = base_model + thinking = true + end - local kwargs = type(payload.chat_template_kwargs) == "table" and payload.chat_template_kwargs or nil - if thinking ~= nil then - kwargs = kwargs or {} - payload.chat_template_kwargs = kwargs - kwargs.thinking = thinking - kwargs.enable_thinking = thinking + local h = ngx.var and ngx.var.http_x_enable_thinking + if type(h) == "string" then + h = h:lower() + if h == "true" then + thinking = true + elseif h == "false" then + thinking = false end - if kwargs then - if kwargs.thinking ~= nil and kwargs.enable_thinking == nil then - kwargs.enable_thinking = kwargs.thinking + end + + local kwargs_hint = type(request_payload) == "table" + and type(request_payload.chat_template_kwargs) == "table" + and request_payload.chat_template_kwargs or nil + local normalize_kwargs = kwargs_hint + and ((kwargs_hint.thinking ~= nil and kwargs_hint.enable_thinking == nil) + or (kwargs_hint.enable_thinking ~= nil and kwargs_hint.thinking == nil)) + + if thinking ~= nil or normalize_kwargs then + local payload = cjson.decode(body_json) + if payload then + if base_model then + payload.model = model changed = true end - if kwargs.enable_thinking ~= nil and kwargs.thinking == nil then - kwargs.thinking = kwargs.enable_thinking + + local kwargs = type(payload.chat_template_kwargs) == "table" and payload.chat_template_kwargs or nil + if thinking ~= nil then + kwargs = kwargs or {} + kwargs.thinking = thinking + kwargs.enable_thinking = thinking + payload.chat_template_kwargs = kwargs changed = true + elseif kwargs then + if kwargs.thinking ~= nil and kwargs.enable_thinking == nil then + kwargs.enable_thinking = kwargs.thinking + changed = true + elseif kwargs.enable_thinking ~= nil and kwargs.thinking == nil then + kwargs.thinking = kwargs.enable_thinking + changed = true + end end - end - end - if changed then - body_json = cjson.encode(payload) + if changed then + body_json = cjson.encode(payload) + end + end end -- Resolve model -> chute_id @@ -400,7 +411,9 @@ function _M.handle() end if not is_streaming then - local decrypted, round_err = _M.e2ee_round_trip(api_key, model, body, false, original_path) + local decrypted, round_err = _M.e2ee_round_trip( + api_key, model, body, false, original_path, nil, payload + ) if not decrypted then if round_err.raw then ngx.status = round_err.status @@ -430,7 +443,8 @@ function _M.handle() ngx.flush(true) end end - end) + end, + payload) if round_err then if round_err.raw then diff --git a/lua/responses_handler.lua b/lua/responses_handler.lua index 10bf166..7b277c2 100644 --- a/lua/responses_handler.lua +++ b/lua/responses_handler.lua @@ -54,7 +54,7 @@ function _M.handle() if not is_streaming then -- Non-streaming local decrypted, round_err = e2ee.e2ee_round_trip( - api_key, model, oai_body, false, "/v1/chat/completions" + api_key, model, oai_body, false, "/v1/chat/completions", nil, oai_request ) if not decrypted then @@ -101,7 +101,8 @@ function _M.handle() ngx.flush(true) end end - end) + end, + oai_request) if round_err then if round_err.raw then From e44a104b07ab70a64323766ba5380694d0ea2325 Mon Sep 17 00:00:00 2001 From: sirouk <8901571+sirouk@users.noreply.github.com> Date: Fri, 29 May 2026 16:07:52 -0400 Subject: [PATCH 4/4] Preserve empty arrays in thinking payload rewrites --- lua/e2ee_handler.lua | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/lua/e2ee_handler.lua b/lua/e2ee_handler.lua index df4f64c..7108ee8 100644 --- a/lua/e2ee_handler.lua +++ b/lua/e2ee_handler.lua @@ -14,9 +14,20 @@ local cjson = require("cjson.safe") local http = require("resty.http") local API_BASE = "https://api.chutes.ai" +local payload_cjson local _M = {} +local function get_payload_cjson() + if not payload_cjson then + payload_cjson = cjson.new() + if payload_cjson.decode_array_with_array_mt then + payload_cjson.decode_array_with_array_mt(true) + end + end + return payload_cjson +end + --- Extract API key from Authorization header or x-api-key function _M.get_api_key() local headers = ngx.req.get_headers() @@ -137,7 +148,8 @@ function _M.e2ee_round_trip(api_key, model, body_json, is_streaming, e2e_path, o or (kwargs_hint.enable_thinking ~= nil and kwargs_hint.thinking == nil)) if thinking ~= nil or normalize_kwargs then - local payload = cjson.decode(body_json) + local json = get_payload_cjson() + local payload = json.decode(body_json) if payload then if base_model then payload.model = model @@ -145,6 +157,9 @@ function _M.e2ee_round_trip(api_key, model, body_json, is_streaming, e2e_path, o end local kwargs = type(payload.chat_template_kwargs) == "table" and payload.chat_template_kwargs or nil + if kwargs and getmetatable(kwargs) == json.array_mt then + kwargs = nil + end if thinking ~= nil then kwargs = kwargs or {} kwargs.thinking = thinking @@ -162,7 +177,7 @@ function _M.e2ee_round_trip(api_key, model, body_json, is_streaming, e2e_path, o end if changed then - body_json = cjson.encode(payload) + body_json = json.encode(payload) end end end