diff --git a/.github/workflows/query-gateway-elasticsearch-integration.yml b/.github/workflows/query-gateway-elasticsearch-integration.yml new file mode 100644 index 000000000000..75ef7dcc5769 --- /dev/null +++ b/.github/workflows/query-gateway-elasticsearch-integration.yml @@ -0,0 +1,48 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: Query Gateway Elasticsearch Integration + +on: + pull_request: + paths: + - apisix/plugins/query-gateway/** + - apisix/cli/config.lua + - t/integration/query-gateway-elasticsearch/** + workflow_dispatch: + +permissions: + contents: read + +jobs: + query-gateway-elasticsearch: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + submodules: recursive + - name: Run query gateway Elasticsearch integration profile + working-directory: t/integration/query-gateway-elasticsearch + run: make run + - name: Upload integration artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: query-gateway-elasticsearch-artifacts + path: t/integration/query-gateway-elasticsearch/artifacts + if-no-files-found: warn diff --git a/Makefile b/Makefile index 667a0774ee0a..31c09b448bde 100644 --- a/Makefile +++ b/Makefile @@ -340,6 +340,9 @@ install: runtime $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/proxy-cache $(ENV_INSTALL) apisix/plugins/proxy-cache/*.lua $(ENV_INST_LUADIR)/apisix/plugins/proxy-cache/ + $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/query-gateway + $(ENV_INSTALL) apisix/plugins/query-gateway/*.lua $(ENV_INST_LUADIR)/apisix/plugins/query-gateway/ + $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/serverless $(ENV_INSTALL) apisix/plugins/serverless/*.lua $(ENV_INST_LUADIR)/apisix/plugins/serverless/ diff --git a/apisix/cli/config.lua b/apisix/cli/config.lua index 98c3ff581218..f416aaa7a389 100644 --- a/apisix/cli/config.lua +++ b/apisix/cli/config.lua @@ -164,6 +164,8 @@ local _M = { ["plugin-limit-count"] = "10m", ["prometheus-metrics"] = "128m", ["plugin-limit-conn"] = "10m", + ["query-gateway-cache"] = "32m", + ["query-gateway-redis-cluster-slot-lock"] = "1m", ["worker-events"] = "10m", ["lrucache-lock"] = "10m", ["balancer-ewma"] = "10m", @@ -254,6 +256,7 @@ local _M = { "proxy-mirror", "graphql-proxy-cache", "proxy-rewrite", + "query-gateway", "workflow", "api-breaker", "graphql-limit-count", diff --git a/apisix/plugins/query-gateway.lua b/apisix/plugins/query-gateway.lua new file mode 100644 index 000000000000..c3e5056187a3 --- /dev/null +++ b/apisix/plugins/query-gateway.lua @@ -0,0 +1,186 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local cache = require("apisix.plugins.query-gateway.cache") +local core = require("apisix.core") +local ngx = ngx + +local plugin_name = "query-gateway" + +local cache_schema = { + type = "object", + properties = { + enabled = {type = "boolean", default = false}, + backend = { + type = "string", + enum = {"local", "redis", "redis-cluster"}, + default = "local", + }, + ttl = {type = "integer", minimum = 1, default = 30}, + fallback_ttl = {type = "integer", minimum = 1, default = 5}, + max_request_body_size = {type = "integer", minimum = 1, default = 262144}, + max_response_body_size = {type = "integer", minimum = 1, default = 1048576}, + cookie_names = { + type = "array", + uniqueItems = true, + items = {type = "string", minLength = 1, maxLength = 256}, + }, + redis_host = {type = "string", minLength = 1}, + redis_port = {type = "integer", minimum = 1, maximum = 65535, default = 6379}, + redis_timeout = {type = "integer", minimum = 1, default = 1000}, + redis_username = {type = "string", minLength = 1}, + redis_password = {type = "string", minLength = 1}, + redis_database = {type = "integer", minimum = 0, default = 0}, + redis_ssl = {type = "boolean", default = false}, + redis_ssl_verify = {type = "boolean", default = false}, + redis_keepalive_timeout = {type = "integer", minimum = 1, default = 10000}, + redis_keepalive_pool = {type = "integer", minimum = 1, default = 100}, + redis_cluster_name = {type = "string", minLength = 1}, + redis_cluster_nodes = { + type = "array", + minItems = 1, + items = {type = "string", minLength = 1}, + }, + redis_cluster_ssl = {type = "boolean", default = false}, + redis_cluster_ssl_verify = {type = "boolean", default = false}, + }, + additionalProperties = false, +} + +local schema = { + type = "object", + properties = { + preserve_original_method_header = { + description = "whether to forward the original request method", + type = "boolean", + default = true, + }, + original_method_header = { + description = "header used to forward the original request method", + type = "string", + default = "X-Original-Method", + minLength = 1, + maxLength = 128, + }, + query = { + type = "object", + properties = { + upstream_method = { + type = "string", + enum = {"post", "query"}, + default = "post", + }, + }, + additionalProperties = false, + }, + post = { + type = "object", + properties = { + cache_enabled = {type = "boolean", default = false}, + read_only = {type = "boolean", default = false}, + }, + additionalProperties = false, + }, + cache = cache_schema, + }, + additionalProperties = false, + encrypt_fields = {"cache.redis_password"}, +} + +local _M = { + version = 0.3, + priority = -1001, + name = plugin_name, + schema = schema, +} + +function _M.check_schema(conf) + local ok, err = core.schema.check(schema, conf) + if not ok then + return false, err + end + + if conf.preserve_original_method_header ~= false + and not core.utils.validate_header_field(conf.original_method_header + or "X-Original-Method") then + return false, "invalid original_method_header" + end + + local cache_conf = conf.cache + if cache_conf and cache_conf.enabled then + if cache_conf.backend == "redis" and not cache_conf.redis_host then + return false, "cache.redis_host is required for the redis backend" + end + + if cache_conf.backend == "redis-cluster" + and (not cache_conf.redis_cluster_name or not cache_conf.redis_cluster_nodes) then + return false, "cache.redis_cluster_name and cache.redis_cluster_nodes are required " .. + "for the redis-cluster backend" + end + end + + return true +end + +function _M.access(conf, ctx) + local method = ngx.req.get_method() + if method ~= "QUERY" and method ~= "POST" then + return + end + + ctx.query_gateway_client_method = method + + local cache_conf = conf.cache + local post_cache_enabled = conf.post and conf.post.cache_enabled and conf.post.read_only + if cache_conf and cache_conf.enabled and (method == "QUERY" or post_cache_enabled) then + local entry, status = cache.fetch(cache_conf, ctx) + if entry then + ctx.query_gateway_cache_hit = true + return cache.serve(entry) + end + + if method == "QUERY" and status == "missing content-type" then + return 400 + end + end + + if method ~= "QUERY" then + return + end + + ctx.query_gateway_original_method = method + + if conf.preserve_original_method_header ~= false then + core.request.set_header(ctx, conf.original_method_header or "X-Original-Method", method) + end + + if not conf.query or conf.query.upstream_method ~= "query" then + ngx.req.set_method(ngx.HTTP_POST) + end +end +function _M.header_filter(conf, ctx) + if conf.cache and conf.cache.enabled then + cache.header_filter(conf.cache, ctx) + end +end + +function _M.body_filter(conf, ctx) + if conf.cache and conf.cache.enabled then + cache.body_filter(conf.cache, ctx) + end +end + +return _M diff --git a/apisix/plugins/query-gateway/cache.lua b/apisix/plugins/query-gateway/cache.lua new file mode 100644 index 000000000000..34986f1b969c --- /dev/null +++ b/apisix/plugins/query-gateway/cache.lua @@ -0,0 +1,455 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local core = require("apisix.core") +local redis = require("apisix.utils.redis") +local rediscluster = require("apisix.utils.rediscluster") +local resty_sha256 = require("resty.sha256") +local to_hex = require("resty.string").to_hex +local ngx = ngx + +local concat = table.concat +local lower = string.lower +local pairs = pairs +local ipairs = ipairs +local math_min = math.min +local table_sort = table.sort +local tonumber = tonumber + +local _M = {} + +local HOP_BY_HOP = { + connection = true, + ["keep-alive"] = true, + ["proxy-authenticate"] = true, + ["proxy-authorization"] = true, + te = true, + trailer = true, + ["transfer-encoding"] = true, + upgrade = true, +} + +local ALLOWED_VARY = { + accept = true, + ["accept-encoding"] = true, + ["accept-language"] = true, +} + +local function sha256_hex(value) + local sha256 = resty_sha256:new() + sha256:update(value) + return to_hex(sha256:final()) +end + +local function shared_dict() + return ngx.shared["query-gateway-cache"] +end + +local function cache_id(conf) + if conf.backend == "redis" then + return "redis:" .. conf.redis_host .. ":" .. (conf.redis_port or 6379) + end + + if conf.backend == "redis-cluster" then + return "redis-cluster:" .. conf.redis_cluster_name + end + + return "local" +end + +local function local_key(key) + return "entry:" .. key +end + +local function breaker_key(conf) + return "breaker:" .. cache_id(conf) +end + +local function use_fallback(conf) + if conf.backend == "local" then + return true + end + + return shared_dict():get(breaker_key(conf)) ~= nil +end + +local function mark_backend_failure(conf, err) + shared_dict():set(breaker_key(conf), true, conf.fallback_ttl) + core.log.warn("query-gateway cache backend unavailable: ", err, + "; using local memory for ", conf.fallback_ttl, " seconds") +end + +local function local_get(key) + return shared_dict():get(local_key(key)) +end + +local function local_set(key, value, ttl) + local ok, err = shared_dict():set(local_key(key), value, ttl) + if not ok then + core.log.warn("failed to store query cache entry locally: ", err) + end + return ok +end + +local function redis_get(conf, key) + local red, err = redis.new(conf) + if not red then + return nil, err + end + + local value + value, err = red:get(key) + local ok, keepalive_err = red:set_keepalive(conf.redis_keepalive_timeout or 10000, + conf.redis_keepalive_pool or 100) + if not ok then + core.log.warn("failed to set redis keepalive: ", keepalive_err) + end + + if value == ngx.null then + return nil + end + return value, err +end + +local function redis_set(conf, key, value, ttl) + local red, err = redis.new(conf) + if not red then + return nil, err + end + + local ok + ok, err = red:set(key, value, "EX", ttl) + local keepalive_ok, keepalive_err = red:set_keepalive(conf.redis_keepalive_timeout or 10000, + conf.redis_keepalive_pool or 100) + if not keepalive_ok then + core.log.warn("failed to set redis keepalive: ", keepalive_err) + end + return ok, err +end + +local function cluster_get(conf, key) + local red, err = rediscluster.new(conf, "query-gateway-redis-cluster-slot-lock") + if not red then + return nil, err + end + + local value + value, err = red:get(key) + if value == ngx.null then + return nil + end + return value, err +end + +local function cluster_set(conf, key, value, ttl) + local red, err = rediscluster.new(conf, "query-gateway-redis-cluster-slot-lock") + if not red then + return nil, err + end + + return red:set(key, value, "EX", ttl) +end + +local function backend_get(conf, key) + if use_fallback(conf) then + return local_get(key), nil, "local-fallback" + end + + if conf.backend == "local" then + return local_get(key), nil, "local" + end + + local value, err + if conf.backend == "redis" then + value, err = redis_get(conf, key) + else + value, err = cluster_get(conf, key) + end + + if err then + mark_backend_failure(conf, err) + return local_get(key), nil, "local-fallback" + end + + return value, nil, conf.backend +end + +local function backend_set(conf, key, value, ttl) + if use_fallback(conf) then + return local_set(key, value, conf.fallback_ttl) + end + + if conf.backend == "local" then + return local_set(key, value, ttl) + end + + local ok, err + if conf.backend == "redis" then + ok, err = redis_set(conf, key, value, ttl) + else + ok, err = cluster_set(conf, key, value, ttl) + end + + if not ok then + mark_backend_failure(conf, err) + return local_set(key, value, conf.fallback_ttl) + end + + return true +end + +local function has_directive(value, directive) + return value and ngx.re.find(lower(value), "(?:^|,)\\s*" .. directive .. "(?:\\s|,|=|$)", "jo") +end + +local function parse_cookie(header, allowed) + local result = {} + local seen = {} + for pair in header:gmatch("[^;]+") do + local name, value = pair:match("^%s*([^=]+)%s*=%s*(.*)%s*$") + if not name or not allowed[name] then + return nil + end + seen[name] = value + end + + for name, _ in pairs(allowed) do + result[#result + 1] = name .. "=" .. (seen[name] or "") + end + table_sort(result) + return concat(result, ";") +end + +local function request_is_cacheable(conf, headers) + if headers["authorization"] or headers["range"] then + return nil, "sensitive request header" + end + + local request_cache_control = headers["cache-control"] + if has_directive(request_cache_control, "no%-store") or + has_directive(request_cache_control, "no%-cache") or + headers["pragma"] and lower(headers["pragma"]):find("no%-cache", 1, false) then + return nil, "request cache directive" + end + + if not headers["content-type"] or headers["content-type"] == "" then + return nil, "missing content-type" + end + + local cookie = headers["cookie"] + local cookie_key = "" + if cookie and cookie ~= "" then + if not conf.cookie_names or #conf.cookie_names == 0 then + return nil, "cookie request" + end + + local allowed = {} + for _, name in ipairs(conf.cookie_names) do + allowed[name] = true + end + cookie_key = parse_cookie(cookie, allowed) + if not cookie_key then + return nil, "unallowlisted cookie" + end + end + + local content_length = tonumber(headers["content-length"]) + if content_length and content_length > conf.max_request_body_size then + return nil, "request body exceeds cache limit" + end + + return cookie_key +end + +function _M.fetch(conf, ctx) + local headers = core.request.headers(ctx) + ctx.query_gateway_request_uri = ctx.query_gateway_request_uri or ctx.var.request_uri + local cookie_key, reason = request_is_cacheable(conf, headers) + if not cookie_key then + return nil, reason + end + + ngx.req.read_body() + local body = ngx.req.get_body_data() + if not body then + return nil, "request body is not held in memory" + end + + if #body > conf.max_request_body_size then + return nil, "request body exceeds cache limit" + end + + local identity = ctx.consumer_name or ctx.var.remote_user or "" + local key_material = concat({ + "v1", + ctx.query_gateway_client_method or "QUERY", + ctx.route_id or ctx.conf_id or "", + ctx.var.scheme or "", + ctx.var.host or "", + ctx.query_gateway_request_uri or "", + headers["content-type"] or "", + headers["content-encoding"] or "", + headers["content-language"] or "", + headers["accept"] or "", + headers["accept-encoding"] or "", + headers["accept-language"] or "", + cookie_key, + identity, + sha256_hex(body), + }, "\0") + local key = "apisix:query-gateway:{" .. sha256_hex(key_material) .. "}" + + local value, _, backend = backend_get(conf, key) + ctx.query_gateway_cache_key = key + ctx.query_gateway_cache_backend = backend + if not value then + return nil, "miss" + end + + local entry, err = core.json.decode(value) + if not entry then + core.log.warn("invalid query cache entry: ", err) + return nil, "invalid entry" + end + + return entry, "hit" +end + +function _M.serve(entry) + ngx.status = entry.status + for name, value in pairs(entry.headers) do + ngx.header[name] = value + end + ngx.header["Apisix-Cache-Status"] = "HIT" + ngx.print(ngx.decode_base64(entry.body)) + return ngx.exit(entry.status) +end + +local function response_is_cacheable(conf, ctx, headers) + if ngx.status ~= 200 or headers["set-cookie"] or headers["www-authenticate"] or + headers["proxy-authenticate"] or headers["content-range"] then + return nil + end + + local cache_control = headers["cache-control"] + if has_directive(cache_control, "private") or has_directive(cache_control, "no%-store") or + has_directive(cache_control, "no%-cache") or has_directive(cache_control, "max%-age=0") or + has_directive(cache_control, "s%-maxage=0") then + return nil + end + + local vary = headers["vary"] + if vary and vary ~= "" then + for item in vary:gmatch("[^,]+") do + item = lower(item:gsub("^%s+", ""):gsub("%s+$", "")) + if not ALLOWED_VARY[item] then + return nil + end + end + end + + local ttl = conf.ttl + local max_age = cache_control and + ngx.re.match(cache_control, "(?:s-maxage|max-age)=(\\d+)", "ijo") + if max_age then + ttl = math_min(ttl, tonumber(max_age[1])) + end + if ttl <= 0 then + return nil + end + + return ttl +end + +function _M.header_filter(conf, ctx) + if ctx.query_gateway_cache_hit or not ctx.query_gateway_cache_key then + return + end + + local headers = ngx.resp.get_headers() + local ttl = response_is_cacheable(conf, ctx, headers) + if not ttl then + ctx.query_gateway_cache_key = nil + return + end + + local stored_headers = {} + for name, value in pairs(headers) do + if not HOP_BY_HOP[lower(name)] and lower(name) ~= "set-cookie" then + stored_headers[name] = value + end + end + + ctx.query_gateway_cache_entry = { + status = ngx.status, + headers = stored_headers, + chunks = {}, + size = 0, + ttl = ttl, + } + ngx.header["Apisix-Cache-Status"] = "MISS" +end + +function _M.body_filter(conf, ctx) + if ctx.query_gateway_cache_hit then + return + end + + local entry = ctx.query_gateway_cache_entry + if not entry then + return + end + + local chunk = ngx.arg[1] + if chunk and #chunk > 0 then + entry.size = entry.size + #chunk + if entry.size > conf.max_response_body_size then + ctx.query_gateway_cache_entry = nil + return + end + entry.chunks[#entry.chunks + 1] = chunk + end + + if not ngx.arg[2] then + return + end + + local payload = core.json.encode({ + status = entry.status, + headers = entry.headers, + body = ngx.encode_base64(concat(entry.chunks)), + }) + local key, ttl = ctx.query_gateway_cache_key, entry.ttl + local cache_conf = conf + ctx.query_gateway_cache_entry = nil + + if cache_conf.backend == "local" then + backend_set(cache_conf, key, payload, ttl) + return + end + + local ok, err = ngx.timer.at(0, function(premature) + if premature then + return + end + backend_set(cache_conf, key, payload, ttl) + end) + if not ok then + core.log.warn("failed to schedule query cache store: ", err) + end +end + +return _M diff --git a/docs/en/latest/config.json b/docs/en/latest/config.json index 5fb645afa3a0..c7f34ef3b7f1 100644 --- a/docs/en/latest/config.json +++ b/docs/en/latest/config.json @@ -110,6 +110,7 @@ "plugins/response-rewrite", "plugins/error-page", "plugins/proxy-rewrite", + "plugins/query-gateway", "plugins/grpc-transcode", "plugins/grpc-web", "plugins/fault-injection", diff --git a/docs/en/latest/plugins/query-gateway.md b/docs/en/latest/plugins/query-gateway.md new file mode 100644 index 000000000000..8934c05609ad --- /dev/null +++ b/docs/en/latest/plugins/query-gateway.md @@ -0,0 +1,114 @@ +--- +title: query-gateway +keywords: + - Apache APISIX + - API Gateway + - QUERY + - HTTP method +description: The query-gateway Plugin provides RFC 10008-aware QUERY caching and optional forwarding to POST-only Upstream services. +--- + + + +## Description + +The `query-gateway` Plugin accepts client `QUERY` and `POST` requests. It provides safe, body-aware caching for `QUERY` requests and optionally for explicitly declared read-only `POST` routes. A client `QUERY` is forwarded as `POST` by default; configure a native QUERY-capable Upstream to preserve it. + +Configure the Plugin only on Routes that explicitly match `request_method == QUERY`. Route matching and request security policies run against the client method. The method is changed immediately before APISIX proxies the request to the Upstream service. + +## Attributes + +| Name | Type | Required | Default | Description | +|------|------|----------|---------|-------------| +| `preserve_original_method_header` | boolean | False | `true` | Forward the original method to Upstream. | +| `original_method_header` | string | False | `X-Original-Method` | Header used to forward the original method. APISIX overwrites an incoming header with the same name. | +| `query.upstream_method` | string | False | `post` | `post` forwards client QUERY requests as POST; `query` preserves QUERY for a native QUERY-capable Upstream. | +| `post.cache_enabled` | boolean | False | `false` | Enables cache eligibility for client POST requests only when `post.read_only` is also `true`. POST is never rewritten. | +| `post.read_only` | boolean | False | `false` | Explicitly declares the POST route safe for caching. | +| `cache.enabled` | boolean | False | `false` | Enables body-aware caching. | +| `cache.backend` | string | False | `local` | `local`, `redis`, or `redis-cluster`. | +| `cache.ttl` | integer | False | `30` | Maximum freshness lifetime in seconds. A shorter upstream max-age is honored. | +| `cache.fallback_ttl` | integer | False | `5` | Node-local cache lifetime while a Redis backend is unavailable. | +| `cache.max_request_body_size` | integer | False | `262144` | Maximum in-memory request body size eligible for cache-key generation. Larger or file-backed bodies bypass cache. | +| `cache.max_response_body_size` | integer | False | `1048576` | Maximum response body size stored in cache. | +| `cache.cookie_names` | array[string] | False | | Explicit request-cookie allowlist. A request containing an unlisted cookie bypasses cache. | +| `cache.redis_*` | object fields | Required for `redis` | | Redis address, TLS, authentication, database, timeout, and keepalive settings. | +| `cache.redis_cluster_*` | object fields | Required for `redis-cluster` | | Redis Cluster name, seed nodes, TLS, authentication, timeout, and keepalive settings. | + +## Example + +Create a Route that accepts client `QUERY` requests and forwards them to a POST-only search service: + +```shell +curl "http://127.0.0.1:9180/apisix/admin/routes/query-search" -X PUT \ + -H "X-API-KEY: ${admin_key}" \ + -d '{ + "uri": "/v1/search", + "vars": [["request_method", "==", "QUERY"]], + "plugins": { + "query-gateway": { + "query": { + "upstream_method": "post" + } + } + }, + "upstream": { + "type": "roundrobin", + "nodes": { + "search.internal:8080": 1 + } + } + }' +``` + +A client can issue a QUERY request with a body: + +```shell +curl "http://127.0.0.1:9080/v1/search" \ + -X QUERY \ + -H "Content-Type: application/json" \ + --data '{"query":"apisix"}' +``` + +The Upstream receives: + +```text +POST /v1/search +X-Original-Method: QUERY +Content-Type: application/json + +{"query":"apisix"} +``` + +## Notes + +- Client `QUERY` requests are forwarded as POST by default. Set `query.upstream_method: query` only for a native QUERY-capable Upstream. +- Client POST requests are passed through unchanged. Enable their cache eligibility only with both `post.cache_enabled: true` and `post.read_only: true`. +- Match client methods explicitly in the Route when the route should serve only one method. +- Enable cache only for read endpoints whose responses are safe to share. +- Cache keys include the client method, route scope, target URI, Content-Type, Content-Encoding, Content-Language, request negotiation headers, consumer identity, allowlisted cookies, and the SHA-256 digest of the unmodified request body. The key does not depend on the Upstream forwarding method. + +## Cache Safety + +The cache is deliberately conservative. It bypasses cache lookup and storage for requests with `Authorization`, `Range`, `Cookie` unless every cookie is allowlisted, `Cache-Control: no-store` or `no-cache`, `Pragma: no-cache`, oversized bodies, and bodies not available in memory. + +It never stores responses with `Set-Cookie`, `WWW-Authenticate`, `Proxy-Authenticate`, `Content-Range`, `Cache-Control: private`, `no-store`, `no-cache`, `max-age=0`, `s-maxage=0`, or unsupported `Vary` values. `Vary: Accept`, `Accept-Encoding`, and `Accept-Language` are included in the key. `Vary: Cookie`, `Authorization`, and `*` bypass cache. + +When Redis or Redis Cluster cannot be reached, the Plugin opens a per-node circuit breaker and uses the local shared-memory cache for `cache.fallback_ttl` seconds. It never fails the client request because the cache backend is unavailable. diff --git a/t/admin/plugins.t b/t/admin/plugins.t index ef70e46d9175..c5ed3fe0e326 100644 --- a/t/admin/plugins.t +++ b/t/admin/plugins.t @@ -157,6 +157,7 @@ clickhouse-logger tencent-cloud-cls inspect example-plugin +query-gateway aws-lambda azure-functions openwhisk diff --git a/t/integration/query-gateway-elasticsearch/Makefile b/t/integration/query-gateway-elasticsearch/Makefile new file mode 100644 index 000000000000..6d72286a736d --- /dev/null +++ b/t/integration/query-gateway-elasticsearch/Makefile @@ -0,0 +1,32 @@ +COMPOSE = docker compose -f docker-compose.yml +ARTIFACTS = artifacts +REQUESTS = $(ARTIFACTS)/requests + +.PHONY: run logs analyze down + +run: + mkdir -p $(ARTIFACTS) + $(COMPOSE) up --build -d apisix elasticsearch + @$(COMPOSE) run --rm client > $(ARTIFACTS)/client.log || { status=$$?; $(MAKE) logs; exit $$status; } + $(MAKE) logs + $(MAKE) analyze + +logs: + $(COMPOSE) logs --no-color apisix > $(ARTIFACTS)/apisix.log + $(COMPOSE) logs --no-color elasticsearch > $(ARTIFACTS)/elasticsearch.log + +analyze: + @while read -r trace_id name method cache; do \ + grep -F "trace=$$trace_id" $(ARTIFACTS)/client.log >/dev/null; \ + grep -F "trace=$$trace_id client_method=$$method" $(ARTIFACTS)/apisix.log >/dev/null; \ + case "$$name" in \ + query-miss|post-miss|query-auth-1|query-auth-2|post-auth-1|post-auth-2) \ + grep -F "$$trace_id" $(ARTIFACTS)/elasticsearch.log >/dev/null ;; \ + query-hit|post-hit|query-no-content-type) \ + ! grep -F "$$trace_id" $(ARTIFACTS)/elasticsearch.log >/dev/null ;; \ + esac; \ + done < $(REQUESTS) + @printf 'validated %s correlated client/APISIX/Elasticsearch requests\n' "$$(wc -l < $(REQUESTS))" + +down: + $(COMPOSE) down --volumes diff --git a/t/integration/query-gateway-elasticsearch/README.md b/t/integration/query-gateway-elasticsearch/README.md new file mode 100644 index 000000000000..70cbca912b6c --- /dev/null +++ b/t/integration/query-gateway-elasticsearch/README.md @@ -0,0 +1,34 @@ +# Query Gateway Elasticsearch Integration Profile + +This profile verifies the complete client-to-APISIX-to-Elasticsearch path with a +real Elasticsearch node. APISIX is built from the checked-out source. The +profile is independent from the regular APISIX test suite and can run locally +or in its dedicated integration workflow. + +Run: + + cd t/integration/query-gateway-elasticsearch + make run + +The test sends ten requests with one JSON search body: + +| Client method | Case | Expected result | +| --- | --- | --- | +| QUERY | first cacheable request | 200, cache MISS | +| QUERY | identical request | 200, cache HIT | +| POST | first read-only cacheable request | 200, cache MISS | +| POST | identical request | 200, cache HIT | +| QUERY | two requests with Authorization | 200, cache bypass | +| POST | two requests with Authorization | 200, cache bypass | +| QUERY | no Content-Type | 400, rejected before upstream | +| POST | no Content-Type | 4xx, rejected by Elasticsearch | + +Every request has a distinct `X-Opaque-ID`. The profile persists client, +APISIX access, and Elasticsearch HTTP-trace logs under `artifacts/`, then +requires each expected trace to appear in the client and APISIX logs. It also +requires Elasticsearch traces for cache misses and credential bypasses, and +requires their absence for cache hits and the rejected QUERY request. + +Elasticsearch HTTP body tracing is enabled with +`es.insecure_network_trace_enabled`. The data is synthetic; never use this +profile with production data or credentials. diff --git a/t/integration/query-gateway-elasticsearch/client.sh b/t/integration/query-gateway-elasticsearch/client.sh new file mode 100644 index 000000000000..3eb7df53d995 --- /dev/null +++ b/t/integration/query-gateway-elasticsearch/client.sh @@ -0,0 +1,89 @@ +#!/bin/sh +set -eu + +artifacts=/artifacts +run_id="qg-es-$(date +%s)-$$" +body='{"query":{"term":{"kind":"query-gateway"}}}' + +log() { + printf '%s trace=%s %s\n' "$(date -u +%FT%TZ)" "$1" "$2" +} + +wait_for_apisix() { + attempts=0 + until curl -sS "$APISIX_URL/" >/dev/null 2>&1; do + attempts=$((attempts + 1)) + if [ "$attempts" -ge 30 ]; then + log "$run_id" "event=apisix_unavailable" + exit 1 + fi + sleep 1 + done +} + +request() { + name=$1 + method=$2 + expected_status=$3 + expected_cache=$4 + content_type=$5 + authorization=$6 + trace_id="$run_id-$name" + headers="$artifacts/$name.headers" + response="$artifacts/$name.body" + + set -- -H "X-Opaque-ID: $trace_id" + if [ "$content_type" = present ]; then + set -- "$@" -H 'Content-Type: application/json' + else + set -- "$@" -H 'Content-Type:' + fi + if [ "$authorization" = present ]; then + set -- "$@" -H 'Authorization: Bearer integration-test' + fi + + log "$trace_id" "event=client_request case=$name method=$method" + status=$(curl -sS -D "$headers" -o "$response" -w '%{http_code}' -X "$method" "$APISIX_URL/search" "$@" --data "$body") + + case "$expected_status" in + 4xx) case "$status" in 4*) ;; *) exit 1 ;; esac ;; + *) [ "$status" = "$expected_status" ] ;; + esac + + case "$expected_cache" in + MISS|HIT) grep -qi "^Apisix-Cache-Status: $expected_cache" "$headers" ;; + NONE) ! grep -qi '^Apisix-Cache-Status:' "$headers" ;; + esac + + if [ "$expected_status" = 200 ]; then + grep -q '"name":"integration"' "$response" + fi + + printf '%s %s %s %s\n' "$trace_id" "$name" "$method" "$expected_cache" >> "$artifacts/requests" + log "$trace_id" "event=client_response case=$name status=$status cache=$expected_cache" +} + +curl -fsS -X PUT "$ELASTICSEARCH_URL/query-gateway-integration" -H 'Content-Type: application/json' -d '{"mappings":{"properties":{"kind":{"type":"keyword"}}}}' >/dev/null +curl -fsS -X POST "$ELASTICSEARCH_URL/query-gateway-integration/_doc/1" -H 'Content-Type: application/json' -d '{"kind":"query-gateway","name":"integration"}' >/dev/null +curl -fsS -X POST "$ELASTICSEARCH_URL/query-gateway-integration/_refresh" >/dev/null +curl -fsS -X PUT "$ELASTICSEARCH_URL/_cluster/settings" -H 'Content-Type: application/json' -d '{"transient":{"logger.org.elasticsearch.http.HttpTracer":"TRACE","logger.org.elasticsearch.http.HttpBodyTracer":"TRACE","http.tracer.include":"*"}}' >/dev/null + +wait_for_apisix + +# Cacheable QUERY and POST requests: first response populates, second reuses. +request query-miss QUERY 200 MISS present absent +request query-hit QUERY 200 HIT present absent +request post-miss POST 200 MISS present absent +request post-hit POST 200 HIT present absent + +# Credentials bypass cache; the repeated requests must remain uncached. +request query-auth-1 QUERY 200 NONE present present +request query-auth-2 QUERY 200 NONE present present +request post-auth-1 POST 200 NONE present present +request post-auth-2 POST 200 NONE present present + +# Missing content type is rejected for cacheable QUERY and rejected upstream for POST. +request query-no-content-type QUERY 400 NONE absent absent +request post-no-content-type POST 4xx NONE absent absent + +printf '%s\n' "$run_id" > "$artifacts/run_id" diff --git a/t/integration/query-gateway-elasticsearch/conf/apisix.yaml b/t/integration/query-gateway-elasticsearch/conf/apisix.yaml new file mode 100644 index 000000000000..7e959c2953bf --- /dev/null +++ b/t/integration/query-gateway-elasticsearch/conf/apisix.yaml @@ -0,0 +1,48 @@ +routes: + - id: query-gateway-elasticsearch-query + uri: /search + vars: + - - request_method + - == + - QUERY + plugins: + proxy-rewrite: + uri: /query-gateway-integration/_search + headers: + X-Opaque-ID: $http_x_opaque_id + query-gateway: + cache: + enabled: true + backend: local + ttl: 30 + upstream: + type: roundrobin + scheme: http + nodes: + "elasticsearch:9200": 1 + + - id: query-gateway-elasticsearch-post + uri: /search + vars: + - - request_method + - == + - POST + plugins: + proxy-rewrite: + uri: /query-gateway-integration/_search + headers: + X-Opaque-ID: $http_x_opaque_id + query-gateway: + post: + cache_enabled: true + read_only: true + cache: + enabled: true + backend: local + ttl: 30 + upstream: + type: roundrobin + scheme: http + nodes: + "elasticsearch:9200": 1 +#END diff --git a/t/integration/query-gateway-elasticsearch/conf/config.yaml b/t/integration/query-gateway-elasticsearch/conf/config.yaml new file mode 100644 index 000000000000..54733772731e --- /dev/null +++ b/t/integration/query-gateway-elasticsearch/conf/config.yaml @@ -0,0 +1,21 @@ +deployment: + role: data_plane + role_data_plane: + config_provider: yaml + +apisix: + node_listen: 9080 + enable_admin: false + +plugins: + - proxy-rewrite + - query-gateway + +nginx_config: + http_configuration_snippet: | + lua_shared_dict prometheus-metrics 128m; + lua_shared_dict prometheus-cache 10m; + lua_shared_dict query-gateway-cache 32m; + http: + access_log: /dev/stdout + access_log_format: '$time_iso8601 trace=$http_x_opaque_id client_method=$request_method request="$request" status=$status upstream_status=$upstream_status cache=$sent_http_apisix_cache_status' diff --git a/t/integration/query-gateway-elasticsearch/docker-compose.yml b/t/integration/query-gateway-elasticsearch/docker-compose.yml new file mode 100644 index 000000000000..ea24e58272c5 --- /dev/null +++ b/t/integration/query-gateway-elasticsearch/docker-compose.yml @@ -0,0 +1,48 @@ +services: + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:8.18.3 + environment: + discovery.type: single-node + xpack.security.enabled: "false" + xpack.security.http.ssl.enabled: "false" + ES_JAVA_OPTS: -Xms512m -Xmx512m -Des.insecure_network_trace_enabled=true + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:9200 >/dev/null"] + interval: 5s + timeout: 3s + retries: 30 + + apisix: + build: + context: ../../.. + dockerfile: docker/debian-dev/Dockerfile + args: + CODE_PATH: . + ENTRYPOINT_PATH: ./docker/debian-dev/docker-entrypoint.sh + INSTALL_BROTLI: ./docker/debian-dev/install-brotli.sh + environment: + APISIX_STAND_ALONE: "true" + volumes: + - ./conf/config.yaml:/usr/local/apisix/conf/config.yaml:ro + - ./conf/apisix.yaml:/usr/local/apisix/conf/apisix.yaml:ro + ports: + - "19080:9080" + depends_on: + elasticsearch: + condition: service_healthy + + client: + image: curlimages/curl:8.12.1 + user: "0:0" + entrypoint: ["/bin/sh", "/tests/client.sh"] + environment: + APISIX_URL: http://apisix:9080 + ELASTICSEARCH_URL: http://elasticsearch:9200 + volumes: + - ./client.sh:/tests/client.sh:ro + - ./artifacts:/artifacts + depends_on: + elasticsearch: + condition: service_healthy + apisix: + condition: service_started diff --git a/t/plugin/query-gateway.t b/t/plugin/query-gateway.t new file mode 100644 index 000000000000..fefcc2b43be4 --- /dev/null +++ b/t/plugin/query-gateway.t @@ -0,0 +1,465 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +BEGIN { + if ($ENV{TEST_NGINX_CHECK_LEAK}) { + $SkipReason = "unavailable for the hup tests"; + } else { + $ENV{TEST_NGINX_USE_HUP} = 1; + undef $ENV{TEST_NGINX_USE_STAP}; + } +} + +use t::APISIX 'no_plan'; + +repeat_each(1); +no_long_string(); +no_shuffle(); +no_root_location(); + +add_block_preprocessor(sub { + my ($block) = @_; + + my $http_config = $block->http_config // ""; + $http_config .= <<_EOC_; +lua_shared_dict query-gateway-cache 32m; + +server { + listen 1986; + + location = /query-gateway-method { + content_by_lua_block { + ngx.say("method: ", ngx.req.get_method()) + ngx.say("x-original-method: ", + ngx.req.get_headers()["x-original-method"]) + } + } + + location = /query-cache { + content_by_lua_block { + local value = ngx.shared["query-gateway-cache"]:incr("test-counter", 1, 0) + ngx.say("method: ", ngx.req.get_method()) + ngx.say("counter: ", value) + } + } + + location = /echo { + content_by_lua_block { + ngx.req.read_body() + ngx.print(ngx.req.get_body_data() or "") + } + } +} +_EOC_ + + $block->set_value("http_config", $http_config); +}); + +run_tests; + +__DATA__ + +=== TEST 1: validate plugin schema +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.query-gateway") + local ok, err = plugin.check_schema({ + original_method_header = "X-Original-Method", + }) + if not ok then + ngx.say(err) + end + + ngx.say("done") + } + } +--- request +GET /t +--- response_body +done + + + +=== TEST 2: reject an invalid original method header +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.query-gateway") + local ok, err = plugin.check_schema({ + original_method_header = "Bad:Header", + }) + ngx.say(ok) + ngx.say(err) + } + } +--- request +GET /t +--- response_body_like eval +qr/false +invalid original_method_header/ + + + +=== TEST 3: add a QUERY route +--- upstream_server_config + location = /query-gateway-method { + content_by_lua_block { + ngx.say("method: ", ngx.req.get_method()) + ngx.say("x-original-method: ", + ngx.req.get_headers()["x-original-method"]) + } + } +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [=[{ + "vars": [["request_method", "==", "QUERY"]], + "plugins": { + "proxy-rewrite": { + "uri": "/query-gateway-method" + }, + "query-gateway": {} + }, + "upstream": { + "nodes": { + "127.0.0.1:1986": 1 + }, + "type": "roundrobin" + }, + "uri": "/query-gateway" + }]=] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed + + + +=== TEST 4: transform QUERY to POST and preserve the original method +--- request +QUERY /query-gateway +--- response_body +method: POST +x-original-method: QUERY + + + +=== TEST 5: update route to forward the request body +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [=[{ + "vars": [["request_method", "==", "QUERY"]], + "plugins": { + "proxy-rewrite": { + "uri": "/echo" + }, + "query-gateway": {} + }, + "upstream": { + "nodes": { + "127.0.0.1:1986": 1 + }, + "type": "roundrobin" + }, + "uri": "/query-gateway/body" + }]=] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed + + + +=== TEST 6: preserve QUERY request body +--- more_headers +Content-Type: application/json +--- request +QUERY /query-gateway/body +{"query":"apisix"} +--- response_body_like eval +qr/{"query":"apisix"}/ + + + +=== TEST 7: do not match POST on a QUERY-only route +--- request +POST /query-gateway/body +{"query":"apisix"} +--- error_code: 404 + + + +=== TEST 8: reject incomplete Redis cache configuration +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.query-gateway") + local ok, err = plugin.check_schema({ + cache = { + enabled = true, + backend = "redis", + }, + }) + ngx.say(ok) + ngx.say(err) + } + } +--- request +GET /t +--- response_body_like eval +qr/false +cache.redis_host is required/ + + + +=== TEST 9: add a body-aware QUERY cache route +--- upstream_server_config + location = /query-cache { + content_by_lua_block { + local value = ngx.shared["query-gateway-cache"]:incr("test-counter", 1, 0) + ngx.say("method: ", ngx.req.get_method()) + ngx.say("counter: ", value) + } + } +--- config + location /t { + content_by_lua_block { + ngx.shared["query-gateway-cache"]:delete("test-counter") + + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/2', + ngx.HTTP_PUT, + [=[{ + "vars": [["request_method", "==", "QUERY"]], + "plugins": { + "proxy-rewrite": { + "uri": "/query-cache" + }, + "query-gateway": { + "cache": { + "enabled": true, + "backend": "local", + "ttl": 30, + "max_request_body_size": 1024, + "max_response_body_size": 1024 + } + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1986": 1 + }, + "type": "roundrobin" + }, + "uri": "/query-gateway/cache" + }]=] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed + + + +=== TEST 10: store a QUERY response under its request body digest +--- more_headers +Content-Type: application/json +--- request +QUERY /query-gateway/cache +{"query":"one"} +--- response_body +method: POST +counter: 1 +--- response_headers +Apisix-Cache-Status: MISS + + + +=== TEST 11: serve the same QUERY body from the local cache +--- more_headers +Content-Type: application/json +--- request +QUERY /query-gateway/cache +{"query":"one"} +--- response_body +method: POST +counter: 1 +--- response_headers +Apisix-Cache-Status: HIT + + + +=== TEST 12: do not collide cache entries for different QUERY bodies +--- more_headers +Content-Type: application/json +--- request +QUERY /query-gateway/cache +{"query":"two"} +--- response_body +method: POST +counter: 2 +--- response_headers +Apisix-Cache-Status: MISS + + + +=== TEST 13: bypass cache for a request cookie +--- more_headers +Content-Type: application/json +Cookie: session=private +--- request +QUERY /query-gateway/cache +{"query":"one"} +--- response_body +method: POST +counter: 3 + + + +=== TEST 14: reject a cacheable QUERY without Content-Type +--- request +QUERY /query-gateway/cache +{"query":"missing-type"} +--- error_code: 400 + + + +=== TEST 15: add a route that preserves QUERY for a native QUERY upstream +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/3', + ngx.HTTP_PUT, + [=[{ + "vars": [["request_method", "==", "QUERY"]], + "plugins": { + "proxy-rewrite": { + "uri": "/query-gateway-method" + }, + "query-gateway": { + "query": { + "upstream_method": "query" + } + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1986": 1 + }, + "type": "roundrobin" + }, + "uri": "/query-gateway/native" + }]=] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed + + + +=== TEST 16: preserve QUERY when the upstream supports it +--- request +QUERY /query-gateway/native +--- response_body +method: QUERY +x-original-method: QUERY + + + +=== TEST 17: add a route that accepts an existing POST query +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/4', + ngx.HTTP_PUT, + [=[{ + "plugins": { + "proxy-rewrite": { + "uri": "/query-gateway-method" + }, + "query-gateway": {} + }, + "upstream": { + "nodes": { + "127.0.0.1:1986": 1 + }, + "type": "roundrobin" + }, + "uri": "/query-gateway/post" + }]=] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed + + + +=== TEST 18: preserve an existing POST request +--- request +POST /query-gateway/post +{"query":"apisix"} +--- response_body +method: POST +x-original-method: nil diff --git a/ui/.gitkeep b/ui/.gitkeep new file mode 100644 index 000000000000..e69de29bb2d1