diff --git a/gateway/build-manifest.yaml b/gateway/build-manifest.yaml index 257736d24f..3865a3d92d 100644 --- a/gateway/build-manifest.yaml +++ b/gateway/build-manifest.yaml @@ -61,7 +61,7 @@ policies: version: v1.0.2 gomodule: github.com/wso2/gateway-controllers/policies/mcp-acl-list@v1 - name: mcp-auth - version: v1.0.3 + version: v1.1.0 gomodule: github.com/wso2/gateway-controllers/policies/mcp-auth@v1 - name: mcp-authz version: v1.0.1 diff --git a/gateway/configs/config-template.toml b/gateway/configs/config-template.toml index c123dcd7a6..0630ad36f2 100644 --- a/gateway/configs/config-template.toml +++ b/gateway/configs/config-template.toml @@ -212,6 +212,18 @@ connect_timeout_ms = 5000 server_header_transformation = "OVERWRITE" server_header_value = "WSO2 API Platform" +# HTTP Connection Manager (downstream) timeouts +# A value of zero disables any of these timeout settings. +[router.http_listener.timeouts] +# Max duration for the entire downstream request +request_timeout = "0s" +# Max duration to receive the complete request headers +request_headers_timeout = "0s" +# Idle timeout for a single HTTP stream/request +stream_idle_timeout = "5m" +# Idle timeout for the downstream connection +idle_timeout = "1h" + [router.policy_engine] host = "policy-engine" port = 9001 diff --git a/gateway/gateway-controller/api/management-openapi.yaml b/gateway/gateway-controller/api/management-openapi.yaml index 640796a9fc..d2ac71b56d 100644 --- a/gateway/gateway-controller/api/management-openapi.yaml +++ b/gateway/gateway-controller/api/management-openapi.yaml @@ -4194,6 +4194,8 @@ components: description: List of API-level policies applied to all operations unless overridden items: $ref: "#/components/schemas/Policy" + resilience: + $ref: "#/components/schemas/Resilience" operations: type: array description: List of HTTP operations/routes @@ -4223,7 +4225,8 @@ components: example: my-upstream-1 basePath: type: string - description: Base path prefix for all endpoints in this upstream (e.g., /api/v2). All requests to this upstream will have this path prepended. + description: Base path prefix for all endpoints in this upstream (e.g., /api/v2). All requests to this upstream will have this path prepended. Must start with '/' and must not end with '/'; omit for root. + pattern: '^/[a-zA-Z0-9\-._~!$&''()*+,;=:@%/]*[^/]$' example: /api/v2 timeout: $ref: "#/components/schemas/UpstreamTimeout" @@ -4243,7 +4246,7 @@ components: example: http://prod-backend-1:5000 weight: type: integer - description: Weight for load balancing (optional, default 100) + description: Relative weight for load balancing across multiple upstream targets. Reserved for future multi-target load balancing; not applied yet (only the first target is currently used). minimum: 0 maximum: 100 example: 80 @@ -4258,6 +4261,25 @@ components: pattern: '^\d+(\.\d+)?(ms|s|m|h)$' example: 5s + Resilience: + type: object + description: > + Backend/route timeout configuration. Maps to Envoy RouteAction timeouts. + Can be set at the API level (applies to all routes) and/or the operation level + (applies to that operation's route). When set at both levels, the operation-level + value takes precedence. When unset, the gateway's global route timeout defaults apply. + properties: + timeout: + type: string + description: Maximum time for the entire route (request to upstream response). "0s" disables the timeout. + pattern: '^\d+(\.\d+)?(ms|s|m|h)$' + example: 15s + idleTimeout: + type: string + description: Per-route stream idle timeout (overrides the listener stream idle timeout for this route). "0s" disables the timeout. + pattern: '^\d+(\.\d+)?(ms|s|m|h)$' + example: 0s + Upstream: type: object oneOf: @@ -4313,6 +4335,8 @@ components: description: List of policies applied only to this operation (overrides or adds to API-level policies) items: $ref: "#/components/schemas/Policy" + resilience: + $ref: "#/components/schemas/Resilience" Policy: type: object @@ -5322,6 +5346,11 @@ components: minLength: 1 maxLength: 253 example: "mcp1.example.com" + upstreamDefinitions: + type: array + description: List of reusable upstream definitions with optional timeout configurations. Referenced by upstream.ref. + items: + $ref: "#/components/schemas/UpstreamDefinition" upstream: description: The backend MCP server url and auth configurations allOf: @@ -5350,6 +5379,15 @@ components: enum: [deployed, undeployed] default: deployed example: deployed + resilience: + $ref: '#/components/schemas/Resilience' + description: > + API-level backend/route timeout configuration. Applies to the traffic-forwarding + routes generated for this MCP proxy (GET/POST/DELETE on the MCP resource path). + Supported at the API level only. Because MCP transports are long-lived streams, + the route timeout defaults to disabled ("0s") for MCP unless a timeout is set here + (unlike REST/LLM, which fall back to the gateway's global route timeout); the idle + timeout remains the liveness guard. MCPTool: type: object @@ -5790,6 +5828,11 @@ components: type: string description: Template name to use for this LLM Provider example: openai + upstreamDefinitions: + type: array + description: List of reusable upstream definitions with optional timeout configurations. Referenced by upstream.ref. + items: + $ref: "#/components/schemas/UpstreamDefinition" upstream: allOf: - $ref: "#/components/schemas/Upstream" @@ -5818,6 +5861,13 @@ components: enum: [deployed, undeployed] default: deployed example: deployed + resilience: + $ref: '#/components/schemas/Resilience' + description: > + API-level backend/route timeout configuration. Applies to all routes generated + for this LLM Provider (the routes that forward traffic upstream). Supported at the + API level only - LLM routes are synthesized by the gateway, so there is no + operation-level override. UpstreamAuth: type: object @@ -6108,6 +6158,13 @@ components: enum: [deployed, undeployed] default: deployed example: deployed + resilience: + $ref: '#/components/schemas/Resilience' + description: > + API-level backend/route timeout configuration. Applies to all routes generated + for this LLM Proxy (the routes that forward traffic upstream). Supported at the + API level only - LLM routes are synthesized by the gateway, so there is no + operation-level override. SecretConfigurationRequest: type: object diff --git a/gateway/gateway-controller/pkg/api/management/generated.go b/gateway/gateway-controller/pkg/api/management/generated.go index 944ff642d3..e0f188b1a3 100644 --- a/gateway/gateway-controller/pkg/api/management/generated.go +++ b/gateway/gateway-controller/pkg/api/management/generated.go @@ -504,6 +504,9 @@ type APIConfigData struct { // Policies List of API-level policies applied to all operations unless overridden Policies *[]Policy `json:"policies,omitempty" yaml:"policies,omitempty"` + // Resilience Backend/route timeout configuration. Maps to Envoy RouteAction timeouts. Can be set at the API level (applies to all routes) and/or the operation level (applies to that operation's route). When set at both levels, the operation-level value takes precedence. When unset, the gateway's global route timeout defaults apply. + Resilience *Resilience `json:"resilience,omitempty" yaml:"resilience,omitempty"` + // SubscriptionPlans List of subscription plan names available for this API SubscriptionPlans *[]string `json:"subscriptionPlans,omitempty" yaml:"subscriptionPlans,omitempty"` @@ -787,10 +790,16 @@ type LLMProviderConfigData struct { // Deprecated: this property has been marked as deprecated upstream, but no `x-deprecated-reason` was set Policies *[]LLMPolicy `json:"policies,omitempty" yaml:"policies,omitempty"` + // Resilience Backend/route timeout configuration. Maps to Envoy RouteAction timeouts. Can be set at the API level (applies to all routes) and/or the operation level (applies to that operation's route). When set at both levels, the operation-level value takes precedence. When unset, the gateway's global route timeout defaults apply. + Resilience *Resilience `json:"resilience,omitempty" yaml:"resilience,omitempty"` + // Template Template name to use for this LLM Provider Template string `json:"template" yaml:"template"` Upstream LLMProviderConfigData_Upstream `json:"upstream" yaml:"upstream"` + // UpstreamDefinitions List of reusable upstream definitions with optional timeout configurations. Referenced by upstream.ref. + UpstreamDefinitions *[]UpstreamDefinition `json:"upstreamDefinitions,omitempty" yaml:"upstreamDefinitions,omitempty"` + // Version Semantic version of the LLM Provider Version string `json:"version" yaml:"version"` @@ -974,6 +983,9 @@ type LLMProxyConfigData struct { Policies *[]LLMPolicy `json:"policies,omitempty" yaml:"policies,omitempty"` Provider LLMProxyProvider `json:"provider" yaml:"provider"` + // Resilience Backend/route timeout configuration. Maps to Envoy RouteAction timeouts. Can be set at the API level (applies to all routes) and/or the operation level (applies to that operation's route). When set at both levels, the operation-level value takes precedence. When unset, the gateway's global route timeout defaults apply. + Resilience *Resilience `json:"resilience,omitempty" yaml:"resilience,omitempty"` + // Version Semantic version of the LLM proxy Version string `json:"version" yaml:"version"` @@ -1078,9 +1090,12 @@ type MCPProxyConfigData struct { DisplayName string `json:"displayName" yaml:"displayName"` // Policies List of MCP Proxy level policies applied - Policies *[]Policy `json:"policies,omitempty" yaml:"policies,omitempty"` - Prompts *[]MCPPrompt `json:"prompts,omitempty" yaml:"prompts,omitempty"` - Resources *[]MCPResource `json:"resources,omitempty" yaml:"resources,omitempty"` + Policies *[]Policy `json:"policies,omitempty" yaml:"policies,omitempty"` + Prompts *[]MCPPrompt `json:"prompts,omitempty" yaml:"prompts,omitempty"` + + // Resilience Backend/route timeout configuration. Maps to Envoy RouteAction timeouts. Can be set at the API level (applies to all routes) and/or the operation level (applies to that operation's route). When set at both levels, the operation-level value takes precedence. When unset, the gateway's global route timeout defaults apply. + Resilience *Resilience `json:"resilience,omitempty" yaml:"resilience,omitempty"` + Resources *[]MCPResource `json:"resources,omitempty" yaml:"resources,omitempty"` // SpecVersion MCP specification version SpecVersion *string `json:"specVersion,omitempty" yaml:"specVersion,omitempty"` @@ -1089,6 +1104,9 @@ type MCPProxyConfigData struct { // Upstream The backend MCP server url and auth configurations Upstream MCPProxyConfigData_Upstream `json:"upstream" yaml:"upstream"` + // UpstreamDefinitions List of reusable upstream definitions with optional timeout configurations. Referenced by upstream.ref. + UpstreamDefinitions *[]UpstreamDefinition `json:"upstreamDefinitions,omitempty" yaml:"upstreamDefinitions,omitempty"` + // Version MCP Proxy version Version string `json:"version" yaml:"version"` @@ -1228,6 +1246,9 @@ type Operation struct { // Policies List of policies applied only to this operation (overrides or adds to API-level policies) Policies *[]Policy `json:"policies,omitempty" yaml:"policies,omitempty"` + + // Resilience Backend/route timeout configuration. Maps to Envoy RouteAction timeouts. Can be set at the API level (applies to all routes) and/or the operation level (applies to that operation's route). When set at both levels, the operation-level value takes precedence. When unset, the gateway's global route timeout defaults apply. + Resilience *Resilience `json:"resilience,omitempty" yaml:"resilience,omitempty"` } // OperationMethod HTTP method @@ -1269,6 +1290,15 @@ type Policy struct { Version string `json:"version" yaml:"version"` } +// Resilience Backend/route timeout configuration. Maps to Envoy RouteAction timeouts. Can be set at the API level (applies to all routes) and/or the operation level (applies to that operation's route). When set at both levels, the operation-level value takes precedence. When unset, the gateway's global route timeout defaults apply. +type Resilience struct { + // IdleTimeout Per-route stream idle timeout (overrides the listener stream idle timeout for this route). "0s" disables the timeout. + IdleTimeout *string `json:"idleTimeout,omitempty" yaml:"idleTimeout,omitempty"` + + // Timeout Maximum time for the entire route (request to upstream response). "0s" disables the timeout. + Timeout *string `json:"timeout,omitempty" yaml:"timeout,omitempty"` +} + // ResourceStatus Server-managed lifecycle information for a resource type ResourceStatus struct { // CreatedAt Timestamp when the resource was first created (UTC) @@ -1612,7 +1642,7 @@ type UpstreamAuthAuthType string // UpstreamDefinition Reusable upstream configuration with optional timeout and load balancing settings type UpstreamDefinition struct { - // BasePath Base path prefix for all endpoints in this upstream (e.g., /api/v2). All requests to this upstream will have this path prepended. + // BasePath Base path prefix for all endpoints in this upstream (e.g., /api/v2). All requests to this upstream will have this path prepended. Must start with '/' and must not end with '/'; omit for root. BasePath *string `json:"basePath,omitempty" yaml:"basePath,omitempty"` // Name Unique identifier for this upstream definition @@ -1626,7 +1656,7 @@ type UpstreamDefinition struct { // Url Backend URL (host and port only, path comes from basePath) Url string `json:"url" yaml:"url"` - // Weight Weight for load balancing (optional, default 100) + // Weight Relative weight for load balancing across multiple upstream targets. Reserved for future multi-target load balancing; not applied yet (only the first target is currently used). Weight *int `json:"weight,omitempty" yaml:"weight,omitempty"` } `json:"upstreams" yaml:"upstreams"` } @@ -5762,285 +5792,287 @@ func HandlerWithOptions(si ServerInterface, options StdHTTPServerOptions) http.H // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+y9+3LjNro4+Co42qkaOxFl+dZJe+rUKbftdDRpd2t8Sc6e2JuGSMjCmCIYArSt5Lhq", - "H2KfcJ/kV7iRIAlSlEzJkqP5Y9IWSeAD8N3xXf5suWQckgAFjLaO/mxRd4TGUPzzuN87IcEQ351CBvkP", - "YURCFDGMxGOXBAw9Mf5PD1E3wiHDJGgdtT5AikAI2QgMSQSg74Pjfg9EJGaIgq1xTBmgDEYMPGI2Ajtt", - "EBDAIoh9HNwB6kM62u6Aa4rA3x5QRDEJACMAjQfIA2yEgP4RB+JPMdEW6tx12mAnQtDDwZ3jY8p2ks8j", - "RIn/gCgfJ/vKw26nu91ptVvoCY5DH7WOWvYxWu3WGD59QsEdG7WO9rrddmuMA/33brsVQsZQxJf//9zc", - "7PwKnT+Onf/pOu9/u7lxbm52br/5lf9++7dWu8UmIZ+IsggHd63ndstDoU8mYxSwSwYZkjs6hLHPWkfq", - "IfJa7dw2nyKKI+SB9Gu+rQwBB/xdf/R3sKVG2gYkAn+Pg+RJB/wyQgGgiPFtMZ+0xb7yM8MURGhMHpAH", - "hhEZyzOM+GENh9gFg5gBV2BIHEEOVVt8dY8mtA1g4IGQ+NjFiAIYIRBGiKJIjEUiEBKGAoahDyKUrkAc", - "RRCPW0e/mgtPgWvdmmdlvFLcVExDH04+wzEqouiP8RgGDj9pOPDlWgM4Rgo7BwhcX3xyhhFGgedPgANI", - "4E+Aj/gR0zYI4vFA/IOG0EW0DUaTcIQC2gYc0Ii6JEJqBzzCKCcB8oi87QyeXUg0A58wZRyALIbtVmJY", - "il43N85vNzcdcPutFbM4vYqTocU9EBOTIfjx6qoP0hd3JKG22i3M0Fh897cIDVtHrf9rJ2UVO4pP7HzR", - "H/LpxjjoyY92E2BgFMEJf6iRoRyS437P8dED8g3ECUMfc8IngpGkYII48BGlgDygKMKeh4K6EPf52AKi", - "PIQ0HiRg9X1YtWnmqyD0YSDwhwL4ALEvcIojORthqs42OfhfWx+JzzH2EvsPKOIInYBdOL88hHFIWYTg", - "uAhYunf6nSxptto59j2GOJi2Vdd6Or45MPAG5Kn+J8/tVoR+jzmP4qsW890mSyKDfyOXmWs6RUMc4CnI", - "GqGYiu1NVumln0mBQsQ30AcMjxHJs6jaiH1dAMt2IFo8FAC+RGMYMOwm4ooMNVvNsAEugVoZ4n64ufG+", - "vbnp8P9YifphRCiz7NFJTBkZgwccsRj6QLy14xG+8VSho57fjgpTh1OjSQYeES92Bf4reZBZFwxxR/3V", - "ccm4Vcq/Ojc3Tgn3MlBuJtDUd1a41DPn5fDVw+/cW6ZUSrGnnShTBolnuLeNcI77vZ/QpLg7p4hB7FOO", - "cTDQEtnchD/56fS81lHL1HX4ljgKHWGIxdD8H+Fvu3v7B4fvvvv+fRcOXA8NZ/2bry9CkCHvmGs0e929", - "d073wOnuXu12j/a7R93u/6SvfBDTemPMtyUjxFvnE9BPse4ntagQR4jygYPY99utQL47njgphjpyAyiJ", - "I5c/9IkLff4DgyymfD6X4QckpFSGMtQ+5Xf4OsC/xwiE8cDHLsAe12SGGEUGkQM2gkz8cY8mXJGClBIX", - "8xUKNpVByrJjKFCEPpc8QB9RwFEFefq4JSsUp8cVryF+ylNnI8daANA45zyMV3iMKIPjEDxyxVPvkwAW", - "UnCnl5ABtARXhiQaQ6EdQ4YczugrgPlg2bBe4cxiiiLwOCIpICaI2d1T2PkinVPomwZXFhuxxaHgiPuA", - "PeS1wThm/OWs5mgjg2rVsQCoQTV5MM/4Iyj5enJiW5y2AB5yUw0lL2znj+o7p7vLj6rLz6nqqPhwfGGt", - "IxbFyAog58XQv0BDGwGeqccgQkMUocBFoHea380MdK5PYo/T1pgzA+f999+9O7QdYWA9O24OUDhEJq0X", - "zg7GjDgp9giLycCINsBjdZ5tjm0egFRaryGM4BgxFGU31MbCjHN+t5855v2CBOs672+/3XKSf25/Y5ey", - "iisWNBjxu8nSxCoF7+TGpD6ibcNm04xVP8uaa/ppEQTFhwsgiN9zIBjTKbbNRewDuVesIxSyNjNx8l61", - "CA+kVJZMP4HKZGomTzGpKNnFcjl9wj/EJLhAv8eICsIzBHKp1LKJJKsI+KLV3tCHOHC4NpEc2gP0Y8ls", - "9MFIqRRwEDEJOjdBbwhStiPsFilFfJ+bwwJdcUAZgh4/DoXl3H6FIECPgASocxNcKXGnPxtBOkIeGKAh", - "iRCgjETwDnWAfs2FAX8LBwAGEyAZxU2wNcYBHsdjsP8OuCMYQZdb3colJCDjC1GwB3fJkvxJyrpvAu2I", - "6NwEGaJ6Ev9zHinZE5I29CHjMwuuoB7K/3CJadLXu5fz0Q7oDcGAsBFQH/YC4SVIhlGOEn0O6e8M3iPK", - "JbmLPM7uOkUpubvndL+fQ0omoFSuwVP2k4XJZvFTv2jRS/UQJjrqCcz17HcTMHHA0B2KhJ0Y4BKtAvBH", - "lvEUl6DIJYFH5XEq38aIxBH/rwcn/D+PCN2LF0jARjTnZJKvVLMOAVw7XbyNDzQh0wSRcRLAyPe4WplY", - "uxyPBJmKLyLoctoI4ygkFFHhwFIEegcZeoQpsVCAGQXkMQB8swUEet4Iuvc4uMvTUF1ZiimNUVShfFHp", - "wSUR4+Y6V5gVe004kKCYlCCEHw6GWJA2yMram4APRrlapUbUbAi6LgoZ8sRgAWEZTocixPcxIPqrCPEV", - "aL6YV5tThuGhB/mFbeljSO+Rd1zCq8/FU4trQLBFvvVKb0gOsHMT9BXQYDCR26YAEd8JlTrliWGEHMV8", - "bUxQqP/ffPPNN0+TP777/n19PahnNXX0OWW3FgLlejaVJn0kdm1/KRrPcw0RTUMSUJST0ank3ZjPZebz", - "GFEK75B0SApsTomUxq6LKB3Gvj8ROtsY4gAHd5JK/hUTBltH741h1QdVOlCVB0/5R0yojPOcDmCBJuwQ", - "52nkQr+VEPTv/MWEk3MTz8T69zZhl2rEhutKbcc0WZTorXrZ5UrpJ0yZie22bRb/rOUyTTe84FmfZTnt", - "FiMM+ickDmwCnz9TVzDq0kDwuIwCUdzScqq/QFqbLVHOC+g3o9a3UdXWTFWrwpUH4hZkRM6bXsVslKE6", - "ldUsif6vQ45vBtZD3/8ybB39WofQ8xbt820WDsWlb5/brRO+PUPsQoaqWY6bvlif7xijJyM3xIQ+TJjt", - "xlIyoQF/KNzsvg8MyMEQ+yjDkPb2dg/fWxn9LKyucoqaPM+2V5bQDis8n22QUB2IwSEyAdq1LReXe9MN", - "LXHr+rp3up3wL2O2DC89POyi7w+6XQftvR84B7vegQO/233nHBy8e3d4eHDQ7Xa7s9glxt4A+Q44/Qy2", - "OBhDHFEmAAF4CAZx4OW9sief//N8Ak6O21/4f79EdzDAf8ioiJP/vL60Ggkpp8j5vSRWAuHnkKJBGnn6", - "i8zEBtRx6BPIbQRuDV6eXoJYEPh0fmNX97niqBX9skMYTxxXXMc5LrSOTNjxkE3bbmSIL/53zU2X0nTX", - "2XsHuu+Out8d7b2rLUwNdqClT8IMUBSRKCtbKjgFjSV5Va5QvbRIjJpC79cCOQxmX8p6iyvpn507KHAJ", - "x63/7hx235v4sEW3O+AEBsAlAYM4AOPYZzj0M0hDsy4rh//vw9nH3mdwcnZx1fuhd3J8dSZ+vQnOe73T", - "/746OTm+/+Xu+LH34fiu98/jnz51rz9+O774if37/Lj78eTy94+XvcH+6b/OPpw8Xh+fn10/nfxx/M8P", - "d59/vgk6nc5NIEY7+3xqmWEG17/kTpnrGmNZHXCuQoZi+SJ0I0JpXiTkVp8jmjkCfzq/1bqVzlKtWKFN", - "Gzjj+F4uDwQ50LKbZuRxNRF7knzVuzWjLH5OPhQg2MR2KZf8Ed+NVMyLmBSYjzOEZAaAmLAOBfR19S/J", - "FBrRvs6eWASFbZ16VIrbjjPPsov/5+WXz30oPckRotKPFIERgh6KJLYyomWqdBgxco+URp/Znr91Yg5o", - "BwdhzK74S1Yu5yvNtwjLL8KJxggY4sAzpjJkl6Hjh3DC+RDX7AWwrXbr9xhFkz6MoIrDGMl/Z/hv+ln1", - "/idgts39sx3Cp0/nx4Knn5CARcS34P2Ti8KSiCS1+foFvny+cigltyuHBGPiobq0cEFihs70iFZS4KMV", - "Q7+sUyZ3ZL5PHn+Dvi8CSIOJ+GcuilL9OjXEhY9cspMqqq6whZqpGreA/thxCWXOAFLkORFkyMdjYZMV", - "cI7jQn07IAGDn82UaC0zAqvuxWAariPhqtwKAYPFOGQj4mWXpE/q49lVq93qf7kU/7nm/3969uns6oz/", - "eXx18mOr3frSv+p9+cxl/49nx6etdusbA4ryuEFxwyx9Op6HpTLZNwCTt/BFDgMuxdYqzjrAwZ2KuVYX", - "1jTxrUuvNKYydHPSAeKaAjOK/KEIfwGZ8Ygb63jfwhaGaueMmGx3BJk4cR/pIL7qExNjtJPtTnag7Mik", - "1zqqineHeV4xBRWzvOW5nQ2Y1+HdO4W47gbC57MB7SREAcQzRrBvlYawb//X+gSxf/p0DvTZzhzNvlYh", - "7JmVKn6VzvLL5Zc98CVEwXEveWshAed3PhlAv18a6v1RPAdbMMRSddsuxnorDfr40ycz3htSQAIE6Ahy", - "fKEuCVEbIK7NCANXxRgkH+QCyTsvjw5Phi5f3ZeS2SW4IoqdhsjlCrmgcLqjGJS5EsitZSA3cnb4v2Sg", - "tC4kG4gfRsgVF3FWIXB61r8443bTKXBATI0N1rvQAZcM+z4YkYDE/Gi2mLrDleqXKyIzGCl+uV17Ual+", - "YVkOQ+PQtxqtV+pJog7zBSRx+SbFZIgl4ZcF7DbD7+t5So0I+novHsdcdbmdJ7S8dEHzxpgXp/7ZiLiW", - "u5qEHHBuioO7DriMw5BEjHJGHngw8oAKzRYZEm1A44EKSm9zdv6Ifc9N36LKoB4SrreCix9OHCH3MQyY", - "mFbMGsU+x7xf1LeSO8vgAJlro52SPhoyZ8yh9eEA+TpR7Bsz9nvbcj3ekUigQsNNwXm4X8Ent25uvrm5", - "6fxvyi9vt/7rKMM9b//stt/tPhtvbP/XzU1n+1v1y+2fe+3n6YZ9WSB5Qg2ZSPKs7lJLCTLuhuqhetkI", - "yfVAO69RpUZ2vRkukLyClmGB4r4hTxnRA4qcMQzgHfKAj4fInbg+kuEytAP6JIx9wZxkWqDwdwjmygXp", - "l8CfSDZocaXd5gPof9b02VIRNR0zPKTzSMkeR58dYV/c48BrHbU++WNT/CIGPaVrqnt3EZcmcU+HAcv7", - "4BC5ViXUNFF/NeyLX6UhcavVaYsOzQ/EeJ+bH8br3Njz672086f4b897FtskrdTUrDRVX62N7vBToKwQ", - "o1DUUVL2njLmDB+OpbWgnAlHLc5BSaQ8pSkd8cORl6DSA3LU+oBghCJA750JiSNHv8D5fOS3jlojxkJ6", - "tLOTZQf8PE3uLLlrxmVki9bYO7jqfne0t3u0u/8/rXai9lW9g70yhJCT5dRH5eovH/H5uYLO7WGpGzTf", - "oHkGzW2hOD+XKSqJPaKVXuWATYSVtpOmYlbGZKqBhwV9RiJmKYDicQqPib+ZqbOIbbnQSzG9SpCd6/cM", - "lJ9JtAoPRV4lMI5CLdiASE00RfRfGbr0zFJff7wR+FZOeJVqZhaOqJhhwgYMzEiZmXLO564GEgd++uJv", - "TLvxU6994kF/tnMjGR40DtmUWeRL02ZIYuVKRntKHb9O8q5jG1RxPIXsiLJzzoUt4AnuXAGQPPz5vhZh", - "GlM2RrxTvS+zqglCBcijxhyiXuNeWSWPIoJVkaX19moOf1XGPs9YYAlGVlteRfdTROLQFlx/yWRSPhxj", - "f+KI13BwZ8acKMfSYALQA4omWeMa05tA7z83UNX1tnpHJZ4n4ecKCuFR9PBQRPOzm2AEA89XnkQaR0Po", - "yhywZBQyFC6udKJT6fakgJGbQDONjrCARcA3GWPGkJe3X20O34OuNUpd8E1bguSXCN/hxLWQgvQhxj5z", - "uHWtfqLCq/J3zv3+/g8gb7XTzaJJEDgj4O/yKYr+LlyrYzgBA6SdtzBQeUr51fCR85hwaLsryjGveTDY", - "wrXmGybLqOYbQ8q+cxhyVKUz6AipIM4NYWOD88CW44bzDFHq3UqYglCmA5YQonDdAFmxx0KCClM5Ad4E", - "mgJdEZSCnjBl/9CYKK5q+TCVNKR8ZgbW7Xdn8cnUVLQWZXZtlI2NsjGbsZbQ3aoaawmA5cZagvVlRptB", - "Fq9hvGXUsAWabznGvziN743KXFtSj3yiE/aFxz+9Sxqrjc4VklPmZmuq3roaUjmHkMluzId1tIh2esTZ", - "QnqmIHfhSvC5FNynSe2Chpv4jCXGZzxN3n5wRiiWuewqg6mj+Wkyy0XmJuBjE/CxqgEfyYVRLdnxNOkb", - "F0xzBVWEino2ERV/xYiK0LgJmqJXzBkzkft8c3+Sd2lI2VXqx5D0mXFi5O5hHU3CZdew4mHKJ3+9zbKZ", - "8qv4JYYC5JbywhCAEqRr0BG1Xqc2683202SVr7WfJnY3ydPE5ht5mizfIZKxxZr1hRgivxjFr+IopgCY", - "DceckrcsUsCBpkPg+2MQ2uIwS2J5qsUS9spWmoGxsFAdKFJaVTpNPNEhIbZMEhVF8ucUKMVTG5znJ/2+", - "cBJZjiK6E0kgtKKYna/qPifvCs1I3iOlsS+JTpmrb2GOWcxdTAt4K11PTzJfgnTV1+lWWXLn2AhFmRGk", - "May+SEYbEOIjKPPCMPNRxa6NsuaneH06mLakJ9uR5vXxym0ugymbqzlbLq6lCKd0RFrLOMy2V4FxonJQ", - "az2u+XdPEsQMnqhM4bCTvvKXqFeASnQy3Eni+pqNpDtyPdxA6bLetBsoXaZGp0Lgw5l5eM2n50zvwZDC", - "aO/E8HK3iqSq+i7gVIJYBpvdo3x+0tdmkbVITIjcUrWPb06p0mcWpTh0uu+c3e8zhZAtBWaIPxPcV0Qm", - "ElZ1hVhsWkqeL1yNEBhA9x4FnsAcQXkRiCNZj5IrW/n2C1VOmBT5bPtaVhT9L+1ZGbvhbq6RwRr5VhLM", - "nS4pZ/atWD/f+FZSK/3cDe0GeqpBOGM3dBKvRtFOz+gaWSs9I8kSnv/rbYZn8z8zHNdgnq2EQ/K3TB6X", - "xq0f7RggHO13u0vNzbDt0wv8MpUI24hf5i9z4jM5c1Kps6oOnRRC8UEKED/QzJzyhJfmyrEYM025ckw9", - "bTbLPjHtptiYYzxGV8oVUjLCee/8TO95TRuVq0SmEZlEUdiqDOE/qmbnj7lyIOoMtqzVA+c3bjVcNc3b", - "diuO8CwWefm68/U4I1xVmkrrvbPhwI+l3ga+/mEcuHKHMLO6PkUpJFmrxF56KS2MMpS1ftFTiFwuy9Pa", - "KE34NTg/tDb2i1kFhMn5V4MqBwGURbHL4gg17D7hsNuLmdctuJMlYPNQrJhiMLkc9w8CwtI+iPYaOH/a", - "nCWZOkvpKEKHh9EAswhGExCQwNGltvgOJ6H6omeFNBYc2YZJV2TP9uOqFhVhRPgaHaF0dHffe+8P94eO", - "t//9O+c7+O7AgfD9nrP7/bv3cO/7vfd7qNuyhUAJo+Il6/8kBhBLv0cTR5YGDiGOpFOWyAKFoidI4AGK", - "fFWM/rjfox3wE5pQIAJfAsKSSoEytiW3Gyh4wBEJhJfyqJXWIRdpklwhaCmbs5WV/NZlV1KcTMmw8ayZ", - "m3PV9f+lDTNLikNZmNnVVR+oh+2ZykWpIlG6alRGVZDfW0tuWeIfScxU1Fu2yeKfgt89g9CHLhoR35OM", - "z/BHDgi5pzt/Yu+5lQ9i63wzp5eqEEIk7GlRgk30xFBbDLZUe1AkaqlCzxPB/8U+o9svdWvlK6XpwxK7", - "WYkGZYXT0BNyY/7CCQkkuVqLfuvaf6r6mwgydPUX0AfJMIlTW85XIZvSsxPh5c3XacutffZqbfm4m781", - "WcHNBl2tOm6lRHsEPp5dtQEn1TboX1+1gSTUNhB02gaKPtuA06vwNH2jg0tnJPhNfbjm68O9GoWaMkh4", - "ATpasfgVxmwkg+0Y8m7Bf/wn4Ec038WlZT6X2NXXefDkOFGTDLRI7u3E3GBrGCHkiG4292iyI1WLRC/d", - "tmFBqRP552wAn8a3L1xCjOG/SeQIYZG0kU8iZ7XD9aHbBg+72x3wQ+z7gOYDA/Vbu51up7stewexFM+5", - "7qO73ETo30LTli3PPqrGS6qshY8iozP9CEnggNHzXuSz4eDO58+YO+KYM+QwpS3wKYO+n7qQdW8nPIZ3", - "qJjS1gTrtFFIzhdqCa4scYUaFWIlwzNVsNyd6EydRRPr9hFSXWpbRaluXV+dbFubjOZ8fPXKiJvewlkB", - "8yFladzIlkpFlC+nl5ENAluv/D6kFN8FaacpdauzhX6Poc8RM5OKvD1fK1vKrDXbSm+aIxSSiOWBau4i", - "1/DRznWMutJ+o+j1bCc2dtzvzXRTwT/YXH2kjnCxJSG2O8PtKGx3h5vv7vwtVS+znvEL9RY3XxzZgd3o", - "7Z0WNEqsMl14SNhORnGi1pHW/yresAwhza/sONd13koUTNuLt5ngy2T/KGKOjD4zFCuR/ZVcZejHxldp", - "T87QUQfppPupixlJqSsLJEZGvXrrgKYfIR3C45YfCcWvz7fPz3kfQu7qQTfHLxRLop0B/jeOYMdDDztU", - "YCTdKeAOZ1PYRTvJvcSyLqfKGPHc11M5NtLIhdSGDjd0uCJ0ONOV4XG/t7KXhRy23DWhJrPMjCntLe26", - "8Ljfq3tTaFwRqkvD0pvCXN+EKl9NqYsm07CkvsOlnm/F5lDtG3nkWX/pS30Zti26RG6EWFXI6ayx0lSM", - "mIG8Tyi7i9Dlvz4BEUPFj28gk6UpfSSRlw9p3Dt4YUClBGLpSbWnemF968IayqxNIu/zurVYs3SRbFFG", - "uLWEAjeahCwPKI3D/Yjuu9E++w/T4ig/kO6UUgXVcV0C4mn4x4VvkzjYBnhomqk4cP3YE22tN+i5KPSc", - "sfKRef6LCGy61NzIokbqc3aSczakVY4p10CRrEZp22ut4GSobzb9QhH5qqoYCrzECZJLNVOnkZk8OaGl", - "KRsFmddUZJIVmaUKLFqiItlV1YJeXy6vdvrXV2BHcgaauD464CufriNQ56v2KUeIxVGAvH8AihAopyGZ", - "EyWm3pG2HFA6PhgQD6N8//e3QGZT7OZdp3uYabAsbOIijDbjN/ftNMqdhRhL6atIOq9CJ4lszmzv9K8T", - "j6C0srRjcA6CS+adkfIuEIswerCl2H08SylOWMwJ2SldAQd3wENKg8pQ4hsknDL5tKGnhcmdFaYlTvA9", - "hsavrYa9jNvbPaD1sLPg6tzoaa+np9nlz7Jupb6o+1ccyLxz4RASZYofYDT5h2FzKvOb62nIsDk9MEIR", - "sl9jNad5TumgX6+XvJKHpnQ7tEWY6/dKQ/HUCx3wA4n4H3GE2USm9aaCVG4x3y99x81VVllom+9ykgIn", - "CvVGSpYDqMMf1K4PJgCLolpkIKJH+Sep4JbVo+tG8uX4ny0V1NIDuLQL+HPpcdn5eWE/ezJ0JAmsZPrS", - "mXbAZ8Jki/3Y90EWz0W7Vx9sBQR8FVc7XwGJboKv6T3RV5UzWBFOkb+rLkj7+aMLLuEYAUizIQNgR5+o", - "DMDNuC9sbLv6tr4R8OuV3biMB8nqpLFX2k8dhrhX4p43Yi22jECH3qlo4QyLreDd98O9wTuInN29/QPn", - "8N133zvv4cB1PDTs8p/4L7ZtEuGxUixZYUkfZ2ASqfen6KFPIgb9ncury+0OSEL+RexWGhkKqLEnttj+", - "dmuARdjbiSr/bgPlA1aRceqdDDyaKNoySCiA/oRhlwIWQfceB3fbVbOaR1Y1s7mMBmanBp3r+grHJ1e9", - "n88MCZz80Puc/PPi7OcvP52dWnVWE8a+D63rMdcLQh8G4Pq6dyrzniHjPHaMmeA1A5xEI6YGlX0xxpii", - "wqwtIwT+HqPsLso24HxmgfXBg6pSDbY0qf0DKA82pGAE6Uj4Q/NO7IEs1e3Agbu7t/80+WMq9Uras8E9", - "jahrCleLoDSpoHYotDl1Mm2tiraXOVSYwo3UWfM3syzz5Mv5+dnFSe/4k+3g0VOIo8kVzkeGC0a7u+fs", - "717t7R8dvj86fF9fTnCk/FwINv9IfK9BQspotcljy+gk/BL8KyYMXiDojjLzyHDWZBj5p6UczygijPno", - "E6esE40iyWe73W7XmrxnfnYdYGYarueYy+wfSRy12q1TOGm1W+ckkMkE6brU8yn3g3q7b2ugUSP4zwea", - "jwb4ly+jg3LgcyRQQIWMSlQPk7PkUe8bZd5J1l2iQ1WSTAWFVJJDLdyvi9010blacZs3BDJ/5tLhXpf3", - "NXKK63ogdfjLjCdQTnGJCjxdMW1YZ1ycPmgbeQ7OMRcXqINXi1IgG1cLt/T1lrwDJ4G6wvqHKDvdV44v", - "R4QzkdQnIBwHT5iy/BnR7amGYhP8ZgqveekR2aa/NsLgcrH76klShClbHG1LuU8YjO4Q48ZlhEQ3MlfY", - "lyRAyq+Wzcf3W7fP7T9zvSCGrdvn27wXYUS4tvAY4XwlORgzUqgipxJfKBiRR+HP+JFQBmTIIMBUWb4q", - "/0HVaNJ5MDoUsAO+8rG/Ag/56E50GBMFniIBhfrgLHggkzZ4HGF3pJ6oHBtzxpjqfDU9OHD9mDIUiSE7", - "4OsYBjH0vwIPUzgQrdtiRsaQYdeYj1tSMqWe8v/62MW5InXqFklX25RbI8e2EqnQlYodSNTJiWZpIIyQ", - "SOhHXgL9qUjwL6t2IeImC0E1OEIuS7Dn+uKToDWR7Ktr7gloU5VTFWEJI+I56rujw263uwNDvPOwZxoB", - "srLDDAhur2QKV7e+adVijOOwHGYsMMrAvAzhZnOsOaMisbTZfQI9MIA+DFzBABFjotdKnjIHkKK+NfIw", - "7V8iKxIkbUxQ4IUEB4xKbyymKXQq502d8XYHHPu+jjGgSd518rrIfxvBB6RSPtVkIQo85HWy4Y4J2ryw", - "VoY5v2dSglErbeLoV5zd+esrlsTrqVOaZu1o9LhSrxv1+yq87JpCJSenOQR5RPhupGrjZhGkvDiulR98", - "MBjBluCrsr5mxISQbsujdMkYUVmfU6PZ9jQe4ewKLjGVPbRbcjGWSrnid8saTQ+d7ja52+1mQPq+K44b", - "jzlH0Ict/7LY5oUKNb7Vdh7joCc3d3dKPQCVQ5ke9G0F47hKEamYh0YKFVD5hiSYr2my6O8nQcDnKQx6", - "Ih8IvUyN7yX6gyT7m9YhvWmJ/3a7Y3rTyp72Ic0XdvC+3VKtMLb/a2tM/5f+7/h/R9t/qycMfoY+9sT8", - "Z1FELCW8xV1ScSE/iCsmNoIMDCH25YWQGinrUAyR29G5I9aLTkrh3fTwTsTBA/ptc4YTVZy30AxKkJMr", - "ytFwdqt+rbcvv6DBh4jco+g4xPVvRc2vNhmAaahCZjetAQuUEffeYZFMJMnnHkHfPxnBIFAldUjwm5sQ", - "0m9YGdtma4LntnqJxlKsFR/KSjfFh9x2FbAa0N3D4T10vAjLJNic8Bdv84/Ve/IHZ/foffc9l7WZX/fk", - "r7fpTovHwrA1lhhG2E14CV/EDxERJgkjIXb1hnXUa8lyF7QnbVUXCF2Rchj4zhlJYpkTLc8Su+SvgSv5", - "GkgQBciEsQi5SGx6ehaPaECJe4+YkzxMtjJ5tszClBbUfUHin0kqxwnalzfF6keEEZf4YIw8LAVJoV4P", - "t2F8HyT4lcdgO+7UZXSyYsfHiMRhq4Bj8w9i4OJcg0zj6h8SIs+Vf5QiBkh6BZLos1pAYQPtyvOHzAiF", - "6uM5pmJrsW3aZbVLgGSnTUfRCsaAEEZZBEOVw0G3s1GXL+Vn+d4F1duik+ey21K3Tob62tiq2ynHrgjK", - "pvEOLgXv0HRisxJ/4sABwf+MNqQF5c9k2HWR90R+JkOzyjn6mlNlTpLUHkN+pDdnGm1n99KmhedUerk3", - "spqgIH9lcenzBgl2FI5bCcP8HCamMKImQNnhs4qyKUynob+cdBqu2xO/cspU3SOwCKOMqlR3HMV3c/pO", - "WZXE2tSjyLpQR/EchtykL1C3tN8H2BNOOVUCVph1HAsMYt+6R6JThuYKugrLGAcmqLuWsyjtLNJ4V9uM", - "DjJjc9vS3rbr09lWBLy85WYmfIHLzmys1sybb5di6vr1jWz1zTzdRuVCXt5n1KKVy4tU8GB0xaA7qo9F", - "Uo5Nzp9Ln1eFF6YOp0aTqJoUbdXoXugHmm1aUXI8nZsbp+RwKAy8AXmaGTT1nRUu9cx5OXz5gqB8E633", - "B3XaZKTOKsPAVHLOEFrThK+p9hSzpPjPZm3VAiLMUJaVm3dh6E8aKKtavaKMElZDq5J4WdSqUmHMiJSy", - "c2pVSpdMh89dny1Ap7owuFSZAaHxZi6rUU9QNBirPB8vNxuPkzdT+I2xapt3CfwFyy71zsxi3U0/kAZT", - "2zeOyo2j8tUdlW+5OFKGwDLT5khvaRlxBYO5qUpJXB7Eg5mKRyafbC6PMjyZb0oZQ77DbBQPHPTAl12s", - "WpewKrPe3OX1B90F5aiFKY1Rrppc5oUw9v3fkrteviiDe2SmL+ceHzH7MR6AM/Faa3mXE5bdednlRBY/", - "mxK6b/+A3zhTVweY5+jJuS6TnY8IuT/u9xbBzOtcyE25fWvr3i2yurLAK7GhHdu9nLJsfvOQzzWISY3l", - "X8YDgYWmm9YYSSkj3vwjmXHA848SBy8e57n0oEpveY51krN256bRa7kwrxBFjn5JFc5P+ptsjqq5o8q+", - "WE5OsmK9Or0qqpmjoU0S1JgddxHeFJM5vfBqxs6QmrlYMWiocKXSNygj61K/RxPJ1czLkg44g+4IqGsU", - "mHkmXdAitJpW9RmCyb2NtUnJ8i5aHhFkI2HJba5Ypl6xtMVX92ii7ho2ty3lty35GsoLuWLZXJFsrkia", - "vCK5LZdxqtAQN2Wz9UdzVWhnIEvZqD2R1aqoyZZoNM8I8FCERRIEAng8jlnaIpP6sawbYeS/S3NNATuV", - "2F5WkLVkS8oSOY2Q6IwxxcfQq75DAYqEU0YVyRnGvm9tNKdqR1tb56ajhD7EMow6U3Np63FEkfubTl75", - "Frw7ACP0xIVxRLc7N8GFTitET9BlIrPQReD//3//P1lqAmDGD0OEJiJ/AiDlv4iMlYAw2bVJpBqKw4J3", - "EAf5RkoSgl24N9h3D7xD9G74Hfx+8N7tertob7gPDwaH7jvvO/T98D3sDnbdPW8fHQwP4bvBd+733nvU", - "HfJvq2sd1Cg51G49mqdY0wyVL/eCISkgjZo9jYBPjmoqConxLIGL0ppVhVPAj+fHJ+o0ZUus9IxLCmp1", - "Zitc9M7p7l7tdo+6sxUumpnkp1Bu3VSmi08OhUPJDcBWwiLaQDAQqDOndNs0WcMFaRLZ8ZCPGMqxEeUc", - "eiwHppiiCl2GH5Bwnz6Q+7wOkzydrQrTXGfxPA3RqutTGNXKallKFpp4YR2wdosRBv2TelXY5NbqGmqp", - "IMkpOXvWFKWioNTl10QXxSQPELs6r1IsW1QT4L+mwI8YC6U7FSs6Fv15ZbqQ8kj+cvllT6ihOjMbXCE4", - "LppiF2eXV+I9vhjhwlc9gLOKONUl34rjqj55kuuq/tMtS/O8c3E/IMwKuV+pg1N1JxHtXQKuuh619jvd", - "zn7LaFW643K8Ee5MuVV3Npl0kdSi831VaABcfboE5sfAjaMIBVzW+AR6aTs+4yUpfDo3wdUIUZT9nNse", - "guKHsv+f6iL949VV/zKTIaquKFU956R1Tc9TnoQTc0VpYxaxur1uN+mZI1HTqMGw828qdXCadBSvIhtj", - "ngw9ChSyOzgym/3cbh02CI7IE6sCohdwxRT6ukWAyNySFBOPxzCaaECNQ3aze8ngnbimNpZuICBnmE+O", - "oCoYs5ETEV9c/7agNxaFNlSzGxSJ2+uQUGZL5hd5jRAE6DGPY2Crf3YOJAfd1jnxmlCEKDVfxlQjojcJ", - "4Bi70PcnwqFAYlEOlMGI6eR3PUoBoyQ8xoJbbd076APxJjWOz7iIMcBrHbUc/r8PZx97n8HJ2cVV74fe", - "yfHVmfj1Jjjv9U7/++rk5Pj+l7vjx96H47veP49/+tS9/vjt+OIn9u/z4+7Hk8vfP172Bvun/zr7cPJ4", - "fXx+dv108sfxPz/cff75Juh0OjeBGO3s86llhvSeZTxx5Hk7rvTnz4r/cpOSy9GsSiXuIAt0uLsIOqxC", - "fxNn41BhRkZJf263DpZLkCI3M4O0Sj1YRd6QoUw3QxAN8oXndlYm7USITysdyDaGcS5KSPgTwCJ8d4dk", - "+1QBKVctOCszpYzwj4kqothHdEJl2dUcKykwgQuUYwIvFiw1LEtzOrUk1Vn38vQy6bQ51cycQ2/7MGE2", - "N73U2wb8od5bBVROTKQq297u4fv3tfS2Kno1lp8n2JWjkgQdFRI2KUEt1CF634mT4laQrWEQ/x3ALJPR", - "RJCVnSMY3Amxqf38L5GbcuKs3ExbX4uogNzmnmrnpgmq8B+JpWWS7Q+76PuDbtdBe+8HzsGud+DA73bf", - "OQcH794dHh4cdGWRBRyINk+iMZiOOfBaedlkyru8IXbbKJnLglYzL6MqN9/KLtSWLZhZzEjECVBFmXuw", - "PBI2AQoIA0MSB95KMhIb5TbDQHx/7IQRecAeihyGxqFfafwJm+DTp3OgvwHJNyBCd5gyFKXWnmII7eTC", - "3p9wWSvfGUzkzaPVbvv06byvZrhKgJrCNH4QI4tu2OoToNxYxTDtLyEKjnuaLfweo2iS8oWsR31ZDMEt", - "lEndt5afn1GGm0daywNk2fo6F+flhq4dXVbb5C2BOSU5/oLeJqD3aRbiK7N5jz2tVlthKFi6xwa2qzt6", - "WU1QF+MWjF/WxEdP/EdxwTbOOaLNyYokKUsp2zBjVgO43mFaZkoNykytzJ0JHPsNDbxUS9VKZhYisiKB", - "cvmviMmarbCY1hhSVYe2JWTvlyjYSTD0scuAk5KmCB6hcKyuG6EfIehNZOHM1WRGkuiqmEGT/KhcGaht", - "VwQlLKtgYpQYCHb+UinzVe27MB742DVL4CnzwWSbFttBOMPxGlgHCaD19H/7OViV7mVo/jOAs2wbwA7a", - "elgDweK5QttuBnxErJzcBxOAGQW90yKdf0Q2zf7DRLSmmI/QdRRS2VasJLHPrhg0rPTMQqUMYp9uCLMG", - "YXKyKKcJr2HzIbbemIlmtzBIK4LbAcpa6LarLg8uXCJLV9RCifQvZJt0V8M2sfoXV9w22fC1Kbd99bjK", - "Iu2RGXyS87oi2zqyug1U8GwbSF24DUgERJD0VHflDG7KzB5OcVUmm/lCn2W7JjhGLmM+sNw2ffr6y6dW", - "e7+jmL+R2JEVDjkQ0pIwc4GQS8mIM3eXZjaDffbkm3TyqUkRc58NR8RCaLzcHBV5bj0j9dnrObT3bA7t", - "DIHP6qHO1LNZQMvSel7tNXJml/qwGw7dKnNjF7zXKQNU3mvRCoUAjiERdFXWlzI2qewY2zba1CfRgEmz", - "hjbgYHO901UZTlCct2AsEfGzeU41nN2Ld3LnGvg3rE2WjF6tmuAA/N/H55+44Pvn5ZfPOhjplVzkOTqf", - "Art2j8vERclwN77yqb7yhBfkfeWBl+ScrbPf/MWsz6KVzuscn8MnXtPyLprcuT0w0nYo2XOk3uCEOf1y", - "hZ3hJWDP4RpfDY/46jnC19H/3QB1z+Dtru3knsG5/RYod055vghNpwbdrYBre8082sKRbXb+bNaWmMen", - "PbMre93I8S9gelwrp3Fuh1/F5T0bE1ldd/eGr83t0V6YpbCjOm9O8WbrIjj8TVuE3lSel3NKH/d7P/FJ", - "6zE+2XXWxvQyfYc1cOuvmMjtqZu4qQ9mQ1/VegNHHj+3ZzZkbkCLUHWIqxySH1WNAOUXUADNRVwFB6HE", - "n0aoS5cyEA8VgGutasi9ydeYeZmCUTbmUgN480CUE4zGtXWM2l0J9vY6DtEtL5aTSEKUHajEI3nvwDWI", - "7dX3gFZwumY57xSNZ+dPGOKfkLiirvSYXoiSKxxWBXoHfAlcBFQpljbADLgwAAEBPgnuuFGqqkUwYl79", - "oKT3ry2Jl4/VPAtfDqsuXBTzPTXKwYnzFqoaX2UGpiS8W7eAt8KTntSK6Wj83NzaDFdhzIbh1mC4JEow", - "Z7VVywJ7WIpOWe2Y0pDIu2pdMEVV8AooQ6oCQcyIozQ8LkNIgGq4q94ka7KEfi6eNS1Ku5VH1qRumx9x", - "qeGfs2u2K+UEU+e8Pix2o97O67xbSd12Jy1IWF6p5iJ5J+OEfIlbIh3y7eu1yQa/DQGSHF3DLhL7uCsu", - "TCLCNm6St6e1J/zuNZj2E65Z1IS/+ErpA08YzZ488DQB2dD/10sceJq8TtbA02QlUwZWImGAn8lbyxbQ", - "tDxDrsDT5NUTBZ7wmtS8UWwox4efJgvPEHia2NMDOIurnxuQBnznWXeaM5DND5ghHeBpstBcgByaNhmN", - "Uzp0mX7xNFmdFIAC+VZBvQn+nzf4/2nyBiP/Bck2xsxyKuXs0f9PkxlD/58mLw1XFCPkM+wd/WA9Kt8k", - "4M4U5C8kx+tG+JeB8EpW49Nk3WL7m6XfWhH+T5Na4f1PkyZi+1edOueRzo2rK9MI7FXj+Feepowgfona", - "cR4nG9b3Z4vil5pm7RD+NRGIb9pGyIXrJ2bRMmP1Z2IRmyj9teNaVQxj0Sr9y8P0azA1w/M7aSBA/2ky", - "PTp/rbSL9YrKXwstoEZI/suJq6lg/BoklPXNvfyuW9LQ1Bj8ddEYNrH3m9j7FzGxTWRS44H3jfLXSt1l", - "ZQPum+HUi+XILwuxf5ps4us3TDVlqm8muL5p7fB1wurfEgOyB9IvkgFtoug3UfSrxkg3imqzIfSvpKU2", - "Hzpfw4mQj5t/W+ppWaT8OkqITZj8Jkz+TSvfU2LkG+fKYzesFx1/ftLvNx4cTyIVN22/G0nnrB8Vf37S", - "z0bFF+vpn8u3+iYvbj4mPgVkuTHx6bzlMfHoAUUTNuJjvc24+EVHph/aItPHbtifMThdYfgrBqcbNLbS", - "sekZXqA5YELGiwtN1yeUj0wvuYnSry8oStyKL80oQlOGXurtTglZFFEoOZ1NP9S6Yd4pzbyhUG+D7Brj", - "DTn1aIZI7wQr6wZ6G+C/qLVauuak22nnJqt4pKLf4Ysz9ZAVjgG3Q10vFDw5jVeLBK+GYNl2UQLNesSB", - "L4S2q6PAkx2qDgLXr72oe2mecteFXucR342rJ1OI7XWCwteEvjiuZxDda1ixrhkDnsBQLwR8IaJSOuqX", - "Snp/Mdug+4q2waYf6VvgVxWso2mtP0KUOTDEU1yiF4iy435viQ5RPWN9d+hxv1fuCL1AUGTDi9Uc93uL", - "c4ZyMJbrBuUzljtAI7lyx8eixMXb7CbarEmm6aGWX1Mhqs2TWdOZujCHZ0JDK+3uNChdszb+k0Drhfk6", - "1aQ1XZ36jBejzajRm9FfCoMt1ZuZEEMRJ/SOb9yXdd2XfLfekOMyJaKmyDyjwNR2Wia0X9dlmQL+IjNM", - "sRu7r9KU0iJWZU28lWVw1/NX6pN4NXdlJQDLtk40MGvirGyenqtclQnVVjsq1Vsv8lMOSaQJdn3ItJ5U", - "bkCzqCaj1/FDrgflcDw2sdhrVuOt6YTUENTzQTYr++zOxwUT1RtU2LvLVNg3PsU3wHvKGcFC9fG5a0vU", - "ZlP8+9kKSkxjUklVCZURLyB6E3rAmhSZWB9pXlVi4uWk9cLaEmUkBK5UpQdMAQT7e85gwhCIYOAl+YYo", - "cIknXfwj9AQ95OIx9NsgjNAQPyFPuiW+whCHv33tgGuKEgL6CU1kfdkJIIFJVopVI4ADl4w5A9IJ1HI0", - "NsJU5GOX+OBmylOZRuO2qhfrrpVsCmBsCmC8JQZbVV+iUeZaobasYFmJRvmgBO9VuOBsRSemgbWpPrHh", - "aCvP0QpMolEFcdnlJRpjRCvHcqTH41VYzqbexKbexHJZJ9+gtckaLuVnXEdM8/89ydiWryI2VtOh0ngP", - "I/SASUy1Fa+VAxhw1Ap96GoTXW5MAzZ+RSGJt2OYz15o4k3JiE3FiU3FibemcJcVmWjcgUCRGyFWfs9x", - "oW8VYOIxhr4PKCMRxzL5dQdcIBZHAVU/GHxSeklJzG4Czo2gy2KxdvGa4OjS80yRG0eYTUAYRyGhiMrb", - "1uKlyaUCeIFUJ6eoe9+g9iC5f7HR3u7y8Os64OdOIvwH8oCTb6OWsK6VDq2lyRlrTFenXh/Ry+8eLjnq", - "UqViKEREgRtNQtGRjAGuMEmFRT3tnYJxTJlwfQl1oHMT8MfKCqXG5zHlKhETyg7my9LP+OYnHWEHaEgi", - "BEIUUUwZClxkw3bpSJQrX1AIrxx8AelIlQM35IVX+ous/yE95wLABJ8uEzqUnnWZqyBVbBku/7PKYDhq", - "3SlFlWs/oQ/ZkETjziMlex2XjHcedlvt1j0O+LEkBzJGDHqQib3QeRiQwQGkyAkhpY8kEnRGQ+QW0bBP", - "KLuL0OW/PoExxAHQn4Lk03YmreOodarf6JuDJ6GFaguOWeuotdfde+d0d53u4dVu92i/e9Tt/g9X6Dwr", - "jO2WsjLLv30Wp/aCs5enK1FaWkM2LiE/XY17kA8wNXgdMMZUkDaJAFbazRAj36MrzOBfKwBcsc30erR3", - "upJR38AxubNUSasuc6im/BdIJUPnmhr53UfRGPKF+rouARdbaneTKHBNz1xkYSpvx0cw8tQn4hhugoCb", - "fy55QNEEjJE7ggGmYynlEqnDv8UeGoeEnwhw5AiiGSsISOCIs0MBuwkUDJHS+g66BzYBJkNuDQFW1Nes", - "5G+LagZbAQEKV7ZXmuYOZhRdAWGONEWywkvtBUFUWCti803xlUSmt9RpZK2t1MJJhQSf6zdl9tTn51N3", - "57J6/lWh9UTCckqPI1QWIN4EmberrSmqOt8K5pMSdUbrTLRL9ZqpXd4ENrXSHXFFQimXAyRjVTiFIq8D", - "etJw0y9TsQuAkZtAjS+YiZy7DSA47HbVzglPnRxGe+eEeYpdoHDQRvwfEauk/BkoRKdKlCl3yvKC/tvS", - "7pLFtGgc7kd034322X+sn9Knkd6r4B2p8WwQxvqY0kv1Ya0Lu0XVqpXhWWqG49bx4xf8U6kfXNWR5P98", - "yrIaTqE0FLcTvVODLMOIeB1v0OEU3snwBCwd6xl+JX7LDmBhKM8NRepVXKvTzPWNqaxLNVdAJ0VR8mfG", - "y3ETpG4ON44irixWuDvaAAVw4Kum/mQMGZcc+E5i7k3ACJ8HRTIM1YujtDA77YAvvme42AQz5ZYEHPgI", - "PGCofC2mBLRJI7nyv6YvZVZxq+RCqbhNullsPCn1heru0cHhK3hSViJ8YKonRSLSRryvk3if5jnRIQ/N", - "eU3iQQIXZyxBjeQc8xsgvgHwAWJfSI86KTqXxgB9Meci751yk9W+gSqscnWvdyywLr5+SuLFK8wO2Agy", - "4KEhDhAF4sbVx2PMpIEOBdMETNxjDlW0kTkGLcv6yB/lonSO3DS67Mur5DvkgalkcoWD0Dc4ryicXs1n", - "vtp5DAWiaTj1ssjYd/7k/+nVrItSJOq6FVIsVJozIi22mATthVH5Bxbnd2EZyg++dA3k83oU8lgkXlaU", - "9BB3LrJghIiGseBfda2P18O67orw+teqt/F55TNzS7BJeI2WX3OjCEu96htLxfDFa1WFlIHnlaUs7bvZ", - "UJbdFl2iKjPFPM28Wrco7XG/1wbGZk4tR3uZAWimmrS9U7BllEjtnfK5ZCPF7ZKSqDDEgoIrQ9XtHyZL", - "mm+AimKsxydXvZ/PWu1W73Pyz4uzn7/8dHa6iJKsdWl7HuN+Tez6ZZj0aisHQmAZGyDykmtXYSka60sw", - "1FfGSK8tWv7KtjlwslJjncqX0ixiL0zS7fxp/jmX3T6PyV5LrcxCtmCz/bUs9gwQwfqZ76tgudc32peP", - "d93X5f+vZa+vEVpbjPcVsdtnN9mXgt+L1bFezWSvjc6vZamvEU1ZzfaG9ZhHNBhE5B5FNbrJ/IIGH8S7", - "zbSUmWK6p7NxwGqb7sln1Y1lLhlx78FVJNvLZD5aXI+ZLGzL7Tbzxhpez9TrxUSleg1f9pfa8CVDWKud", - "mpoFNeVFWdReWP8Xc/p8E5jsQxUTScEAezhCruRIgLIIQVHGcoDYI0IB/+qSuPeIAdfHfOdE6MNPcHgP", - "gWSNqtJliCLHJUEgxwKYEl+cR5lbJYN1ixH55hTNBFvaR1yqiyZLrUV8zRzzpi1NXS9OlkLfUIMaEx+a", - "5khFFal+v5oMntb175jI30TX3uwulPaxoVwdcphUh5zMitegm0019PV62mRO69Ua20yHYtn2UgaiNXGt", - "LZIjVDa7yWxWtUOtMULXbW8yq1s3+p5BH2hKu6lBfq/j9VsjiuNYX8B5b0lCeP4mFVmo6sTBm4tsomdF", - "QSxbu1e8GRJekw4W2VNZ5z4WBY/WiwnypV0tKmluTXtb5LhC40zBVlVz+Txh0/Ji0/LitepVVnDk1/Gj", - "bHmxnEQSJInEpvFHadb29pr15ViosJimvK1gq45FsvWlsu/ZenVkQNNAbZp0bHjwGmnFBSaxHHV42T08", - "/nIcKqm5sUQOtenpsenp8Zqctqy7x0bxfWnzkRXTepvrPjLdybJaPUj+gpp2ctJvQpZteo9seo80K93W", - "phnJMuQHjQf14nIv40EzQbli359YnehcNedMobmX8aA6LvcXBNkIRca7Cw3H1fAsNxbXmFjt945i9ens", - "O49yJxz0wHG6BBD1+WKCgo143rcaFixxuF5M8OGyY4I1ga16QHDKCAwWqBF8kaHAcuJ8HHD5ZZ067YXF", - "4srxGwvEzQ+37ChcTRxWUa72fhN/O0P8raaJtxV8m1BVc9SfU39mirlViDlDwG2ygJcan3rVpWG2Wqan", - "a1uD6For0LWDatVxvGZEbRUIr2AEKXDWJ5Z2AQQ+LYpW7dHUEFr5XlPxs2pRa0K1daV3I1rINNJ6tVDZ", - "taAmFSdrYLXXtLZcMzU+haJeXvyCxKO8L1gmob1Rhb+7bIV/pS7vVtLJuRYcqYo1LFyVf1HkvoanZti+", - "XFJDMfsGB5sesL+GasP6xOnrk1j3IP3Uyf0ykmsgPL+EsNY3Nj8h/WYpf2pU/nqpMZtg/E0w/svY7iYg", - "qclI/AVIhEodbDUD8BfAu5fBo18Qca8h2kTcbxhtSvBrFCZTGnm/GB33FWLu3zhTsgTZL5wpbYLsN0H2", - "K8dcNwptYxH2r6rNNhpYX+UeWbmo+jevPlvD6NdWWm3C6Ddh9G/bOCiPoV+WhFBN9EuvnC4Qi6OAAt2M", - "XULo+wC6DD8g8OP58YnuxN8BfR9iEXotmTcFMEIgQHwDcOD6sYe8KVdSqg30ejHoBcesjAi5l/tSuz2U", - "aqa9FRAQ6jPZ3txIVcZtm6jcPCHWCePWd1GU2+bRJGTkLoLhCLsiP4UiN440xY1gxIWB7Aq/NSTRGLIj", - "8PVxRJH721fwLXh3wPUl4I5gRLdvAh3PdYcfVHVnFdUlNbUs3UpljRM+8sA335DARd98o1U8jelcjbsJ", - "JHFTRlRyTTJSeqGlgIRU/PVV/vkVJPSd+B/485vgqyLO0Ri6Dt/Lr/oqzHr/9SjpA4iEEEDxXQBZHCEq", - "o2gq78AudQ/6dWM1C4naSblMs7deU4Zedvi+DZaKvhgSd7WczqpfknYShB5i5HNzxPcFt4NhiCCXewAG", - "EzCMOVaqK+8I3CGWEFLntR0Qf80rs2PNmNTFOTJzDzL5BioYFlOgsqJWMKIiE8RgSLLFCrIqlXLnT/mP", - "qTdofRSNId8UfwIiNCYPiBpyowPOtPNBs3oP+fgBRVi8B5m6TePnk5yo7wM8HiMPQ4b8ibBKUtkAUntl", - "av7DOsqISn+B2qIkGSMD0B1mo3jgqI22A5Oe6gLa5kngVignQVuUatvWKj1BwG7wgtch/ZruRuEONAlf", - "qYWpf9CfGHKYBEiKX+InajBlJKQ3gabu4M5QB012ID/kjDLVe02Nl+u7Fu02DlAgtHLk2ZRLi6fxDTOP", - "El/jUhlI99X1QptTLsWuRDPUmJzVDOEdxEFnw9dmcpRtyS3fXhKT42BwsxuziaBc8d1xzEato19vOUpK", - "qG1k/Ym40AdqNDFzuxVHfuuoNWIsPNrZ8fkLI0LZ0fvu++4ODPHOOAFz52G3VaTFU+Leo2jnp3iAogAx", - "RI1aBfnh7+RVjMOPMSK+j6LSeW6TPctPeHJxfQoSPic1Yd1gj6Ykbeu5V4TeNtj5Sb8fkSeMjNHOT/qA", - "/zipHk4+1EEMV58ugYsizjpd4ULho/94ddW/BHEo+1sBrjQOFR6n052kX80O/6dP5xzWB+yhCFyhcejz", - "YTL+cWNl9rdfNmmtuead4mkybfxpp2QbPHXyqrHUD5mRbp//TwAAAP//3gTWnNBdAgA=", + "H4sIAAAAAAAC/+y9+3bjNvIw+CoYbfbE7oiyfOvEzpkzq7adjiZ2t8aX5LcTedMQCVkYUyRDgLaVjPfs", + "Q+wTfk/yHdxIkAQpSqZkydH8MWmLJFAA6l6Fqj8btj8OfA95lDSO/2wQe4TGkP+z0+ue+N4Q351CCtkP", + "QegHKKQY8ce271H0RNk/HUTsEAcU+17juPEBEgQCSEdg6IcAui7o9Log9COKCNgaR4QCQmFIwSOmI7DT", + "BJ4PaAixi707QFxIRtstcEMQ+OoBhQT7HqA+QOMBcgAdIaB+xB7/k0+0hVp3rSbYCRF0sHdnuZjQnfjz", + "EBHffUCEjZN+5WG31d5uNZoN9ATHgYsaxw3zGI1mYwyfzpF3R0eN4712u9kYY0/9vdtsBJBSFLLl/z/9", + "/s6v0PqjY/27bR391u9b/f7O7btf2e+3XzWaDToJ2ESEhti7azw3Gw4KXH8yRh69opAisaNDGLm0cSwf", + "IqfRzGzzKSI4RA5IvmbbShGwwNfqo6/BlhxpG/gh+Dry4ict8MsIeYAgyrZFf9Lk+8rODBMQorH/gBww", + "DP2xOMOQHdZwiG0wiCiwOYZEIWRQNflX92hCmgB6Dgh8F9sYEQBDBIIQERTysfwQBD5FHsXQBSFKVsCP", + "wovGjeNf9YUnwDVu9bPSXslvKiaBCyef4BjlUfTHaAw9i500HLhirR4cI4mdAwRuLs+tYYiR57gTYAHf", + "cyfAReyISRN40XjA/0ECaCPSBKNJMEIeaQIGaEhsP0RyBxyfEkYC/iNytlN4dinQDJxjQhkAaQzbLcWw", + "BL36feu3fr8Fbr8xYhajV34yJL8HfGJ/CH68vu6B5MUdQaiNZgNTNObffRWiYeO48X/sJKxiR/KJnc/q", + "QzbdGHtd8dFuDAwMQzhhDxUyFEPS6XUtFz0gV0OcIHAxI3yfM5IETBB5LiIE+A8oDLHjIK8qxD02Noco", + "C2GICHYx8mw0bYzL5M3nZoNEg3g5PReWbbb+Kghc6HG8IwA+QOxyXGTEQUeYSJyIEebXxkffZZh+hd0H", + "FDJCiJebO/fsyqKA0BDBcR6wZM/VO2mSbjQzbH8MsTdte27UdGxzoOcM/Kfqn/CD+D1ivI2tms93Gy/J", + "H/wH2VRf0ykaYg9PQfIQRYRvb7xKJ/lMCCKffwNdQPEY+VnWVpkgbnJgmQ5EiZUcwFdoDD2K7VjM+UPF", + "jlPsg0muRoopPPT7zjf9fov9x8gMHkY+oYY9OokI9cfgAYc0gi7gb+04Ptt4ItFRzW9GhanDydEE4w99", + "J7I5/ks5kloXDHBL/tWy/XGjkO+1+n2rgOtpKDcTaPI7I1zymfVy+Krhd+YtXZol2NOMlTCNxFNc30Q4", + "nV73JzTJ784pohC7hGEc9JQk1zfhT3Y6Xadx3NB1JLYllkRHGGA+NPtH8Nvu3v7B4ftvvztqw4HtoOGs", + "f7P1hQhS5HSYJrTX3ntvtQ+s9u71bvt4v33cbv87eeUDn9YZY7YtKeHfuJiAXoJ1P8lFBThEhA3sRa7b", + "bHji3fHESjDUEhtA/ChkMqHh+jZ02Q8U0oiw+WyKH7gMSFOG3KfsDt94+PcIgSAauNgG2GEa0BCjUCNy", + "QEeQ8j/u0YQpYJAQ38ZshZxNpZCy6BhyFKHOJQvQR+QxVEGOOm7BCvnpMYVtiJ+y1FnLseYA1M45C+M1", + "HiNC4TgAj0xhVfvEgYUE3KklpAAtwJWhH44h16ohRRZj9CXAfDBsWDd3ZhFBIXgc+QkgOojp3ZPY+SJd", + "leupGlfmG7HFoGCI+4Ad5DTBOKLs5bTGaSKDcpUzB6hGNVkwz9gjKPh6fGJbjLYAHjITD8UvbGeP6lur", + "vcuOqs3Oqeyo2HBsYY1jGkbICCDjxdC9REMTAZ7JxyBEQxQy/Q10T7O7mYLOdv3IYbQ1ZszAOvru2/eH", + "piP0jGfHzAgCh0in9dzZwYj6VoI93NLSMKIJ8FieZ5NhmwMgEVZvAEM4RhSF6Q01sTDtnN/vp455PyfB", + "2tbR7TdbVvzP7XdmKSu5Yk6D4b/rLI2vkvNOZoSqI9rWbD3FWNWztJmnnuZBkHw4BwL/PQOCNp1k20zE", + "Pvj3knUEXNamJo7fKxfhnpDKgunHUOlMTecpOhXFu1gsp0/Yh9j3LtHvESKc8DSBXCi1TCLJKAI+K7U3", + "cCH2LKZNxIf2AN1IMBt1MEIqeQxE7HutvtcdgoTtcLtFSBHXZWY0R1fsEYqgw45DYjmzeyHw0CPwPdTq", + "e9dS3KnPRpCMkAMGaOiHCBDqh/AOtYB6zYYeewt7AHoTIBhF39saYw+PozHYfw/sEQyhzax16UrikLGF", + "SNi9u3hJ7iRh3X1POTBafS9FVE/8f9Yj8fe4pA1cSNnMnCvIh+I/TGLq9PX+5Xy0BbpDMPDpCMgPux73", + "LsTDSAeLOofkdwrvEWGS3EYOY3etvJTc3bPa380hJWNQStfgSPvJwGTT+KleNOilaggdHdUE+nr22zGY", + "2KPoDoXcTvRwgVYB2CPDeJJLEGT7nkPEcUqfyMiPQvZfB07Yfx4Ruucv+B4dkYxzSrxSzjo4cM1k8SY+", + "UIdM40TGSAAj12FqZWztMjziZMq/CKHNaCOIwsAniHDHlyTQO0jRI0yIhQBMCfAfPcA2m0Og5g2hfY+9", + "uywNVZWlmJAIhSXKFxGeXz+kzFxnCrNkrzEH4hSTEAT338EAc9IGaVnb99hghKlVckTFhqBto4Aihw/m", + "+TTF6VCI2D56vvoqRGwFii9m1eaEYTjoQXxhWvoYknvkdAp49QV/anANcLbItl7qDfEBtvpeTwINBhOx", + "bRIQ/h1XqROeGITIkszXxAS5+v/u3bt3T5M/vv3uqLoe1DWaOuqc0lsLgXRZ60qTOhKztr8Ujee5gogm", + "ge8RlJHRieTdmM9F5vMYEQLvkHBIcmxOiJREto0IGUauO+E62xhiD3t3gkr+FfkUNo6PtGHlB2U6UJkH", + "T/pHdKi085wOYI4mzBBnaeRSvRUT9O/sxZiTMxNPx/ojk7BLNGLNdSW3Y5osivVWtexipfQcE6pju2mb", + "+T8ruUyTDc+6SWdaTrNBfQrdEz/yTAKfPZOhGxls4DwupUDkt7SY6i+R0mYLlPMc+s2o9W1UtTVT1cpw", + "5cG3czIi400vYzbSUJ3KapZE/zcBwzcN66Hrfh42jn+tQuhZi/b5Ng2H5NK3z83GCdueIbYhReUsx05e", + "rM53tNHjkWtiQh8m1BTpFExowB5yN7vrAg1yMMQuSjGkvb3dwyMjo5+F1ZVOUZHnmfbKkBJihOeTCRKi", + "EjgYRDpAu6bl4mJvuqYlbt3cdE+3Y/6lzZbipYeHbfTdQbttob2jgXWw6xxY8Nvd99bBwfv3h4cHB+12", + "uz2LXaLtDRDvgNNPYIuBMcQhoRwQgIdgEHlO1it78unvFxNw0ml+Zv/9HN5BD/8hsilO/n5zZTQSEk6R", + "8XsJrATczyFEgzDy1BepiTWoo8D1IbMRmDV4dXoFIk7g0/mNWd1niqNS9IsOYTyxbB6Os2xoHNmnnSGd", + "tt1IE1/s74qbLqTprrX3HrTfH7e/Pd57X1mYauxASZ+YGaAw9MO0bCnhFCQS5FW6QvnSIjFqCr3fcOTQ", + "mH0h682vpHd2YSHP9hlu/U/rsH2k48MW2W6BE+gB2/coxB4YRy7FgZtCGpJ2WVnsfx/OPnY/gZOzy+vu", + "D92TzvUZ/7XvXXS7p/9zfXLSuf/lrvPY/dC56/6z89N5++bjN+PLn+h/LjrtjydXv3+86g72T/919uHk", + "8aZzcXbzdPJH558f7j793PdarVbf46OdfTo1zDCD619wp1S4RltWC1zIVKNIvAjt0CckKxIyq88QzRwJ", + "Q63fKkWl01TLV2jSBs4YvhfLA04OpCjSjBymJmJHkK98t2KWxc/xhxwEk9gu5JI/4ruRzHnhkwL9cYqQ", + "9AQQHdYhh76q/iWYQi3a19kTDSG3rROPSn7bcepZevH/vPr8qQeFJzlERPiRQjBC0EGhwFbqK5kqHEbU", + "v0dSo09tz1etiAHawl4Q0Wv2kpHLuVLzzcPyC3eiUR8MsedoU2myS9PxAzhhfIhp9hzYRrPxe4TCSQ+G", + "UOZhjMS/U/w3+ax8/2Mwm/r+mQ7h/Pyiw3n6ie/R0HcNeP9ko6AgI0luvnqBLZ+tHArJbYshwdh3UFVa", + "uPQjis7UiEZSYKPlU7+MU8YxMtf1H3+DrssTT70J/2cm+1L+OjXFhY1csJMyGy+3hYqpalFAd2zZPqHW", + "ABLkWCGkyMVjbpPlcI7hQnU7IAaDnc2UbC09A6tqYDBJ1xFwlW4Fh8FgHNKR76SXpE7q49l1o9nofb7i", + "/7lh/396dn52fcb+7Fyf/NhoNj73rrufPzHZ/+NZ57TRbLzToCjOG+QRZuHTcRwslMmeBpiIwuc5DLji", + "Wys56wB7dzJXWwasSexbF15pTETK56QFeJgCU4LcIU9/AanxfDtSecK5LQzkzmm53PYIUn7iLlJJfOUn", + "xsdoxtsd70DRkQmvdViWJw+zvGIKKqZ5y3MznWiv0sJ3cvngNaTdpxPh/QB5EM+Y+b5VmPq+/Y/1SX4/", + "P78A6mxnzoJfq9T31Eolv0pm+eXq8x74HCCv043fWkii+p3rD6DbK0wR/8ifgy0YYKG6bedzxKUG3Tk/", + "1/PEIQG+hwAZQYYvxPYD1ASIaTPcwJU5BvEHmQT01suzyuOhi1f3uWB2AS7PficBsplCzimc7EgGpa8E", + "MmsZiI2cHf7PKSiNC0kn8AchsnkgzigETs96l2fMbjoFFoiItsFqF1rgimLXBSPf8yN2NFtUxnCF+mXz", + "zAzq57/crryoRL+oMdufonHgGo3da/kkVqPZwuN8fp3SUkQW89kcVehp+9U8rFrmfbUXOxFTeW6Xn0/f", + "ApcqX4ErAWqgVoiGrVdOti88qnmz7vNT/6zloAt8iZMwmHzB3l0LXEVB4IeUMNHmOTB0gExW53dGmoBE", + "A5mm32QC7hG7jp28RaSLYegzTR5c/nBicU0IQ4/yafmsYeQyWvxFfivklUiXELeWlJvWRUNqjRm0Lhwg", + "V125e6dnw28bEgZaAr1lsryuShzul0iOrX7/Xb/f+m8iQW63/nGckie3f7ab73eftTe2/9Hvt7a/kb/c", + "/rnXfJ7u6ihKrY/pPJVbn9bmKqmFWrSsGhEXjRAHTJpZHTNxO1Sb4RKJoLxIlOQRmCxlhA8otMbQg3fI", + "AS4eIntiu0gkEJEW6PlB5HJ2LS5Ycg8QFzdMtfjsuRMhGAzOxdvslYKfFX02ZI5RS0+YaT0Sf4+hzw63", + "uO6x5zBG5I51hQRR6EjtW2Yi8Ew9gXsqMVpEyANkG9Vy3Wj/VbO4fhWm1a0yMAxWBTsQ7X1mkGmvM/PX", + "rfbSzp/8v13nmW+TsNsTQ1s3BpR+vsNOgdBc1kZea0sEVyJyUhImEvaTdK8cN5hs8EPpO07oiB2OCAsL", + "n9Bx4wOCIQoBubcmfhRa6gUmVEK3cdwYURqQ452dNDtg56lzZ8FdU040U/7K3sF1+9vjvd3j3f1/N5qx", + "Ilz2DnaKEEJMllGoZfCjeMTn5xI6NyfqbtB8g+YpNDclJ/1cpKjEFpoyA6RLOhZWynKcilkpI7ICHub0", + "GYGYhQDyxwk8Ov6mpk4jtiHEmWB6mSC7UO9pKD+TaOU+m6xKoB2FXLAGkZxoiui/1qyEmaW++ngj8I2c", + "8DrRzAwcUTLDmA1omJEwMxmuyARL4pBG8uJvVAU2kjhGHFN4NnMjkTA1DuiUWcRL02aIswcLRntKXOFW", + "/K5lGlRyPInsiNALxoUN4HHuXAKQOPz5vuaJK1M2hr9Tvi+zqglcBciixhyiXuFeUU2UPIKVkaUxnjeH", + "By/leUhZYDFGllte+QhKBoHnWYUBc+cbJo2s840h+N8FDALs3ZEZ5ETCjDNDmEhhHtgyFDH7ECUGbUUp", + "tSiddcOpN5x6Nk035mSrqunGABZrujHWF2m8Glm8huabkmEL1H0zHHNx4vKNCivTHRHxRN3/5u7SJMQw", + "lhudqWcmdfXGVKG/kuIs3o35sI7k0U6NOFuGyBTkzsUdngvBfZpUrqu3CfcvMdz/NHn7sf6AL3PZxe4S", + "L93TZJYo0CZ/YJM/sKr5A7G3vZLseJr0NO/8vLkHc0WyA0l1mzD2XzGMHWju9yn6yJyB6sznG6d11hUi", + "ZF6h/0PQZ8r5kQl+WYqEi2Jf/GHCX3+9TbOn4vjnEuOvmaW8MO5agHQ1OrDW69RmDSc+TVY5lvg0MbtX", + "niYmn8rTZPmOlJQNV68PRVMV8snkMng9BcB0dt+U67P8JjJQdAhcdwwCU/JbQQJFuVjCTtFKUzDmFqqi", + "84XFjZP7DyoOb7rQIEP3f06Bkj81wXlx0utx55LhKMI7fheBlNRUc2W6ZPwu14zEFb4k4SDWRTNlFvQx", + "81fokjrSUtdTk8x3T7fs62SrDFe46AiFqRGEES2/iEcb+L6LoMjPxNRFJbs2Sput/PXpYJru3piONKvH", + "l25zEUzpK4OzXQk11IIUDkxjNYHZ9srTTlQMaiwLNf/uCYKYwYOVql910pN+FvkKkPdtNDcUekDhhI6E", + "G3M93EfJst60+yhZpkKnXLT5TD+8+m+JTG8hkMBobiTwcneMoKrqruNEgtR4B2F2D/bFSU+ZU8YaJwGy", + "C9VFtqmFyqJeU+HQar+3dr9L1fE11Efx3ZngvvbFPbiypgaLvR2R5SfXIwQG0L5HnsMxjlNsCKJQlFNk", + "Slq2e8Bf84JFQo4mjCmqVv6X9jWN7WA302FgjbxNMU1O1x1m9jYZP994mxK/xYUdmF0WiU5lje3Aiv08", + "ec9FSvtK+y1Ssj2Wgr/epqQR+zMlSzSx0Ih5P3tL595J+vTxjgbC8X67vdQrAqZ9eoGnqhRha/FU/WVO", + "fCb3ViJ1VtXFlUDIP0gAYgeamlOc8NKcWwbzri7nlq6BzubriI3dKVb3GI/RtXQOFYxw0b04U3te0Wpn", + "yp5uVsf5KKbyP/iPstnZY6Yc8AKADWNZv/nNfQVXRYO/2YhCPIuPonjd2UKZIS6rGaU0+tlw4MdC/wtb", + "/zDybLFDmBqdwbxGkSgiYq6JlFQsGYoivOgpQDaT5UnRkjo8PYwfGjv1RbQEwvj8y0EVgwBCw8imUYhq", + "digx2M1VxqtWwkkTsH4oRkzRmFyG+3ueT5PGhubiNH+a3EepAkjJKFyHh+EA0xCGE+D5nqVqYLEdVvxN", + "NJMQxoIl+iOpUunpRlnloiIIfbZGiysd7d0j5+hwf2g5+9+9t76F7w8sCI/2rN3v3h/Bve/2jvZQu2FK", + "JuNGxUvWf84H4Eu/RxNL1OwNIA6Fm9oXlQN5sw7PAQS5skp8p9clLfATmhDAU4g8n8Yl/ESWUGY3kPeA", + "Q9/jftvjRlIgnN/WYwpBQ1rTjbTkNy67lOJG0HNcZOJZM3fNquoRTTpgFlRtMjCz6+sekA+bM9VxktWb", + "VDmnlKogvjfWwjJkkvoRlfmDaWfCn5zfPYPAhTYa+a4jGJ/moR34/j3Z+RM7z41sOmDr3Zx+u1wyFren", + "eW003qxCbjHYkv0+ES9yCh2HF1DLNw7dfq1uoNnSZ+qQ+SmUok9RJTT0hOyIvXDie4LMjVW8VTE/Wc6N", + "p3na6gvogniYODwg5iuRacmZ8wT/+guvZdY+e/m1bAbTV3WWZDNBV6kwWyGxH4OPZ9dNwEi8CXo3100g", + "CLwJOH03gaTrJmB0zj1U71R674yMYlPwrf6Cb69Gobrs4t6DllJIfoURHYl0R4qcW/C3vwN2RPOFgA3z", + "2b5Z7Z0HTzqxeqWhRRwB5XODrWGIkMXb09yjyY5QSWJ9dtuEBYXO55/TqZAK3z4zyTKG//FDiwuZuJ98", + "nLusHLUP7SZ42N1ugR8i1wUkm2Kp3tpttVvtbdEMiCZ4znQm1bYmRP/hGrroYfZRdlKSVRlcFGot6kdI", + "AAe05ve8QRH27lz2jNojhjlDBlPSC59Q6LqJ61k1a8JjeIeyLuZ6WKeJQi5TMjN7YYGreKK/uDlm0QIX", + "MOAS/cx78CeAKykdO26L4UeUiIrRA8TDsbJVaqfXlfG+LaE/ENUwXNyH2GaHsSMxLVEm8p/wXlLxC18T", + "8f22FgCGVDRi49+SZnpEqYQIrM21XhODRB5BtKkf0tdEZV2nt0aGo4VONBGHmC3066Jr8bYhjQqFlhhQ", + "BpDY2/HgmiLFQxWYUOSh0PhufL9I7Ua/0Sb9BjMDmY0oRpAvp+tVt0lWRXS+2ZJpxtv/2BqT/5L/jv87", + "2v7KbKkWrOwCPvFGf+yFmIEwJhgiuYVbkk/yImwqgKY88rMsYPdw/hU8mwlEDzIY8rgLYgxaTWShEei2", + "TSb9YqZeurHb6BESVVxeJtJv3VyfbBvb6mac59UK5+tu+FkBcyGhSYralj/GlHdEFhSlHtQIbLWGE5AQ", + "fOclvdVkIHgL/R5BlyFf7DhgzHN7vubNhBqrDRYmtYQo8EOaBaq+nBEt+DHXMareErWiVwGx0U6vO1MI", + "kH2wiSkmESa+JQE2R5nMKGyOM+nv7nyV2F/pkNOlfOucjcjOLtXNPilYFbs7VGEp7pTQik8xASgMpJI3", + "DEMIv0Z6nJsqb8UWmOnF21Sed7x/BFFLJLpqlge/oBrHCNVj7aukC21gyYO0kv1UxaqEWipKe4Zahwbj", + "gLqDLhnCYdqMH/Bfn2+fn7POuUxMbwyxl47tyWJYpDXA/8EhbDnoYYdwjCQ7OdxhbArbaCcO+C0r6lvE", + "iOeO+2bYSC2R3g0dbuhwRehwplg8M81WNQrPYMvE3xWZpWZMaG9pcfhOr1s1BK/F3mU0vjAEn+kUUubM", + "LPRhplr0VPdIVnM+miIVPa3URToQ8VJnn2mLrpAdIlqW3T7rtQzCR0xB3vMJvQvR1b/OAU9OZMc3EPUc", + "CHn0QyebPb138MLcbQHE0u/9n6qF9YwLq+nyf3zJJ6tb8zULb8wWoT6zlpBnh5OAZgElUbAfkn073Kd/", + "0y2O4gNpT6mmUp4wySGehn9M+NaJg02Ah7qZij3bjRzeyH2DnotCzxmLs+nnv4iMwSvFjQxqpDpnKz5n", + "TVplmHIFFElrlKa9VgpOivpm0y8kka+qiiHBi50gmVut8jRSk8cntDRlIyfz6kr5MyKzUIF5E2Ak+ggb", + "0Ovz1fVO7+Ya7AjOQGLXRwt8YdO1OOp8UUGXENEo9JDzPSAIgWIaEtcv+dQ7wpYDylM88B2MSCZU8hbI", + "bIrdvGu1D1MtxblNnIfRZPxmvp1GubMQYyF95UnnVegkls2p7Z3+dewRFFaWcgzOQXDxvDNS3iWiIUYP", + "ptu8H88SiuMWc0x2UlfA3h1wkNSgUpT4BgmnSD5t6GlhcmeFaYkRfJei8WurYS/j9mYPaDXszLk6N3ra", + "6+lpZvmzrKjUZxl/xZ4occEdQmAMJ+ABhpPvNZtTmt9MT0OazemAEQqROYxVn+bJNqm4O7XtR54phulT", + "6Er7ktnPUh7q0u3QdHVDvVeY4ypfaIEf/JD9EYWYTkQmSCJIxRaz/VIxbqay8htSfJfju6WY0O+Z3stl", + "OYAqP0ju+mACMK/75w94Wjb7JBHcfKbKd4Az/M90e9zQ9bqw7/1z4XGZ+XluP7sityp1/ZkHnUkLfPJF", + "RhDPjkrjOW9w7IItzwdfeGjnC/DDvvcliRN92TYl2aTSKbKx6py0nz+74AqOEYAknTIAdtSJisz2lPvC", + "xLbLo/W1gF+tws9VNIhXJ4w9zY+RkxvdAve8lmuxpSU6dE9503K+JWmXjn003Bu8h8ja3ds/sA7ff/ud", + "dQQHtuWgYZv9xH4xbRNPAhNiyQhL8jgFE6/ycYoeen5IobtzdX213QLxXRqem5SkTgOi7Ynp0kyzMcA8", + "L/SEVwhCoQmUD1imjsp3UvAoomiKJCEPuhOKbQJoCO177N1tl82qH1nZzPoyapidaHSuSrl0Tq67P59p", + "Ejj+ofsp/ufl2c+ffzo7NeqsOow9FxrXo68XBC70wM1N91QUFICU8dgxppzXDHCcrqtlKzamzMuLYJuu", + "WsHfI5TeRdH4ns3Msd57kIX0RSYbI7XvgfRgQwJGkIy4PzTrxB6IbgIWHNi7e/tPkz+mUq+gPRPc04i6", + "onA1CEqdCirfFdCnjqetVHT7KoMKU7iRPGv2Zpplnny+uDi7POl2zk0Hj54CHE6ucfbqBGe0u3vW/u71", + "3v7x4dHx4VF1OcGQ8lPuNsZH33VqJKSUVhs/NozuB5+9f0U+hZcI2qPUPCLfOx5G/Gmo/DUKfUpddM4o", + "60ShSPzZbrvdNt6K1T+78TDVDdcLzGT2j34UNpqNUzhpNBsXvidu2yTrks+nxAfVdt9WQKNa8J8NNB8N", + "sC9fRgfFwGdIIIcKKZWoGianyaPaN9K8E6y7QIcqJZkSCiklh0q4XxW7K6JzueI2bwpk9syFw70q76vl", + "FNf1QKrwlxlPoJjiYhV4umJas864OH3QNPIcnGMuLlAFrxalQNauFm6p8JaIgfueDGF9zyvj96Tjy+Lp", + "TH7iE+COgydMaPaMyPZUQ7EOfjOF17z0iEzT32hpcJncfXULRNVtS9dh3JLuEwrDO0SZcRmq4mlss3wP", + "Sb9autCF27h9bv6ZaVczbNw+32a9CCOfaQuPIc4WrYQR9XMFK+XNMAJG/iP3Z/zoEwpEyiDARFq+8v6D", + "LH6mLorFdd7AFzb2F+AgFzEiIqJyWsihkB/we1ZN8DjC9kg+kddh9Bkjoi50xpdpbDciFIV8yBb4MoZe", + "BN0vyY0aNvUYUmxr8zFLStSqIOy/LrZx9gJYX3cGy60RYxuJlOtK+SZJ8uT4JTAQhIhXykAOyFfgM9dF", + "cQ1JNThENo2x5+bynNOauLAly3tyaBOVU1Y3CkLfseR3x4ftdnsHBnjnYU83AkTJlBkQ3Fw0Ga5uKeWy", + "xWjHYTjMbB3ENOGaKyEyTHN96IABdKFncwaIKOXtoLKUOYAE9YyZh0mLJVHqI+60hDwn8LFHifDGYpJA", + "Jy+FyjPeboGO66ocAxIXNIhf5xdER/AByTvRcrIAeQ5yZDVCrY3T1ztf87XFZTcQv2QqnnzPXcayHqKf", + "uciWIJ2WwrSTymFq/fb//u2rftRu773/emv73TfN7/9+/H/9n7yh087tVy8vfqOv29EpUCt+OLHUK9bu", + "/CVkC/IEtQuFVWpeqpuVWunOEu++4gxCgmRLdD4ifDeS5b/TiFlc/9vIhz5oDGiL83NRQjikXDloChSy", + "/TEiogSxQu/tabzJ2uXcaSpbajbEYky06kKKH5BcrWGxqsfSOHIpDnSqltvWApd6/eNhRKMQidctKZvT", + "I34vLl3LqiETRMGWKB0yQvI2o/wME2BHYYg86k544dDtFIF81+bYhseMESpcE38ZXBK5ileu0WUwxl5X", + "nO2uwUA33K1O8Oy2hF8WXvm9Nl2q5hupXYIVrCgf5vA9j82TG/REPNDuYAMnVpsEt+s3Dkm/wf/bbo9J", + "v5FGtprv0P4MXezw+c/C0Dc0SeAhtPxCfuCRNX7FewixK+JgcqS0HzVAdktdmTHGdwmBd9OzWhEDD6i3", + "9RlOZPnzXJs+Ts02L2+V8PadavvyCxp8CP17FHYCXD0YrH+1ufiYZGikdtOYp0Gob99bNBT3Z7JXrqDr", + "noyg58kSXb73mx0T0m9Y+hj05i/PTfkSiYRUzT8UlbPyD5nJzmHVoLuHw3toOSEWd38zOg9/m30s3xM/", + "WLvHR+0jpiSkft0Tv94mO80fc3teW2IQYjvmJWwRP4Q+t8SoH2BbbVhLvhYvd0F70pR1xtC1XwwD2znt", + "blzqRIsvx12x18C1eA3EiALEPbkQ2YhvenIWj2hAfPseUSt+GG9l/GyZhW4NqPuC+446qXRitC9uV9gL", + "ferbvgvGyMFCkOTqf8n6HTF+ZTHYjDtVGZ2o5PMx9KOgkcOx+QfRcHGuQaZx9Q8xkWcKYggRAwS9AkH0", + "aS0gt4Fm3f1DaoRcf4cMU8n7etPmaOXSQOlpk1GUgjHwfUpoCAN5dYVsp5NNX8rPst1hyrdF3RlMb0vV", + "+jnya22rbqccuyQoQ/cdNLjivEPRick4/okBBzj/0xpE55Q/nWFXRd4T8ZnISCvm6GtOlRlJUnkM8ZHa", + "nGm0nd5LkxaeUenF3ojqpJz8pcGnzhvE2JE7bikMs3PomEJ9OQFKD59WlHVhOg39xaTTcN183y2jTFU9", + "AoMwSqlKVceRfDej7xRVXa1MPZKsc3VZL2AA/CHIUbdwHwyww32RsqQ0N+sYFmjEvnWPeC8ixRVU8Zkx", + "9nRQdw1nUdi7qfZ+4ykdZMa244Vdx9en5zjP83nL7aLYApd9obNcM6+/IZWu61c3suU38/RzFgt5eSdn", + "g1Yu4sfgQeuyQ3ZkX5y4ypqYP1M1QNabmDqcHE2galwEWqF7ruNyuglOwfG0+n2r4HAI9JyB/zQzaPI7", + "I1zymfVy+LKFgtkmGsMmVdruJM4qzcCUck4TWtOEr6725C+HsZ/1Ws05RJihzDMz74LAnby0TPM0rSqt", + "hFXQqgRe5rWqRBhTX0jZObUqqUsmw2eihgvQqS41LlVkQCi8mctqVBPkDcYyz8fLzcZO/GYCvzZWZfMu", + "hj9n2SXemVmsu+kHUuON/o2jcuOofHVH5VuuCZUisNS0GdJb2kXAnMFcV4EoJg+iwUw1M+NPNsGjFE9m", + "m1LEkO8wHUUDCz2wZeeL9cWsSi+zd3XzQXVVOm5gQiKUKaKXeiGIXPe3ONbLFqVxj9T0xdzjI6Y/RgNw", + "xl9rLC84YdidlwUn0vhZl9B9+wf8xpm6PMAsR4/PdZnsfOT7951edxHMvEpAbkr0ral6QYmi0hyv+Ia2", + "THE5adn85iCXaRCTCsu/igYcC3U3rTaSVEac+UfS05/nHyXyXjzOc+FBFUZ5Ouput3LnJslzmSyzAIWW", + "ekk21IjL/G+Oqr6jSr9YTE6iUL88vTKqmaNBVpxTmR53Ed4UnTm9MDRjZkj1BFY0GsqFVHoaZaRd6vdo", + "IriaHixpgTNoj4AMo8DUM+GC5hnlpKxvGYzjNsbmRcsLtDwiSEfcktuEWKaGWJr8q3s0kbGGTbSlONqS", + "LR29kBDLJkSyCZHUGSK5LZZxsr4SM2XTZVczxXdnIEveozWR1bKWy1ZEhJLtoBDzux8I4PE4oknLXeJG", + "d+kMdWWuSWCnEtvL6tAWbEnR/VUtJTplTLEx1KrvkIdC7pSRtYGGkesaG1DKktnGVtzJKIELsUijTpWa", + "2nocEWT/pu7sfAPeH4ARemLCOCTbrb53qW5ToidoU36h0kbgf/1//7+osAEwZYfBUxOROwGQsF/4RR3P", + "p6KbG79hyQ8L3kHsZRusCQh24d5g3z5wDtH74bfwu8GR3XZ20d5wHx4MDu33zrfou+ERbA927T1nHx0M", + "D+H7wbf2d84Rag/Zt+UlHipUWmo2HvVTrGiGipe73tDPIY2cPcmAj49qKgrx8QyJi8KalfViwI8XnRN5", + "mqJVXnLGBXXEWrPVa3pvtXevd9vH7dnqNc1M8lMot+pNqstzi8Ch4AZgK2YRTX6BhWGmuDCm2imKu19I", + "kciOg1xEUYaNSOfQYzEw+Zu50Kb4AXH36YN/n9Vh4qezFZ+a6yyepyFaeVkOrUhbJUvJQBMvLH/WbFCf", + "QvekWvE5sbWqdFwiSDJKzp7xilJeUKqqc7y7anz9EdvqOilfNi+iwH5NgB9RGgh3KpZ0zPt9i+tC0iP5", + "y9XnPa6Gqgvp4BrBcd4Uuzy7uubvscVwF77sKZ5WxImqdJcfV/bPFFxX9rNvGJpqXvD4ADcrxH4lDk7Z", + "lIV3tfGY6nrc2G+1W/sNrYXxjs3whrszxVbdIeOlN1WCz3VlfQVwfX4F9I+1m2euD52kTaf2khA+rb53", + "PUIEpT9ntgen+KHoCyq70v94fd27Sl2MlSFKWcY67tjTdaQn4URfUdKPhq9ur92OWwUJ1NRKT+z8hwgd", + "XBDGNLLR5knRI0chs4MjtdnPzcZhjeDwe2JlQHQ9pphCV3VG4De3BMVE4zEMJwpQ7ZDt9F5SeMfD1NrS", + "NQRkDPPJ4lQFIzqyQt/l4d8GdMa8vojs8YNCHr0OfEJNNQz43UcIPPSYxTGw1Tu7AIKDbqtSAIpQuCjV", + "X8ZEIaIz8eAY29B1J9yh4Ee8CiqFIVV3/tUoOYwS8GgLbjRVy6QPvjOpcHxaIEYDr3HcsNj/Ppx97H4C", + "J2eX190fuied6zP+a9+76HZP/+f65KRz/8td57H7oXPX/Wfnp/P2zcdvxpc/0f9cdNofT65+/3jVHeyf", + "/uvsw8njTefi7Obp5I/OPz/cffq577Varb7HRzv7dGqYIYmzjCeWOG/LFv78WfFfbFIcHE2rVDwGmaPD", + "3UXQYRn66zgbBRIzUkr6c7NxsFyC5HczU0gr1YNV5A0pyrRTBFEjX3hupmXSTojYtMKBTIyNcr2IUzYN", + "8d0dEl1jOaRMtWCsTJcy3D8m7k27iEyIqDabYSU5JnCJMkzgxYKlgmWpTyeXJDtuX51exQ1Gp5qZc+ht", + "HybU5KYXetuAPVR7K4HKiIlEZdvbPTw6qqS3ldGrtvwswa4clcToKJGwTglqoA7e8o+fFLOCTH2S2O8A", + "ppmMIoK07BxB746LTeXnf4ncFBOn5WbSEp9nBWQ291Q5N3VQuf+ILy112f6wjb47aLcttHc0sA52nQML", + "frv73jo4eP/+8PDgoC1qPGCPd7fi/dBUzoHTyMomXd5lDbHbWslc1PGaeRlld/ON7EJu2YKZxYxEHAOV", + "l7kHyyNhHSDPp2DoR56zkozERLn1MBDXHVtB6D9gB4UWRePALTX+uE1wfn4B1Dcg/gaE6A4TisLE2pMM", + "oRkH7N0Jk7XincFERB6Ndtv5+UVPznAdAzWFafzAR+ZNwOUnQLqx8mnanwPkdbqKLfweoXCS8IW0R31Z", + "DMHOVYfdN1bdn1GG60dayQNk2PoqgfNiQ9eMLqtt8hbAnJAce0FtE1D7NAvxFdm8HUep1UYYcpZuR8N2", + "GaMXRRRVDXJRP4u3AkBP7EceYBtnHNH6ZHmSFBWkTZgxqwFc7TANMyUGZapE6M4Ejt2aBl6qpWokMwMR", + "GZFAuvxXxGRNF5ZMagzJqkPbArKjJQp23xu62KbASkiTJ48QOJbhRuiGCDoTUS90NZmRILoyZlAnPypW", + "BirbFV4By8qZGAUGgpm/lMp8WXoviAYutvUKfNJ80NmmwXbgznC8BtZBDGg1/d98Dkalexma/wzgLNsG", + "MIO2HtaAt3iu0DSbAR8RLSb3wQRgSkD3NE/nH5FJs/8w4R055iN0lYVUtBUrSeyzKwY1Kz2zUCmF2CUb", + "wqxAmIwsimnCqdl8iIwRM97jF3pJIXQzQGkL3RTqcuDCJbJwRS2USP9Ctkl7NWwTo39xxW2TDV+bEu2r", + "xlUWaY/M4JOc1xXZVJnVTSCTZ5tA6MJN4IeAJ0lPdVfO4KZM7eEUV2W8mS/0WTYrgqPdZcwmlpumT15/", + "+dRy73ck89cudqSFQwaEpCTMXCBkrmREqdilfpvBPHv8TTL51EsRc58NQ8RcarzYHJl5bjwj+dnrObT3", + "TA7tFIHP6qFO1bNZQKfWal7tNXJmF/qwa07dKnJj57zXCQOU3mveAcYHDENCaMtbX9LYJKJRblPrzh9n", + "A8Y9KpqAgc30TlvecIL8vDljCX03fc+pgrN78U7uFBrXrk0WjF6ummAP/N+di3Mm+P559fmTSkZ6JRd5", + "hs6nwK7c4+LiomC4G1/5VF95zAuyvnLPie+crbPf/MWsz6CVzuscn8MnXtHyzpvcmT3Qru0Qf88SeoMV", + "ZPTLFXaGF4A9h2t8NTziq+cIX0f/dw3UPYO3u7KTewbn9lug3Dnl+SI0nQp0twKu7TXzaHNHtt7wtF5b", + "Yh6f9syu7HUjx7+A6XEjncaZHX4Vl/dsTGR13d0bvja3R3thlsKObDg6xZutiuCwN00ZelN5XsYp3el1", + "f2KTVmN8otmuieml2i0r4NZfMRHbU/XipjqYDX2V6w0MedzMnpmQuQYtQtYhLnNIfpQ1AqRfQAI0F3Hl", + "HIQCf2qhLlXKgD+UAK61qiH2Jltj5mUKRtGYS03gzQJRTDAK19Yxa3cl2NvrOES3nEhMIghRdKDij0Tc", + "gWkQ26vvAS3hdPVy3ikaz86fMMA/IR6iLvWYXvKSKwxWCXoLfPZsBGQplibAFNjQA54PXN+7Y0aprBZB", + "fT30g+Lev6ZLvGys+ln4clh1LlDM9lQrB8fPm6tqbJUpmOL0btX53ghPclIrpqOxc7MrM1yJMRuGW4Hh", + "+mGMOautWubYw1J0ynLHlIJExKpVwRRZwcsjFMkKBBH1LanhMRnie6iCu+pNsiZD6ufiWdOitFtxZHXq", + "ttkRl5r+Obtmu1JOMHnO68NiN+rtvM67ldRtd5KChMWVai7jd1JOyJe4JZIh375eG2/w2xAg8dHV7CIx", + "j7viwiT06cZN8va09pjfvQbTfsIVi5qwF1/p+sATRrNfHniagHTq/+tdHHiavM6tgafJSl4ZWIkLA+xM", + "3tptAUXLM9wVeJq8+kWBJ7wmNW8kG8rw4afJwm8IPE3M1wMYi6t+NyBJ+M6y7uTOQPp+wAzXAZ4mC70L", + "kEHTOrNxCocu0i+eJqtzBSBHvmVQb5L/503+f5q8wcx/TrK1MbOMSjl79v/TZMbU/6fJS9MV+QjZG/aW", + "erAelW9icGdK8ueS43Uz/ItAeCWr8Wmybrn99dJvpQz/p0ml9P6nSR25/atOnfNI59rVlWkE9qp5/CtP", + "U1oSv0DtKIuTNev7s2XxC02zcgr/mgjEN20jZNL1Y7Nombn6M7GITZb+2nGtMoaxaJX+5Wn6FZia5vmd", + "1JCg/zSZnp2/VtrFemXlr4UWUCEl/+XEVVcyfgUSSvvmXh7rFjQ0NQd/XTSGTe79Jvf+RUxsk5lUe+J9", + "rfy1VHdZ2YT7ejj1Yjnyy1Lsnyab/PoNU02Y6ptJrq9bO3ydtPq3xIDMifSLZECbLPpNFv2qMdKNolpv", + "Cv0raan1p85XcCJk8+bflnpalCm/jhJikya/SZN/08r3lBz52rny2A6qZcdfnPR6tSfH+6HMmzbHRpI5", + "q2fFX5z00lnx+Xr6F+Ktns6L68+JTwBZbk58Mm9xTjx6QOGEjthYbzMvftGZ6YemzPSxHfRmTE6XGP6K", + "yekaja10bnqKFygOGJPx4lLT1QllM9MLIlHq9QVliRvxpR5FaMrQS43uFJBFHoXi09n0Q62a5p3QzBtK", + "9dbIrjbekFGPZsj0jrGyaqK3Bv6LWqsla467nbb6acUjEf0WW5yuh6xwDrgZ6mqp4PFpvFomeDkEy7aL", + "YmjWIw98IbRdngUe71B5Erh67UXdS7OUuy70Oo/4rl09mUJsr5MUvib0xXA9hehOzYp1xRzwGIZqKeAL", + "EZXCUb9U0vuL2QbtV7QNNv1I3wK/KmEddWv9ISLUggGe4hK9RIR2et0lOkTVjNXdoZ1et9gReokgvw3P", + "V9PpdRfnDGVgLNcNymYsdoCGYuWWi3mJi7fZTbRek0zRQyW/pkRUkyezojN1YQ7PmIZW2t2pUbpibewn", + "jtYL83XKSSu6OtUZL0abkaPXo7/kBluqNzMmhjxOqB3fuC+rui/Zbr0hx2VCRHWReUqBqey0jGm/qssy", + "AfxFZphkN2ZfpS6lea7Kmngri+Cu5q9UJ/Fq7spSAJZtnShg1sRZWT89l7kqY6otd1TKt17kpxz6oSLY", + "9SHTalK5Bs2inIxexw+5HpTD8FjHYqdejbeiE1JBUM0HWa/sMzsfF0xUb1Bhby9TYd/4FN8A7ylmBAvV", + "x+euLVGZTbHvZysoMY1JxVUl5I14DtGb0APWpMjE+kjzshITLyetF9aWKCIhcC0rPWACINjfswYTikAI", + "PSe+b4g823eEi3+EnqCDbDyGbhMEIRriJ+QIt8QXGODgty8tcENQTEA/oYmoLzsBvqeTlWTVCGDP9seM", + "AakL1GI0OsKE38cu8MHNdE9lGo2bql6su1ayKYCxKYDxlhhsWX2JWplridqygmUlauWDArxX4YKzFZ2Y", + "Btam+sSGo608R8sxiVoVxGWXl6iNEa0cyxEej1dhOZt6E5t6E8tlnWyD1ubWcCE/Yzpicv/fEYxt+Spi", + "bTUdSo33IEQP2I+IsuKVcgA9hlqBC21loouNqcHGLykk8XYM89kLTbwpGbGpOLGpOPHWFO6iIhO1OxAI", + "skNEi+MclyqqAGOPMXRdQKgfMiwTX7fAJaJR6BH5g8YnhZfUj2jfY9wI2jTia+evcY4uPM8E2VGI6QQE", + "URj4BBERbc0HTa4kwAukOjFF1XiD3IM4/mKivd3l4deNx87dD/EfyAFWto1azLpWOrWWxGesMF2eenVE", + "L449XDHUJVLFkIiIPDucBLwjGQVMYRIKi3zaPQXjiFDu+uLqQKvvscfSCiXa5xFhKhHlyg5my1LP2ObH", + "HWEHaOiHCAQoJJhQ5NnIhO3CkShWvqAUXjH4Aq4jlQ5ckxde6i+i/ofwnHMAY3y6iulQeNbFXQWhYot0", + "+Z/lDYbjxp1UVJn2E7iQDv1w3Hok/l7L9sc7D7uNZuMee+xY4gMZIwodSPleqHsYkMIBJMgKICGPfsjp", + "jATIzqNhzyf0LkRX/zoHY4g9oD4F8afN1LWO48apeqOnDx6nFsot6NDGcWOvvffeau9a7cPr3fbxfvu4", + "3f43U+gcI4zNhrQyi7995qf2grMXpytQWlhDJi4hPl2NOMgHmBi8FhhjwknbDwGW2s0QI9chK8zgXysB", + "XLLNJDzaPV3JrG9g6dxZqKRlwRyiKP8FUknTuaZmfvdQOIZsoa6qS8DEltzdOAtc0TMTWZiI6PgIho78", + "hB9D3/OY+Wf7DyicgDGyR9DDZCykXCx12LfYQePAZycCLDECb8YKPN+z+Nkhj/Y9CUMotb6D9oFJgImU", + "W02A5fU1I/mbsprBlucDiSvbK01zBzOKLs+nljBF0sJL7oWPCLdW+Obr4ivOTG/I00hbW4mFkwgJNtdv", + "0uypzs+n7s5V+fyrQuuxhGWUHoWoKEG8DjJvlltTRHa+5cwnIeqU1hlrl/I1Xbvseya10h4xRUIqlwMk", + "clUYhSKnBbrCcFMvE74LgPp9T47PmYmYuwkgOGy35c5xT50YRnnnuHmKbSBx0ET8HxEtpfwZKERdlShS", + "7qTlBd23pd3Fi2mQKNgPyb4d7tO/rZ/Sp5DeKeEdifGsEcb6mNJL9WGtC7tF5aqV5lmqh+NW8ePn/FOJ", + "H1zWkWT/fEqzGkahJODRie6pRpZB6DstZ9BiFN5K8QQsHOspfsV/Sw9gYCjPNWXqlYTVSSp8oyvrQs3l", + "0AlRFP+Z8nL0vcTNYUdhyJTFEndHEyAPDlzZ1N8fQ8okB74TmNv3qM/mQaFIQ3WiMCnMTlrgs+toLjbO", + "TJklAQcuAg8YSl+LLgFN0kis/K/pS5lV3Eq5UChu424WG09KdaG6e3xw+AqelJVIH5jqSRGItBHv6yTe", + "p3lOVMpDfV6TaBDDxRiLV+Fyjv4N4N8A+ACxy6VHlSs6V9oAPT7nIuNOmckqR6Byq1zd8I4B1sXXT4m9", + "eLnZAR1BChw0xB4igEdcXTzGVBjokDNNQHkccyizjfQxSNGtj+xRLkrnyEyjyr68yn2HLDClTC53ECqC", + "84rC6dV85qt9jyFHNDVfvcwz9p0/2X+6Feui5Im6aoUUA5VmjEiDLSZAe2FW/oHB+Z1bhvSDL10D+bQe", + "hTwWiZclJT14zEUUjODZMAb8K6/18XpY114RXv9a9TY+rfzN3AJs4l6j5dfcyMNSrfrGUjF88VpV7srA", + "88pSlvLdbCjLbIsuUZWZYp6mXq1alLbT6zaBtplTy9FepQCaqSZt9xRsaSVSu6dsLtFIcbugJCoMMKfg", + "0lR184fxkuYboKQYa+fkuvvzWaPZ6H6K/3l59vPnn85OF1GStSptz2Pcr4ldvwyTXm7lgAssbQP4veTK", + "VVjyxvoSDPWVMdIri5a/sm0OrLTUWKfypSSN2AuTdDt/6n/OZbfPY7JXUivTkC3YbH8tiz0FhLd+5vsq", + "WO7Vjfbl4137dfn/a9nra4TWBuN9Rez22U32peD3YnWsVzPZK6Pza1nqa0RTRrO9Zj3mEQ0GoX+Pwgrd", + "ZH5Bgw/83Xpaykwx3ZPZGGCVTff4s/LGMlfUt+/BdSjay6Q+WlyPmTRsy+0288YaXs/U60VHpWoNX/aX", + "2vAlRVirfTU1DWrCi9KovbD+L/r02SYw6YcyJ5KAAXZwiGzBkQChIYK8jOUA0UeEPPbVlW/fIwpsF7Od", + "46kPP8HhPQSCNcpKlwEKLdv3PDEWwMR3+XkUuVVSWLcYka9PUU+ypXnEpbpo0tSax9fUMW/a0lT14qQp", + "9A01qNHxoW6OlFeRqverSeFpVf+Ojvx1dO1N70JhHxvC1CGLCnXISq14DbrZlENfradN6rRerbHNdCiW", + "bS+lIFoT19oiOUJps5vUZpU71GojdNX2JrW6daPvGfSBurSbCuT3Ol6/NaI4hvU5nHeWJITnb1KRhqpK", + "Hry+yDp6VuTEsrF7xZsh4TXpYJE+lXXuY5HzaL2YIF/a1aKU5ta0t0WGK9TOFExVNZfPEzYtLzYtL16r", + "XmUJR34dP8qWE4lJBEH6Id809ii5tb29Zn05FiospilvK9iqY5Fsfanse7ZeHSnQFFCbJh0bHrxGWnGO", + "SSxHHV52D4+/HIeKa24skUNtenpsenq8Jqct6u6xUXxf2nxkxbTe+rqPTHeyrFYPkr+gph2f9JuQZZve", + "I5veI/VKt7VpRrIM+UGiQbW83KtoUE9SLt/3J1olO1fOOVNq7lU0KM/L/QVBOkKh9u5C03EVPMvNxdUm", + "lvu9I1l9MvvOo9gJCz0wnC4ARH6+mKRgLZ/3raYFCxyulhN8uOycYEVgq54QnDACjQUqBF9kKrCYOJsH", + "XBysk6e9sFxcMX5tibjZ4ZadhauIwyjK5d5v8m9nyL9VNPG2km9jqqqP+jPqz0w5txIxZ0i4jRfwUuNT", + "rbowzVbJ9GRta5BdawS6clKtPI7XzKgtA+EVjCAJzvrk0i6AwKdl0co9mppCK96rK39WLmpNqLaq9K5F", + "C5lGWq+WKrsW1CTzZDWsdurWlitejU+gqHYvfkHiUcQLlklob1Thby9b4V+p4N1KOjnXgiOVsYaFq/Iv", + "ytxX8FRM2xdLqilnX+Ng0xP211BtWJ88fXUS656knzi5X0ZyNaTnFxDW+ubmx6RfL+VPzcpfLzVmk4y/", + "ScZ/GdvdJCTVmYm/AIlQqoOtZgL+Anj3Mnj0CzLuFUSbjPsNo00Ifo3SZAoz7xej475Czv0bZ0qGJPuF", + "M6VNkv0myX7lmOtGoa0tw/5VtdlaE+vL3CMrl1X/5tVnYxr92kqrTRr9Jo3+bRsHxTn0y5IQsol+Ycjp", + "EtEo9AhQzdgFhK4LoE3xAwI/XnROVCf+Fui5EPPUa8G8CYAhAh5iG4A9240c5EwJSck20OvFoBecszLy", + "/XuxL5XbQ8lm2lueDwJ1JtubiFRp3raOyvUTYpU0bhWLIsw2DycB9e9CGIywze+nEGRHoaK4EQyZMBBd", + "4beGfjiG9Bh8eRwRZP/2BXwD3h8wfQnYIxiS7b6n8rnu8IOs7iyzuoSmlqZboawxwkcOePfO92z07p1S", + "8RSmMzWu7wniJtSXl2vikZKAlgQSEv7XF/HnFxDTd+x/YM/73hdJnKMxtC22l19UKMwY/3oU9AH4hRBA", + "8J0HaRQiIrJoSmNgV6oH/bqxmoVk7SRcpt6o15Shl52+b4KlpC+GwF0lp9Pql6CdGKGHGLnMHHFdzu1g", + "ECDI5B6A3gQMI4aVMuQdgjtEY0JqvbYD4q8ZMusoxiQD50i/e5C6byCTYTEB8lbUCmZUpJIYNEm2WEFW", + "plLu/Cn+MTWC1kPhGLJNcScgRGP/ARFNbrTAmXI+KFbvIBc/oBDz9yCV0TR2PvGJui7A4zFyMKTInXCr", + "JJENILFXpt5/WEcZUeovkFsUX8ZIAXSH6SgaWHKjzcAkp7qAtnkCuBW6k6AsSrlta3U9gcOu8YLXIf2K", + "7kbuDtQJX6qFiX/QnWhy2PeQEL++G6vBhPoB6XuKur07TR3U2YH4kDHKRO/VNV6m7xq028hDHtfKkWNS", + "Lg2exjfMPAp8jUtlIO1X1wtNTrkEu2LNUGFyWjOEdxB7rQ1fm8lRtiW2fHtJTI6BwcxuTCeccvl3nYiO", + "Gse/3jKUFFCbyPrct6EL5Gh85mYjCt3GcWNEaXC8s+OyF0Y+ocdH7aP2DgzwzjgGc+dht5GnxVPfvkfh", + "zk/RAIUeoohotQqyw9+JUIzFjjH0XReFhfPcxnuWnfDk8uYUxHxOaMKqwR5JSNrUcy8PvWmwi5NeL/Sf", + "MNJGuzjpAfbjpHw48VAlMVyfXwEbhYx12tyFwkb/8fq6dwWiQPS3AkxpHEo8TqY7Sb6aHf7z8wsG6wN2", + "UAiu0Thw2TAp/7i2MvPbL5u00lzzTvE0mTb+tFMyDZ44eeVY8ofUSLfP/zsAAP//scUeTvFhAgA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/gateway/gateway-controller/pkg/config/api_validator.go b/gateway/gateway-controller/pkg/config/api_validator.go index bb0698f66a..7b11f5c184 100644 --- a/gateway/gateway-controller/pkg/config/api_validator.go +++ b/gateway/gateway-controller/pkg/config/api_validator.go @@ -26,6 +26,7 @@ import ( "time" api "github.com/wso2/api-platform/gateway/gateway-controller/pkg/api/management" + "github.com/wso2/api-platform/gateway/gateway-controller/pkg/constants" ) // APIValidator validates API configurations using rule-based validation @@ -263,8 +264,29 @@ func (v *APIValidator) validateUpstreamRef(label string, ref *string, upstreamDe return errors } -// validateUpstreamDefinitions validates the upstreamDefinitions array +// validateUpstreamDefinitions validates the upstreamDefinitions array. Delegates to the shared +// validateUpstreamDefinitionsList so RestApi, LLM Provider, and MCP validate identically. func (v *APIValidator) validateUpstreamDefinitions(definitions *[]api.UpstreamDefinition) []ValidationError { + return validateUpstreamDefinitionsList("spec.upstreamDefinitions", definitions) +} + +// upstreamDefinitionNameRegex enforces the same name constraint as the CRD/OpenAPI +// (UpstreamDefinition.name pattern), so a definition accepted over the management API cannot carry a +// name that CRD admission would reject. The name is also used for Envoy cluster naming. +var upstreamDefinitionNameRegex = regexp.MustCompile(`^[a-zA-Z0-9\-_]+$`) + +// upstreamBasePathRegex enforces the same basePath constraint as the CRD/OpenAPI: it must start with +// "/" and must not end with "/" (root is expressed by omitting the field). basePath is prepended to +// the upstream path during routing, so a malformed value (missing leading slash, trailing slash) +// would silently produce a bad upstream request path — this rejects it at deploy time instead. +var upstreamBasePathRegex = regexp.MustCompile(`^/[a-zA-Z0-9\-._~!$&'()*+,;=:@%/]*[^/]$`) + +// validateUpstreamDefinitionsList validates an upstreamDefinitions array. fieldPrefix is the path +// to the array (e.g. "spec.upstreamDefinitions"). It is shared by the RestApi, LLM Provider, and +// MCP validators so the three kinds validate upstream definitions identically. The connect timeout +// is validated against the shared CRD duration pattern (constants.ResilienceDurationRegex) so +// gateway-side validation matches CRD admission (compound/unitless/negative values are rejected). +func validateUpstreamDefinitionsList(fieldPrefix string, definitions *[]api.UpstreamDefinition) []ValidationError { var errors []ValidationError if definitions == nil { @@ -275,38 +297,58 @@ func (v *APIValidator) validateUpstreamDefinitions(definitions *[]api.UpstreamDe namesSeen := make(map[string]bool) for i, def := range *definitions { - // Validate name + // Validate name (must match the CRD/OpenAPI constraint: 1-100 chars, ^[a-zA-Z0-9\-_]+$). if def.Name == "" { errors = append(errors, ValidationError{ - Field: fmt.Sprintf("spec.upstreamDefinitions[%d].name", i), + Field: fmt.Sprintf("%s[%d].name", fieldPrefix, i), Message: "Upstream definition name is required", }) continue } + if len(def.Name) > 100 { + errors = append(errors, ValidationError{ + Field: fmt.Sprintf("%s[%d].name", fieldPrefix, i), + Message: "Upstream definition name must be 1-100 characters", + }) + } + if !upstreamDefinitionNameRegex.MatchString(def.Name) { + errors = append(errors, ValidationError{ + Field: fmt.Sprintf("%s[%d].name", fieldPrefix, i), + Message: "Upstream definition name must match ^[a-zA-Z0-9\\-_]+$ (letters, numbers, hyphens, underscores)", + }) + } // Check for duplicate names if namesSeen[def.Name] { errors = append(errors, ValidationError{ - Field: fmt.Sprintf("spec.upstreamDefinitions[%d].name", i), + Field: fmt.Sprintf("%s[%d].name", fieldPrefix, i), Message: fmt.Sprintf("Duplicate upstream definition name '%s'", def.Name), }) continue } namesSeen[def.Name] = true + // Validate basePath (when set) against the CRD/OpenAPI pattern. Validated raw (no trim) so it + // matches CRD admission exactly; omit the field for root. + if def.BasePath != nil && !upstreamBasePathRegex.MatchString(*def.BasePath) { + errors = append(errors, ValidationError{ + Field: fmt.Sprintf("%s[%d].basePath", fieldPrefix, i), + Message: "Invalid basePath (must start with '/' and must not end with '/'; omit for root)", + }) + } + // Validate upstreams array if len(def.Upstreams) == 0 { errors = append(errors, ValidationError{ - Field: fmt.Sprintf("spec.upstreamDefinitions[%d].upstreams", i), + Field: fmt.Sprintf("%s[%d].upstreams", fieldPrefix, i), Message: "At least one upstream target is required", }) } for j, upstream := range def.Upstreams { - // Validate URL if upstream.Url == "" { errors = append(errors, ValidationError{ - Field: fmt.Sprintf("spec.upstreamDefinitions[%d].upstreams[%d].url", i, j), + Field: fmt.Sprintf("%s[%d].upstreams[%d].url", fieldPrefix, i, j), Message: "URL is required", }) continue @@ -315,20 +357,20 @@ func (v *APIValidator) validateUpstreamDefinitions(definitions *[]api.UpstreamDe parsedURL, err := url.Parse(upstream.Url) if err != nil { errors = append(errors, ValidationError{ - Field: fmt.Sprintf("spec.upstreamDefinitions[%d].upstreams[%d].url", i, j), + Field: fmt.Sprintf("%s[%d].upstreams[%d].url", fieldPrefix, i, j), Message: fmt.Sprintf("Invalid URL format: %v", err), }) } else { if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" { errors = append(errors, ValidationError{ - Field: fmt.Sprintf("spec.upstreamDefinitions[%d].upstreams[%d].url", i, j), + Field: fmt.Sprintf("%s[%d].upstreams[%d].url", fieldPrefix, i, j), Message: "URL must use http or https scheme", }) } if parsedURL.Host == "" { errors = append(errors, ValidationError{ - Field: fmt.Sprintf("spec.upstreamDefinitions[%d].upstreams[%d].url", i, j), + Field: fmt.Sprintf("%s[%d].upstreams[%d].url", fieldPrefix, i, j), Message: "URL must include a host", }) } @@ -337,7 +379,7 @@ func (v *APIValidator) validateUpstreamDefinitions(definitions *[]api.UpstreamDe // configured exclusively via upstreamDefinitions[].basePath. if parsedURL.Path != "" && parsedURL.Path != "/" { errors = append(errors, ValidationError{ - Field: fmt.Sprintf("spec.upstreamDefinitions[%d].upstreams[%d].url", i, j), + Field: fmt.Sprintf("%s[%d].upstreams[%d].url", fieldPrefix, i, j), Message: "URL must not include a path; set the base path in upstreamDefinitions[].basePath instead", }) } @@ -346,14 +388,14 @@ func (v *APIValidator) validateUpstreamDefinitions(definitions *[]api.UpstreamDe // (host[:port] only), so it would be silently dropped. Reject it. if parsedURL.RawQuery != "" || parsedURL.ForceQuery { errors = append(errors, ValidationError{ - Field: fmt.Sprintf("spec.upstreamDefinitions[%d].upstreams[%d].url", i, j), + Field: fmt.Sprintf("%s[%d].upstreams[%d].url", fieldPrefix, i, j), Message: "URL must not include a query string; only host[:port] is used", }) } if parsedURL.Fragment != "" { errors = append(errors, ValidationError{ - Field: fmt.Sprintf("spec.upstreamDefinitions[%d].upstreams[%d].url", i, j), + Field: fmt.Sprintf("%s[%d].upstreams[%d].url", fieldPrefix, i, j), Message: "URL must not include a fragment; only host[:port] is used", }) } @@ -363,23 +405,30 @@ func (v *APIValidator) validateUpstreamDefinitions(definitions *[]api.UpstreamDe if upstream.Weight != nil { if *upstream.Weight < 0 || *upstream.Weight > 100 { errors = append(errors, ValidationError{ - Field: fmt.Sprintf("spec.upstreamDefinitions[%d].upstreams[%d].weight", i, j), + Field: fmt.Sprintf("%s[%d].upstreams[%d].weight", fieldPrefix, i, j), Message: "Weight must be between 0 and 100", }) } } } - // Timeout validation is limited to connect timeout; request and idle - // timeouts are no longer supported at the upstream definition level. + // Timeout validation is limited to connect timeout. Enforce the same single-unit duration + // pattern as the CRD/OpenAPI so gateway validation cannot diverge from CRD admission, then + // a ParseDuration guard for pathological overflow — mirroring validateResilienceTimeouts. if def.Timeout != nil && def.Timeout.Connect != nil { timeoutStr := strings.TrimSpace(*def.Timeout.Connect) if timeoutStr != "" { - _, err := time.ParseDuration(timeoutStr) - if err != nil { + if !constants.ResilienceDurationRegex.MatchString(timeoutStr) { + errors = append(errors, ValidationError{ + Field: fmt.Sprintf("%s[%d].timeout.connect", fieldPrefix, i), + Message: "Invalid timeout format (expected a single-unit duration like '30s', '1m', '500ms')", + }) + } else if _, err := time.ParseDuration(timeoutStr); err != nil { + // The pattern guarantees a single-unit value; ParseDuration is a final guard + // against pathological overflow (e.g. "99999999999999999999s"). errors = append(errors, ValidationError{ - Field: fmt.Sprintf("spec.upstreamDefinitions[%d].timeout.connect", i), - Message: fmt.Sprintf("Invalid timeout format: %v (expected format: '30s', '1m', '500ms')", err), + Field: fmt.Sprintf("%s[%d].timeout.connect", fieldPrefix, i), + Message: fmt.Sprintf("Invalid timeout format: %v", err), }) } } @@ -389,6 +438,20 @@ func (v *APIValidator) validateUpstreamDefinitions(definitions *[]api.UpstreamDe return errors } +// upstreamRefResolves reports whether ref names one of the provided upstream definitions. +func upstreamRefResolves(ref string, definitions *[]api.UpstreamDefinition) bool { + if definitions == nil { + return false + } + refName := strings.TrimSpace(ref) + for _, def := range *definitions { + if def.Name == refName { + return true + } + } + return false +} + // validateRestData validates the data section of the configuration for RestApi kind func (v *APIValidator) validateRestData(spec *api.APIConfigData) []ValidationError { var errors []ValidationError @@ -436,12 +499,63 @@ func (v *APIValidator) validateRestData(spec *api.APIConfigData) []ValidationErr errors = append(errors, v.validateUpstream("sandbox", spec.Upstream.Sandbox, spec.UpstreamDefinitions)...) } + // Validate API-level resilience block + errors = append(errors, v.validateResilience("spec.resilience", spec.Resilience)...) + // Validate operations errors = append(errors, v.validateOperations(spec.Operations)...) return errors } +// validateResilience validates a resilience block (timeout / idleTimeout). Both fields +// are optional duration strings; "0s" is allowed (disables the timeout), negative and +// malformed values are rejected. fieldPrefix is the path to the block (e.g. +// "spec.resilience" or "spec.operations[2].resilience"). +func (v *APIValidator) validateResilience(fieldPrefix string, r *api.Resilience) []ValidationError { + return validateResilienceTimeouts(fieldPrefix, r) +} + +// validateResilienceTimeouts validates the timeout fields of a resilience block. +func validateResilienceTimeouts(fieldPrefix string, r *api.Resilience) []ValidationError { + var errors []ValidationError + if r == nil { + return errors + } + + validate := func(field string, value *string) { + if value == nil { + return + } + s := strings.TrimSpace(*value) + if s == "" { + return + } + // Enforce the same single-unit format as the CRD admission controller (see + // constants.ResilienceDurationPattern). This rejects compound durations ("1h30m"), + // negatives ("-30s"), and unitless values ("0", "30"), while accepting "0s" to disable. + if !constants.ResilienceDurationRegex.MatchString(s) { + errors = append(errors, ValidationError{ + Field: field, + Message: "Invalid timeout format (expected a single-unit duration like '30s', '1m', '500ms', or '0s' to disable; compound, negative, and unitless values are not allowed)", + }) + return + } + // The pattern guarantees a parseable, non-negative, single-unit value; ParseDuration is a + // final guard against pathological overflow. + if _, err := time.ParseDuration(s); err != nil { + errors = append(errors, ValidationError{ + Field: field, + Message: fmt.Sprintf("Invalid timeout format: %v", err), + }) + } + } + + validate(fieldPrefix+".timeout", r.Timeout) + validate(fieldPrefix+".idleTimeout", r.IdleTimeout) + return errors +} + // validateAsyncData validates the data section of the configuration for http/rest kind func (v *APIValidator) validateAsyncData(spec *api.WebhookAPIData) []ValidationError { var errors []ValidationError @@ -621,6 +735,9 @@ func (v *APIValidator) validateOperations(operations []api.Operation) []Validati Message: "Operation path has unbalanced braces in parameters", }) } + + // Validate operation-level resilience block + errors = append(errors, v.validateResilience(fmt.Sprintf("spec.operations[%d].resilience", i), op.Resilience)...) } return errors diff --git a/gateway/gateway-controller/pkg/config/api_validator_test.go b/gateway/gateway-controller/pkg/config/api_validator_test.go index f93a0d58d4..8745ee6cdf 100644 --- a/gateway/gateway-controller/pkg/config/api_validator_test.go +++ b/gateway/gateway-controller/pkg/config/api_validator_test.go @@ -478,6 +478,60 @@ func TestAPIValidator_ValidateOperations(t *testing.T) { } } +func TestAPIValidator_ValidateResilience(t *testing.T) { + v := NewAPIValidator() + + tests := []struct { + name string + apiRes *api.Resilience + opRes *api.Resilience + wantError bool + errField string + }{ + {name: "No resilience is valid", wantError: false}, + {name: "Valid API-level timeout + idleTimeout", apiRes: &api.Resilience{Timeout: stringPtr("15s"), IdleTimeout: stringPtr("0s")}, wantError: false}, + {name: "Valid operation-level timeout", opRes: &api.Resilience{Timeout: stringPtr("2s")}, wantError: false}, + {name: "0s is allowed (disabled)", apiRes: &api.Resilience{Timeout: stringPtr("0s"), IdleTimeout: stringPtr("0s")}, wantError: false}, + {name: "Invalid API-level timeout format", apiRes: &api.Resilience{Timeout: stringPtr("15seconds")}, wantError: true, errField: "spec.resilience.timeout"}, + {name: "Invalid API-level idleTimeout format", apiRes: &api.Resilience{IdleTimeout: stringPtr("abc")}, wantError: true, errField: "spec.resilience.idleTimeout"}, + {name: "Negative API-level timeout rejected", apiRes: &api.Resilience{Timeout: stringPtr("-5s")}, wantError: true, errField: "spec.resilience.timeout"}, + {name: "Compound API-level timeout rejected (must match CRD pattern)", apiRes: &api.Resilience{Timeout: stringPtr("1h30m")}, wantError: true, errField: "spec.resilience.timeout"}, + {name: "Compound API-level idleTimeout rejected", apiRes: &api.Resilience{IdleTimeout: stringPtr("1m30s")}, wantError: true, errField: "spec.resilience.idleTimeout"}, + {name: "Unitless API-level timeout rejected", apiRes: &api.Resilience{Timeout: stringPtr("30")}, wantError: true, errField: "spec.resilience.timeout"}, + {name: "Bare 0 rejected (unit required)", apiRes: &api.Resilience{Timeout: stringPtr("0")}, wantError: true, errField: "spec.resilience.timeout"}, + {name: "Fractional single-unit timeout allowed", apiRes: &api.Resilience{Timeout: stringPtr("1.5s")}, wantError: false}, + {name: "Invalid operation-level timeout format", opRes: &api.Resilience{Timeout: stringPtr("nope")}, wantError: true, errField: "spec.operations[0].resilience.timeout"}, + {name: "Compound operation-level timeout rejected", opRes: &api.Resilience{Timeout: stringPtr("1h30m")}, wantError: true, errField: "spec.operations[0].resilience.timeout"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := createValidRestAPIConfig() + cfg.Spec.Resilience = tt.apiRes + cfg.Spec.Operations[0].Resilience = tt.opRes + + errors := v.Validate(cfg) + hasExpectedError := false + for _, e := range errors { + if e.Field == tt.errField { + hasExpectedError = true + break + } + } + if tt.wantError && !hasExpectedError { + t.Errorf("expected error for field %s, got: %v", tt.errField, errors) + } + if !tt.wantError { + for _, e := range errors { + if strings.Contains(e.Field, "resilience") { + t.Errorf("unexpected resilience error: %v", e) + } + } + } + }) + } +} + func TestAPIValidator_ValidateAllHTTPMethods(t *testing.T) { v := NewAPIValidator() diff --git a/gateway/gateway-controller/pkg/config/config.go b/gateway/gateway-controller/pkg/config/config.go index f9e4e14319..7710ee622c 100644 --- a/gateway/gateway-controller/pkg/config/config.go +++ b/gateway/gateway-controller/pkg/config/config.go @@ -585,8 +585,17 @@ type VHostEntry struct { // HTTPListenerConfig holds HTTP listener related configuration of an API type HTTPListenerConfig struct { - ServerHeaderTransformation string `koanf:"server_header_transformation"` // Options: "APPEND_IF_ABSENT", "OVERWRITE", "PASS_THROUGH" - ServerHeaderValue string `koanf:"server_header_value"` // Custom value for the Server header + ServerHeaderTransformation string `koanf:"server_header_transformation"` // Options: "APPEND_IF_ABSENT", "OVERWRITE", "PASS_THROUGH" + ServerHeaderValue string `koanf:"server_header_value"` // Custom value for the Server header + Timeouts HCMTimeouts `koanf:"timeouts"` // HTTP Connection Manager (downstream) timeouts +} + +// HCMTimeouts holds HTTP Connection Manager (downstream/connection) timeouts. +type HCMTimeouts struct { + RequestTimeout time.Duration `koanf:"request_timeout"` // HCM request_timeout (default 0s = disabled) + RequestHeadersTimeout time.Duration `koanf:"request_headers_timeout"` // HCM request_headers_timeout (default 0s = disabled) + StreamIdleTimeout time.Duration `koanf:"stream_idle_timeout"` // HCM stream_idle_timeout (default 5m) + IdleTimeout time.Duration `koanf:"idle_timeout"` // common_http_protocol_options.idle_timeout (default 1h) } // PolicyEngineConfig holds policy engine ext_proc filter configuration @@ -976,6 +985,12 @@ func defaultConfig() *Config { HTTPListener: HTTPListenerConfig{ ServerHeaderTransformation: commonconstants.OVERWRITE, ServerHeaderValue: commonconstants.ServerName, + Timeouts: HCMTimeouts{ + RequestTimeout: 0, // 0s = disabled (Envoy default) + RequestHeadersTimeout: 0, // 0s = disabled (Envoy default) + StreamIdleTimeout: 5 * time.Minute, // Envoy default + IdleTimeout: 1 * time.Hour, // Envoy default (connection-level) + }, }, }, Analytics: AnalyticsConfig{ @@ -1676,6 +1691,26 @@ func (c *Config) validateTimeoutConfig() error { timeouts.ConnectTimeoutMs, constants.MaxReasonableTimeoutMs) } + // Validate HCM (downstream/connection) timeouts. Unlike the upstream timeouts above, + // 0 is a valid value here: which denotes "disabled scenario" + // only reject negative values and unreasonably large ones are rejected. + maxConnTimeout := time.Duration(constants.MaxReasonableConnectionTimeoutMs) * time.Millisecond + hcmTimeouts := map[string]time.Duration{ + "request_timeout": c.Router.HTTPListener.Timeouts.RequestTimeout, + "request_headers_timeout": c.Router.HTTPListener.Timeouts.RequestHeadersTimeout, + "stream_idle_timeout": c.Router.HTTPListener.Timeouts.StreamIdleTimeout, + "idle_timeout": c.Router.HTTPListener.Timeouts.IdleTimeout, + } + for name, v := range hcmTimeouts { + if v < 0 { + return fmt.Errorf("router.http_listener.timeouts.%s must not be negative, got: %s", name, v) + } + if v > maxConnTimeout { + return fmt.Errorf("router.http_listener.timeouts.%s (%s) exceeds maximum reasonable timeout of %s", + name, v, maxConnTimeout) + } + } + return nil } diff --git a/gateway/gateway-controller/pkg/config/config_test.go b/gateway/gateway-controller/pkg/config/config_test.go index c154d47649..6a23cfe474 100644 --- a/gateway/gateway-controller/pkg/config/config_test.go +++ b/gateway/gateway-controller/pkg/config/config_test.go @@ -1012,6 +1012,61 @@ func TestConfig_ValidateTimeoutConfig(t *testing.T) { } } +func TestConfig_ValidateHCMTimeouts(t *testing.T) { + maxConn := time.Duration(constants.MaxReasonableConnectionTimeoutMs) * time.Millisecond + tests := []struct { + name string + timeouts HCMTimeouts + wantErr bool + errContains string + }{ + { + name: "Envoy defaults are valid", + timeouts: HCMTimeouts{RequestTimeout: 0, RequestHeadersTimeout: 0, StreamIdleTimeout: 5 * time.Minute, IdleTimeout: time.Hour}, + }, + { + name: "All zero (disabled) is valid", + timeouts: HCMTimeouts{}, + }, + { + name: "Custom positive values are valid", + timeouts: HCMTimeouts{RequestTimeout: 30 * time.Second, RequestHeadersTimeout: 10 * time.Second, StreamIdleTimeout: time.Minute, IdleTimeout: 2 * time.Hour}, + }, + { + name: "Negative request_timeout rejected", + timeouts: HCMTimeouts{RequestTimeout: -1 * time.Second}, + wantErr: true, + errContains: "router.http_listener.timeouts.request_timeout must not be negative", + }, + { + name: "Negative stream_idle_timeout rejected", + timeouts: HCMTimeouts{StreamIdleTimeout: -5 * time.Second}, + wantErr: true, + errContains: "router.http_listener.timeouts.stream_idle_timeout must not be negative", + }, + { + name: "idle_timeout exceeding max reasonable rejected", + timeouts: HCMTimeouts{IdleTimeout: maxConn + time.Second}, + wantErr: true, + errContains: "router.http_listener.timeouts.idle_timeout", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := validConfig() + cfg.Router.HTTPListener.Timeouts = tt.timeouts + err := cfg.Validate() + if tt.wantErr { + assert.Error(t, err) + assert.Contains(t, err.Error(), tt.errContains) + } else { + assert.NoError(t, err) + } + }) + } +} + func TestConfig_ValidatePolicyEngineConfig(t *testing.T) { tests := []struct { name string @@ -1669,6 +1724,50 @@ func TestDefaultConfig(t *testing.T) { assert.Equal(t, "info", cfg.Controller.Logging.Level) assert.False(t, cfg.Controller.Server.SkipInvalidDeploymentsOnStartup) assert.Equal(t, uint32(5000), cfg.Router.Upstream.Timeouts.ConnectTimeoutMs, "default router.upstream.timeouts.connect_timeout_ms should be 5s (5000 ms)") + + // HCM timeout defaults must match Envoy's documented defaults. + hcm := cfg.Router.HTTPListener.Timeouts + assert.Equal(t, time.Duration(0), hcm.RequestTimeout, "default request_timeout should be 0s (disabled)") + assert.Equal(t, time.Duration(0), hcm.RequestHeadersTimeout, "default request_headers_timeout should be 0s (disabled)") + assert.Equal(t, 5*time.Minute, hcm.StreamIdleTimeout, "default stream_idle_timeout should be 5m") + assert.Equal(t, time.Hour, hcm.IdleTimeout, "default idle_timeout should be 1h") +} + +func TestLoadConfig_HCMTimeouts(t *testing.T) { + t.Run("explicit values parse from toml", func(t *testing.T) { + tmpDir := t.TempDir() + configPath := filepath.Join(tmpDir, "config.toml") + toml := ` +[router.http_listener.timeouts] +request_timeout = "30s" +request_headers_timeout = "10s" +stream_idle_timeout = "2m" +idle_timeout = "30m" +` + require.NoError(t, os.WriteFile(configPath, []byte(toml), 0o644)) + + cfg, err := LoadConfig(configPath) + require.NoError(t, err) + hcm := cfg.Router.HTTPListener.Timeouts + assert.Equal(t, 30*time.Second, hcm.RequestTimeout) + assert.Equal(t, 10*time.Second, hcm.RequestHeadersTimeout) + assert.Equal(t, 2*time.Minute, hcm.StreamIdleTimeout) + assert.Equal(t, 30*time.Minute, hcm.IdleTimeout) + }) + + t.Run("omitted section falls back to Envoy defaults", func(t *testing.T) { + tmpDir := t.TempDir() + configPath := filepath.Join(tmpDir, "config.toml") + require.NoError(t, os.WriteFile(configPath, []byte(""), 0o644)) + + cfg, err := LoadConfig(configPath) + require.NoError(t, err) + hcm := cfg.Router.HTTPListener.Timeouts + assert.Equal(t, time.Duration(0), hcm.RequestTimeout) + assert.Equal(t, time.Duration(0), hcm.RequestHeadersTimeout) + assert.Equal(t, 5*time.Minute, hcm.StreamIdleTimeout) + assert.Equal(t, time.Hour, hcm.IdleTimeout) + }) } func TestConfig_CaseInsensitiveAlgorithm(t *testing.T) { diff --git a/gateway/gateway-controller/pkg/config/llm_validator.go b/gateway/gateway-controller/pkg/config/llm_validator.go index 9a1ca3cf08..a33c909524 100644 --- a/gateway/gateway-controller/pkg/config/llm_validator.go +++ b/gateway/gateway-controller/pkg/config/llm_validator.go @@ -378,8 +378,10 @@ func (v *LLMValidator) validateProviderSpec(spec *api.LLMProviderConfigData) []V }) } - // Validate upstreams - errors = append(errors, v.validateUpstreamWithAuth(fmt.Sprintf("spec.upstream"), &spec.Upstream)...) + // Validate upstream definitions (name/url/connect timeout), then the upstream itself (which + // may reference one of them via `ref`). + errors = append(errors, validateUpstreamDefinitionsList("spec.upstreamDefinitions", spec.UpstreamDefinitions)...) + errors = append(errors, v.validateUpstreamWithAuth("spec.upstream", &spec.Upstream, spec.UpstreamDefinitions)...) // Validate access control errors = append(errors, v.validateAccessControl("spec.accessControl", &spec.AccessControl)...) @@ -387,6 +389,10 @@ func (v *LLMValidator) validateProviderSpec(spec *api.LLMProviderConfigData) []V // The deprecated `policies` list must not coexist with the new policy lists errors = append(errors, v.validatePolicyListExclusivity(spec.GlobalPolicies, spec.OperationPolicies, spec.Policies)...) + // Validate API-level resilience (timeout / idleTimeout). LLM kinds support resilience at + // the API level only. + errors = append(errors, validateResilienceTimeouts("spec.resilience", spec.Resilience)...) + return errors } @@ -412,9 +418,10 @@ func (v *LLMValidator) validatePolicyListExclusivity(globalPolicies *[]api.Polic return nil } -// validateUpstreamWithAuth validates an UpstreamWithAuth configuration +// validateUpstreamWithAuth validates an UpstreamWithAuth configuration. The upstream may specify +// either a direct `url` or a `ref` to one of the provided upstream definitions (exactly one). func (v *LLMValidator) validateUpstreamWithAuth(fieldPrefix string, - upstream *api.LLMProviderConfigData_Upstream) []ValidationError { + upstream *api.LLMProviderConfigData_Upstream, definitions *[]api.UpstreamDefinition) []ValidationError { var errors []ValidationError if upstream == nil { @@ -425,14 +432,23 @@ func (v *LLMValidator) validateUpstreamWithAuth(fieldPrefix string, return errors } - // Validate URL - if upstream.Url == nil || *upstream.Url == "" { + // Validate url XOR ref + hasURL := upstream.Url != nil && strings.TrimSpace(*upstream.Url) != "" + hasRef := upstream.Ref != nil && strings.TrimSpace(*upstream.Ref) != "" + switch { + case hasURL && hasRef: errors = append(errors, ValidationError{ - Field: fmt.Sprintf("%s.url", fieldPrefix), - Message: "Upstream URL is required", + Field: fieldPrefix, + Message: "Specify exactly one of 'url' or 'ref'", + }) + case !hasURL && !hasRef: + errors = append(errors, ValidationError{ + Field: fieldPrefix, + Message: "Must specify either 'url' or 'ref'", }) - } else { - parsedURL, err := url.Parse(*upstream.Url) + case hasURL: + // Trim first, consistent with the hasURL check, so surrounding whitespace does not fail parsing. + parsedURL, err := url.Parse(strings.TrimSpace(*upstream.Url)) if err != nil { errors = append(errors, ValidationError{ Field: fmt.Sprintf("%s.url", fieldPrefix), @@ -449,6 +465,13 @@ func (v *LLMValidator) validateUpstreamWithAuth(fieldPrefix string, Message: "Upstream URL must include a host", }) } + case hasRef: + if !upstreamRefResolves(*upstream.Ref, definitions) { + errors = append(errors, ValidationError{ + Field: fmt.Sprintf("%s.ref", fieldPrefix), + Message: fmt.Sprintf("Referenced upstream definition '%s' not found in upstreamDefinitions", strings.TrimSpace(*upstream.Ref)), + }) + } } // Validate auth if present @@ -594,6 +617,10 @@ func (v *LLMValidator) validateProxyData(spec *api.LLMProxyConfigData) []Validat // The deprecated `policies` list must not coexist with the new policy lists errors = append(errors, v.validatePolicyListExclusivity(spec.GlobalPolicies, spec.OperationPolicies, spec.Policies)...) + // Validate API-level resilience (timeout / idleTimeout). LLM kinds support resilience at + // the API level only. + errors = append(errors, validateResilienceTimeouts("spec.resilience", spec.Resilience)...) + return errors } diff --git a/gateway/gateway-controller/pkg/config/llm_validator_additional_test.go b/gateway/gateway-controller/pkg/config/llm_validator_additional_test.go index 00f19e97cc..bd343404b5 100644 --- a/gateway/gateway-controller/pkg/config/llm_validator_additional_test.go +++ b/gateway/gateway-controller/pkg/config/llm_validator_additional_test.go @@ -275,7 +275,7 @@ func TestLLMValidator_ValidateUpstreamWithAuth_Nil(t *testing.T) { validator := NewLLMValidator() t.Run("Nil upstream", func(t *testing.T) { - errors := validator.validateUpstreamWithAuth("test", nil) + errors := validator.validateUpstreamWithAuth("test", nil, nil) if len(errors) != 1 { t.Errorf("Expected 1 error for nil upstream, got %d", len(errors)) @@ -297,7 +297,7 @@ func TestLLMValidator_UpstreamURLValidation(t *testing.T) { Url: &malformed, } - errors := validator.validateUpstreamWithAuth("test", upstream) + errors := validator.validateUpstreamWithAuth("test", upstream, nil) found := false for _, err := range errors { if err.Field == "test.url" && err.Message != "" { @@ -317,7 +317,7 @@ func TestLLMValidator_UpstreamURLValidation(t *testing.T) { Url: &invalidScheme, } - errors := validator.validateUpstreamWithAuth("test", upstream) + errors := validator.validateUpstreamWithAuth("test", upstream, nil) found := false for _, err := range errors { if err.Field == "test.url" && err.Message == "Upstream URL must use http or https scheme" { diff --git a/gateway/gateway-controller/pkg/config/llm_validator_test.go b/gateway/gateway-controller/pkg/config/llm_validator_test.go index 21c05f4475..86b5d9f2c1 100644 --- a/gateway/gateway-controller/pkg/config/llm_validator_test.go +++ b/gateway/gateway-controller/pkg/config/llm_validator_test.go @@ -1288,8 +1288,8 @@ func TestValidateLLMProvider_Upstream(t *testing.T) { Url: nil, }, expectError: true, - errorField: "spec.upstream.url", - errorPart: "required", + errorField: "spec.upstream", + errorPart: "either 'url' or 'ref'", }, { name: "empty URL", @@ -1297,7 +1297,7 @@ func TestValidateLLMProvider_Upstream(t *testing.T) { Url: stringPtr(""), }, expectError: true, - errorField: "spec.upstream.url", + errorField: "spec.upstream", }, { name: "invalid URL - no protocol", @@ -1902,3 +1902,235 @@ func TestValidate_UnsupportedConfigType(t *testing.T) { assert.Equal(t, "config", errors[0].Field) assert.Contains(t, errors[0].Message, "Unsupported configuration type") } + +// assertHasFieldError fails unless errs contains a validation error on the given field. Shared by +// the resilience and upstream-ref tests across the LLM and MCP validators (same package). +func assertHasFieldError(t *testing.T, errs []ValidationError, field string) { + t.Helper() + for _, e := range errs { + if e.Field == field { + return + } + } + t.Fatalf("expected a validation error on field %q, got %+v", field, errs) +} + +// upstreamDef builds a valid upstream definition (one host-only target) with an optional connect +// timeout. connect == "" leaves the timeout unset. Shared by the LLM and MCP upstream-ref tests. +func upstreamDef(name, connect string) api.UpstreamDefinition { + def := api.UpstreamDefinition{ + Name: name, + Upstreams: []struct { + Url string `json:"url" yaml:"url"` + Weight *int `json:"weight,omitempty" yaml:"weight,omitempty"` + }{ + {Url: "http://backend:8080"}, + }, + } + if connect != "" { + def.Timeout = &api.UpstreamTimeout{Connect: stringPtr(connect)} + } + return def +} + +// ============================================================================ +// Resilience validation +// ============================================================================ + +func validProviderWithResilience(r *api.Resilience) api.LLMProviderConfiguration { + return api.LLMProviderConfiguration{ + ApiVersion: "gateway.api-platform.wso2.com/v1", + Kind: api.LLMProviderConfigurationKindLlmProvider, + Metadata: api.Metadata{Name: "openai"}, + Spec: api.LLMProviderConfigData{ + DisplayName: "my-provider", + Version: "v1.0", + Template: "openai", + Upstream: api.LLMProviderConfigData_Upstream{Url: stringPtr("https://api.openai.com")}, + AccessControl: api.LLMAccessControl{Mode: api.AllowAll}, + Resilience: r, + }, + } +} + +func validProxyWithResilience(r *api.Resilience) api.LLMProxyConfiguration { + return api.LLMProxyConfiguration{ + ApiVersion: api.LLMProxyConfigurationApiVersionGatewayApiPlatformWso2Comv1, + Kind: api.LLMProxyConfigurationKindLlmProxy, + Metadata: api.Metadata{Name: "openai-proxy"}, + Spec: api.LLMProxyConfigData{ + DisplayName: "my-proxy", + Version: "v1.0", + Provider: api.LLMProxyProvider{Id: "openai"}, + Resilience: r, + }, + } +} + +func TestValidateLLMProvider_Resilience(t *testing.T) { + validator := NewLLMValidator() + + t.Run("valid timeout and idleTimeout", func(t *testing.T) { + errs := validator.Validate(validProviderWithResilience(&api.Resilience{ + Timeout: stringPtr("30s"), + IdleTimeout: stringPtr("0s"), + })) + assert.Empty(t, errs) + }) + + t.Run("nil resilience is fine", func(t *testing.T) { + errs := validator.Validate(validProviderWithResilience(nil)) + assert.Empty(t, errs) + }) + + t.Run("malformed timeout is rejected", func(t *testing.T) { + errs := validator.Validate(validProviderWithResilience(&api.Resilience{Timeout: stringPtr("30")})) + assertHasFieldError(t, errs, "spec.resilience.timeout") + }) + + t.Run("compound timeout is rejected (must match CRD pattern)", func(t *testing.T) { + errs := validator.Validate(validProviderWithResilience(&api.Resilience{Timeout: stringPtr("1h30m")})) + assertHasFieldError(t, errs, "spec.resilience.timeout") + }) + + t.Run("0s is accepted (disables)", func(t *testing.T) { + errs := validator.Validate(validProviderWithResilience(&api.Resilience{Timeout: stringPtr("0s")})) + assert.Empty(t, errs) + }) + + t.Run("negative timeout is rejected", func(t *testing.T) { + errs := validator.Validate(validProviderWithResilience(&api.Resilience{Timeout: stringPtr("-5s")})) + assertHasFieldError(t, errs, "spec.resilience.timeout") + }) + + t.Run("malformed idleTimeout is rejected", func(t *testing.T) { + errs := validator.Validate(validProviderWithResilience(&api.Resilience{IdleTimeout: stringPtr("abc")})) + assertHasFieldError(t, errs, "spec.resilience.idleTimeout") + }) +} + +func TestValidateLLMProxy_Resilience(t *testing.T) { + validator := NewLLMValidator() + + t.Run("valid timeout", func(t *testing.T) { + errs := validator.Validate(validProxyWithResilience(&api.Resilience{Timeout: stringPtr("75s")})) + assert.Empty(t, errs) + }) + + t.Run("nil resilience is fine", func(t *testing.T) { + errs := validator.Validate(validProxyWithResilience(nil)) + assert.Empty(t, errs) + }) + + t.Run("malformed timeout is rejected", func(t *testing.T) { + errs := validator.Validate(validProxyWithResilience(&api.Resilience{Timeout: stringPtr("fast")})) + assertHasFieldError(t, errs, "spec.resilience.timeout") + }) + + t.Run("negative idleTimeout is rejected", func(t *testing.T) { + errs := validator.Validate(validProxyWithResilience(&api.Resilience{IdleTimeout: stringPtr("-1s")})) + assertHasFieldError(t, errs, "spec.resilience.idleTimeout") + }) +} + +// ============================================================================ +// Upstream ref validation +// ============================================================================ + +func providerWithUpstream(defs *[]api.UpstreamDefinition, up api.LLMProviderConfigData_Upstream) api.LLMProviderConfiguration { + return api.LLMProviderConfiguration{ + ApiVersion: "gateway.api-platform.wso2.com/v1", + Kind: api.LLMProviderConfigurationKindLlmProvider, + Metadata: api.Metadata{Name: "openai"}, + Spec: api.LLMProviderConfigData{ + DisplayName: "my-provider", + Version: "v1.0", + Template: "openai", + UpstreamDefinitions: defs, + Upstream: up, + AccessControl: api.LLMAccessControl{Mode: api.AllowAll}, + }, + } +} + +func TestValidateLLMProvider_UpstreamRef(t *testing.T) { + validator := NewLLMValidator() + + t.Run("valid ref resolves to a definition", func(t *testing.T) { + defs := &[]api.UpstreamDefinition{upstreamDef("openai-backend", "6s")} + errs := validator.Validate(providerWithUpstream(defs, api.LLMProviderConfigData_Upstream{Ref: stringPtr("openai-backend")})) + assert.Empty(t, errs) + }) + + t.Run("ref not found in definitions", func(t *testing.T) { + defs := &[]api.UpstreamDefinition{upstreamDef("other", "6s")} + errs := validator.Validate(providerWithUpstream(defs, api.LLMProviderConfigData_Upstream{Ref: stringPtr("missing")})) + assertHasFieldError(t, errs, "spec.upstream.ref") + }) + + t.Run("both url and ref rejected", func(t *testing.T) { + defs := &[]api.UpstreamDefinition{upstreamDef("openai-backend", "6s")} + errs := validator.Validate(providerWithUpstream(defs, api.LLMProviderConfigData_Upstream{ + Url: stringPtr("https://api.openai.com"), + Ref: stringPtr("openai-backend"), + })) + assertHasFieldError(t, errs, "spec.upstream") + }) + + t.Run("malformed connect timeout rejected (must match CRD pattern)", func(t *testing.T) { + defs := &[]api.UpstreamDefinition{upstreamDef("openai-backend", "1h30m")} + errs := validator.Validate(providerWithUpstream(defs, api.LLMProviderConfigData_Upstream{Ref: stringPtr("openai-backend")})) + assertHasFieldError(t, errs, "spec.upstreamDefinitions[0].timeout.connect") + }) + + t.Run("valid fractional connect timeout accepted", func(t *testing.T) { + defs := &[]api.UpstreamDefinition{upstreamDef("openai-backend", "500ms")} + errs := validator.Validate(providerWithUpstream(defs, api.LLMProviderConfigData_Upstream{Ref: stringPtr("openai-backend")})) + assert.Empty(t, errs) + }) + + t.Run("valid basePath accepted", func(t *testing.T) { + def := upstreamDef("openai-backend", "6s") + def.BasePath = stringPtr("/api/v2") + errs := validator.Validate(providerWithUpstream(&[]api.UpstreamDefinition{def}, api.LLMProviderConfigData_Upstream{Ref: stringPtr("openai-backend")})) + assert.Empty(t, errs) + }) + + t.Run("basePath without leading slash rejected", func(t *testing.T) { + def := upstreamDef("openai-backend", "6s") + def.BasePath = stringPtr("api/v2") + errs := validator.Validate(providerWithUpstream(&[]api.UpstreamDefinition{def}, api.LLMProviderConfigData_Upstream{Ref: stringPtr("openai-backend")})) + assertHasFieldError(t, errs, "spec.upstreamDefinitions[0].basePath") + }) + + t.Run("basePath with trailing slash rejected", func(t *testing.T) { + def := upstreamDef("openai-backend", "6s") + def.BasePath = stringPtr("/api/v2/") + errs := validator.Validate(providerWithUpstream(&[]api.UpstreamDefinition{def}, api.LLMProviderConfigData_Upstream{Ref: stringPtr("openai-backend")})) + assertHasFieldError(t, errs, "spec.upstreamDefinitions[0].basePath") + }) + + t.Run("connect timeout that overflows time.Duration rejected", func(t *testing.T) { + defs := &[]api.UpstreamDefinition{upstreamDef("openai-backend", "99999999999999999999s")} + errs := validator.Validate(providerWithUpstream(defs, api.LLMProviderConfigData_Upstream{Ref: stringPtr("openai-backend")})) + assertHasFieldError(t, errs, "spec.upstreamDefinitions[0].timeout.connect") + }) + + t.Run("definition name with invalid characters rejected (CRD pattern)", func(t *testing.T) { + defs := &[]api.UpstreamDefinition{upstreamDef("bad name!", "6s")} + errs := validator.Validate(providerWithUpstream(defs, api.LLMProviderConfigData_Upstream{Ref: stringPtr("bad name!")})) + assertHasFieldError(t, errs, "spec.upstreamDefinitions[0].name") + }) + + t.Run("definition name over 100 chars rejected", func(t *testing.T) { + long := strings.Repeat("a", 101) + defs := &[]api.UpstreamDefinition{upstreamDef(long, "6s")} + errs := validator.Validate(providerWithUpstream(defs, api.LLMProviderConfigData_Upstream{Ref: stringPtr(long)})) + assertHasFieldError(t, errs, "spec.upstreamDefinitions[0].name") + }) + + t.Run("url with surrounding whitespace is accepted", func(t *testing.T) { + errs := validator.Validate(providerWithUpstream(nil, api.LLMProviderConfigData_Upstream{Url: stringPtr(" https://api.openai.com ")})) + assert.Empty(t, errs) + }) +} diff --git a/gateway/gateway-controller/pkg/config/mcp_validator.go b/gateway/gateway-controller/pkg/config/mcp_validator.go index 51b47d49fb..2cc5ad24ee 100644 --- a/gateway/gateway-controller/pkg/config/mcp_validator.go +++ b/gateway/gateway-controller/pkg/config/mcp_validator.go @@ -149,8 +149,14 @@ func (v *MCPValidator) validateSpec(spec *api.MCPProxyConfigData) []ValidationEr // Validate context errors = append(errors, v.validateContextAndVhost(spec.Context, spec.Vhost)...) - // Validate upstream - errors = append(errors, v.validateUpstream("spec.upstream", &spec.Upstream)...) + // Validate upstream definitions (name/url/connect timeout), then the upstream itself (which + // may reference one of them via `ref`). + errors = append(errors, validateUpstreamDefinitionsList("spec.upstreamDefinitions", spec.UpstreamDefinitions)...) + errors = append(errors, v.validateUpstream("spec.upstream", &spec.Upstream, spec.UpstreamDefinitions)...) + + // Validate API-level resilience (timeout / idleTimeout). MCP supports resilience at the API + // level only; the route timeout defaults to disabled for MCP when unset (see mcp-timeout-divergence.md). + errors = append(errors, validateResilienceTimeouts("spec.resilience", spec.Resilience)...) return errors } @@ -192,8 +198,9 @@ func (v *MCPValidator) validateContextAndVhost(context, vhost *string) []Validat return errors } -// validateUpstream validates the upstream configuration -func (v *MCPValidator) validateUpstream(fieldPrefix string, upstream *api.MCPProxyConfigData_Upstream) []ValidationError { +// validateUpstream validates the upstream configuration. The upstream may specify either a direct +// `url` or a `ref` to one of the provided upstream definitions (exactly one). +func (v *MCPValidator) validateUpstream(fieldPrefix string, upstream *api.MCPProxyConfigData_Upstream, definitions *[]api.UpstreamDefinition) []ValidationError { var errors []ValidationError if upstream == nil { @@ -204,38 +211,53 @@ func (v *MCPValidator) validateUpstream(fieldPrefix string, upstream *api.MCPPro return errors } - if upstream.Url == nil || *upstream.Url == "" { + // Validate url XOR ref + hasURL := upstream.Url != nil && strings.TrimSpace(*upstream.Url) != "" + hasRef := upstream.Ref != nil && strings.TrimSpace(*upstream.Ref) != "" + switch { + case hasURL && hasRef: errors = append(errors, ValidationError{ - Field: fmt.Sprintf("%s.url", fieldPrefix), - Message: "Upstream URL is required", + Field: fieldPrefix, + Message: "Specify exactly one of 'url' or 'ref'", }) - return errors - } - - // Validate URL format - parsedURL, err := url.Parse(*upstream.Url) - if err != nil { + case !hasURL && !hasRef: errors = append(errors, ValidationError{ - Field: fmt.Sprintf("%s.url", fieldPrefix), - Message: fmt.Sprintf("Invalid URL format: %v", err), - }) - return errors - } - - // Ensure scheme is http or https - if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" { - errors = append(errors, ValidationError{ - Field: fmt.Sprintf("%s.url", fieldPrefix), - Message: "Upstream URL must use http or https scheme", + Field: fieldPrefix, + Message: "Must specify either 'url' or 'ref'", }) - } + case hasRef: + if !upstreamRefResolves(*upstream.Ref, definitions) { + errors = append(errors, ValidationError{ + Field: fmt.Sprintf("%s.ref", fieldPrefix), + Message: fmt.Sprintf("Referenced upstream definition '%s' not found in upstreamDefinitions", strings.TrimSpace(*upstream.Ref)), + }) + } + case hasURL: + // Validate URL format (trim first, consistent with the hasURL check so surrounding + // whitespace does not fail parsing). + parsedURL, err := url.Parse(strings.TrimSpace(*upstream.Url)) + if err != nil { + errors = append(errors, ValidationError{ + Field: fmt.Sprintf("%s.url", fieldPrefix), + Message: fmt.Sprintf("Invalid URL format: %v", err), + }) + } else { + // Ensure scheme is http or https + if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" { + errors = append(errors, ValidationError{ + Field: fmt.Sprintf("%s.url", fieldPrefix), + Message: "Upstream URL must use http or https scheme", + }) + } - // Ensure host is present - if parsedURL.Host == "" { - errors = append(errors, ValidationError{ - Field: fmt.Sprintf("%s.url", fieldPrefix), - Message: "Upstream URL must include a host", - }) + // Ensure host is present + if parsedURL.Host == "" { + errors = append(errors, ValidationError{ + Field: fmt.Sprintf("%s.url", fieldPrefix), + Message: "Upstream URL must include a host", + }) + } + } } // Validate auth if present diff --git a/gateway/gateway-controller/pkg/config/mcp_validator_test.go b/gateway/gateway-controller/pkg/config/mcp_validator_test.go index 107d14ee78..7a5b31c6fb 100644 --- a/gateway/gateway-controller/pkg/config/mcp_validator_test.go +++ b/gateway/gateway-controller/pkg/config/mcp_validator_test.go @@ -22,6 +22,7 @@ import ( "strings" "testing" + "github.com/stretchr/testify/assert" api "github.com/wso2/api-platform/gateway/gateway-controller/pkg/api/management" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/constants" ) @@ -472,13 +473,13 @@ func TestMCPValidator_ValidateUpstream(t *testing.T) { name: "Nil URL", upstream: &api.MCPProxyConfigData_Upstream{Url: nil}, wantError: true, - errField: "spec.upstream.url", + errField: "spec.upstream", }, { name: "Empty URL", upstream: &api.MCPProxyConfigData_Upstream{Url: stringPtr("")}, wantError: true, - errField: "spec.upstream.url", + errField: "spec.upstream", }, { name: "Invalid URL scheme", @@ -656,3 +657,121 @@ func TestMCPValidator_ValidateUpstreamAuth(t *testing.T) { }) } } + +// ============================================================================ +// Upstream ref validation +// ============================================================================ + +func mcpWithUpstream(defs *[]api.UpstreamDefinition, up api.MCPProxyConfigData_Upstream) api.MCPProxyConfiguration { + ctx := "/everything" + specVersion := constants.SPEC_VERSION_2025_JUNE + return api.MCPProxyConfiguration{ + ApiVersion: api.MCPProxyConfigurationApiVersionGatewayApiPlatformWso2Comv1, + Kind: api.MCPProxyConfigurationKindMcp, + Metadata: api.Metadata{Name: "everything"}, + Spec: api.MCPProxyConfigData{ + DisplayName: "Everything", + Version: "v1.0", + Context: &ctx, + SpecVersion: &specVersion, + UpstreamDefinitions: defs, + Upstream: up, + }, + } +} + +func TestValidateMCP_UpstreamRef(t *testing.T) { + validator := NewMCPValidator() + + t.Run("valid ref resolves to a definition", func(t *testing.T) { + defs := &[]api.UpstreamDefinition{upstreamDef("mcp-backend", "6s")} + errs := validator.Validate(mcpWithUpstream(defs, api.MCPProxyConfigData_Upstream{Ref: stringPtr("mcp-backend")})) + assert.Empty(t, errs) + }) + + t.Run("ref not found in definitions", func(t *testing.T) { + defs := &[]api.UpstreamDefinition{upstreamDef("other", "6s")} + errs := validator.Validate(mcpWithUpstream(defs, api.MCPProxyConfigData_Upstream{Ref: stringPtr("missing")})) + assertHasFieldError(t, errs, "spec.upstream.ref") + }) + + t.Run("both url and ref rejected", func(t *testing.T) { + defs := &[]api.UpstreamDefinition{upstreamDef("mcp-backend", "6s")} + errs := validator.Validate(mcpWithUpstream(defs, api.MCPProxyConfigData_Upstream{ + Url: stringPtr("http://backend:8080"), + Ref: stringPtr("mcp-backend"), + })) + assertHasFieldError(t, errs, "spec.upstream") + }) + + t.Run("malformed connect timeout rejected (must match CRD pattern)", func(t *testing.T) { + defs := &[]api.UpstreamDefinition{upstreamDef("mcp-backend", "1h30m")} + errs := validator.Validate(mcpWithUpstream(defs, api.MCPProxyConfigData_Upstream{Ref: stringPtr("mcp-backend")})) + assertHasFieldError(t, errs, "spec.upstreamDefinitions[0].timeout.connect") + }) + + t.Run("valid connect timeout accepted", func(t *testing.T) { + defs := &[]api.UpstreamDefinition{upstreamDef("mcp-backend", "5s")} + errs := validator.Validate(mcpWithUpstream(defs, api.MCPProxyConfigData_Upstream{Ref: stringPtr("mcp-backend")})) + assert.Empty(t, errs) + }) +} + +// ============================================================================ +// Resilience validation +// ============================================================================ + +func mcpWithResilience(r *api.Resilience) api.MCPProxyConfiguration { + ctx := "/everything" + specVersion := constants.SPEC_VERSION_2025_JUNE + return api.MCPProxyConfiguration{ + ApiVersion: api.MCPProxyConfigurationApiVersionGatewayApiPlatformWso2Comv1, + Kind: api.MCPProxyConfigurationKindMcp, + Metadata: api.Metadata{Name: "everything"}, + Spec: api.MCPProxyConfigData{ + DisplayName: "Everything", + Version: "v1.0", + Context: &ctx, + SpecVersion: &specVersion, + Upstream: api.MCPProxyConfigData_Upstream{Url: stringPtr("http://backend:3001")}, + Resilience: r, + }, + } +} + +func TestValidateMCP_Resilience(t *testing.T) { + validator := NewMCPValidator() + + t.Run("valid timeout and idleTimeout", func(t *testing.T) { + errs := validator.Validate(mcpWithResilience(&api.Resilience{ + Timeout: stringPtr("30s"), + IdleTimeout: stringPtr("0s"), + })) + assert.Empty(t, errs) + }) + + t.Run("nil resilience is fine (defaults applied downstream)", func(t *testing.T) { + errs := validator.Validate(mcpWithResilience(nil)) + assert.Empty(t, errs) + }) + + t.Run("0s is accepted (explicit disable)", func(t *testing.T) { + errs := validator.Validate(mcpWithResilience(&api.Resilience{Timeout: stringPtr("0s")})) + assert.Empty(t, errs) + }) + + t.Run("malformed timeout is rejected", func(t *testing.T) { + errs := validator.Validate(mcpWithResilience(&api.Resilience{Timeout: stringPtr("30")})) + assertHasFieldError(t, errs, "spec.resilience.timeout") + }) + + t.Run("compound timeout is rejected (must match CRD pattern)", func(t *testing.T) { + errs := validator.Validate(mcpWithResilience(&api.Resilience{Timeout: stringPtr("1h30m")})) + assertHasFieldError(t, errs, "spec.resilience.timeout") + }) + + t.Run("negative idleTimeout is rejected", func(t *testing.T) { + errs := validator.Validate(mcpWithResilience(&api.Resilience{IdleTimeout: stringPtr("-5s")})) + assertHasFieldError(t, errs, "spec.resilience.idleTimeout") + }) +} diff --git a/gateway/gateway-controller/pkg/constants/constants.go b/gateway/gateway-controller/pkg/constants/constants.go index b0741044af..4833f0502d 100644 --- a/gateway/gateway-controller/pkg/constants/constants.go +++ b/gateway/gateway-controller/pkg/constants/constants.go @@ -71,6 +71,10 @@ const ( // Configuration Validation Constants MaxReasonableTimeoutMs = uint32(3600000) // 1 hour in milliseconds MaxReasonablePolicyTimeoutMs = uint32(60000) // 60 seconds in milliseconds + + // MaxReasonableConnectionTimeoutMs caps connection-level timeouts (request, request-headers,etc.), + // allowing higher values than MaxReasonableTimeoutMs to support long-lived idle connections. + MaxReasonableConnectionTimeoutMs = uint32(86400000) // 24 hours in milliseconds // Cipher Suite Validation CipherInvalidChars1 = ";" @@ -175,6 +179,12 @@ const ( // System policy constants ANALYTICS_SYSTEM_POLICY_NAME = "wso2_apip_sys_analytics" ANALYTICS_SYSTEM_POLICY_VERSION = "v1" + + // ResilienceDurationPattern is the single source of truth for the format of resilience + // timeout strings. + // - accepted: "30s", "500ms", "1m", "2h", "1.5s", and "0s" (zero disables the timeout) + // - rejected: compound durations ("1h30m"), negatives ("-30s"), and unitless values ("0", "30") + ResilienceDurationPattern = `^\d+(\.\d+)?(ms|s|m|h)$` ) // DP->CP artifact push timing. The bottom-up (DP->CP) push waits for the local deployment @@ -195,3 +205,8 @@ var WILDCARD_HTTP_METHODS = []string{ "DELETE", "OPTIONS", } + +// ResilienceDurationRegex is the compiled ResilienceDurationPattern, shared by the management-API +// validator and the xDS downstream parser so both enforce exactly what the +// CRD admission controller enforces. +var ResilienceDurationRegex = regexp.MustCompile(ResilienceDurationPattern) \ No newline at end of file diff --git a/gateway/gateway-controller/pkg/models/runtime_deploy_config.go b/gateway/gateway-controller/pkg/models/runtime_deploy_config.go index 8dec6fa8c7..e5111228ec 100644 --- a/gateway/gateway-controller/pkg/models/runtime_deploy_config.go +++ b/gateway/gateway-controller/pkg/models/runtime_deploy_config.go @@ -62,8 +62,13 @@ type Route struct { } // RouteTimeout holds parsed timeout values for a route. +// Timeout and IdleTimeout come from the resilience block (operation-level overriding +// API-level). A nil field means "not configured" — the global route timeout default +// applies. A non-nil zero value means "explicitly disabled". type RouteTimeout struct { - Connect *time.Duration + Connect *time.Duration + Timeout *time.Duration // route timeout -> RouteAction.Timeout + IdleTimeout *time.Duration // route idle timeout -> RouteAction.IdleTimeout } // RouteUpstream links a route to its upstream cluster. diff --git a/gateway/gateway-controller/pkg/transform/restapi.go b/gateway/gateway-controller/pkg/transform/restapi.go index 8d404dca3d..053fdb3223 100644 --- a/gateway/gateway-controller/pkg/transform/restapi.go +++ b/gateway/gateway-controller/pkg/transform/restapi.go @@ -24,6 +24,7 @@ import ( "net/url" "strconv" "strings" + "time" commonconstants "github.com/wso2/api-platform/common/constants" versionutil "github.com/wso2/api-platform/common/version" @@ -151,8 +152,22 @@ func (t *RestAPITransformer) Transform(cfg *models.StoredConfig) (*models.Runtim return nil, fmt.Errorf("sandbox upstream is configured but resolves to the same vhost %q as the main upstream; configure distinct vhosts to avoid route conflicts", effectiveMainVHost) } + // Resolve API-level resilience timeouts once; operation-level values override these. + apiTimeout, apiIdleTimeout, err := xds.ResolveResilience(apiData.Resilience) + if err != nil { + return nil, fmt.Errorf("invalid API-level resilience: %w", err) + } + // Build routes and policy chains for each operation for _, op := range apiData.Operations { + // Operation-level resilience overrides API-level (per field); nil leaves the + // global route timeout default in effect. + opTimeout, opIdleTimeout, err := xds.ResolveResilience(op.Resilience) + if err != nil { + return nil, fmt.Errorf("invalid resilience for operation %s %s: %w", op.Method, op.Path, err) + } + routeTimeout := buildRouteTimeout(opTimeout, apiTimeout, opIdleTimeout, apiIdleTimeout) + vhosts := []string{effectiveMainVHost} if hasSandbox { vhosts = append(vhosts, effectiveSandboxVHost) @@ -168,6 +183,7 @@ func (t *RestAPITransformer) Transform(cfg *models.StoredConfig) (*models.Runtim OperationPath: op.Path, Vhost: vhost, AutoHostRewrite: mainAutoHostRewrite, + Timeout: routeTimeout, Upstream: models.RouteUpstream{ ClusterKey: mainUpstream.ClusterKey, UseClusterHeader: useClusterHeader, @@ -246,6 +262,24 @@ func (t *RestAPITransformer) Transform(cfg *models.StoredConfig) (*models.Runtim } // collectAPIPolicies validates and collects API-level policies into SDK format. +// buildRouteTimeout applies operation-over-API precedence (per field) and returns a +// *models.RouteTimeout, or nil when neither level configured any timeout (so the global +// route timeout default applies). +func buildRouteTimeout(opTimeout, apiTimeout, opIdle, apiIdle *time.Duration) *models.RouteTimeout { + timeout := opTimeout + if timeout == nil { + timeout = apiTimeout + } + idle := opIdle + if idle == nil { + idle = apiIdle + } + if timeout == nil && idle == nil { + return nil + } + return &models.RouteTimeout{Timeout: timeout, IdleTimeout: idle} +} + func (t *RestAPITransformer) collectAPIPolicies(policies *[]api.Policy) map[string]policyenginev1.PolicyInstance { result := make(map[string]policyenginev1.PolicyInstance) if policies == nil { diff --git a/gateway/gateway-controller/pkg/transform/restapi_test.go b/gateway/gateway-controller/pkg/transform/restapi_test.go index 5edf451df9..2f5835715c 100644 --- a/gateway/gateway-controller/pkg/transform/restapi_test.go +++ b/gateway/gateway-controller/pkg/transform/restapi_test.go @@ -21,6 +21,7 @@ package transform import ( "net/url" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -84,6 +85,79 @@ func makeRestAPIStoredConfig(apiPolicies []api.Policy, opPolicies []api.Policy) } } +// makeRestAPIStoredConfigWithResilience builds a RestAPI StoredConfig whose single +// operation (GET /hello) carries optional API-level and operation-level resilience blocks. +func makeRestAPIStoredConfigWithResilience(apiRes, opRes *api.Resilience) *models.StoredConfig { + apiData := api.APIConfigData{ + DisplayName: "Test API", + Context: "/test", + Version: "1.0.0", + Resilience: apiRes, + Operations: []api.Operation{ + {Method: "GET", Path: "/hello", Resilience: opRes}, + }, + Upstream: struct { + Main api.Upstream `json:"main" yaml:"main"` + Sandbox *api.Upstream `json:"sandbox,omitempty" yaml:"sandbox,omitempty"` + }{ + Main: api.Upstream{Url: ptrStr("http://backend:8080")}, + }, + } + return &models.StoredConfig{ + UUID: "test-api", + Kind: string(api.RestAPIKindRestApi), + Configuration: api.RestAPI{Kind: api.RestAPIKindRestApi, Metadata: api.Metadata{Name: "test-api"}, Spec: apiData}, + } +} + +func TestRestAPITransformer_ResiliencePrecedence(t *testing.T) { + const routeKey = "GET|/test/hello|main.local" + transformer := NewRestAPITransformer(testRouterCfg(), &config.Config{}, map[string]models.PolicyDefinition{}) + + t.Run("operation-level overrides API-level", func(t *testing.T) { + cfg := makeRestAPIStoredConfigWithResilience( + &api.Resilience{Timeout: ptrStr("10s"), IdleTimeout: ptrStr("30s")}, + &api.Resilience{Timeout: ptrStr("2s")}, + ) + rdc, err := transformer.Transform(cfg) + require.NoError(t, err) + rt := rdc.Routes[routeKey].Timeout + require.NotNil(t, rt) + require.NotNil(t, rt.Timeout) + assert.Equal(t, 2*time.Second, *rt.Timeout, "operation timeout should win") + require.NotNil(t, rt.IdleTimeout) + assert.Equal(t, 30*time.Second, *rt.IdleTimeout, "idleTimeout falls back to API-level when op omits it") + }) + + t.Run("API-level applies when operation omits resilience", func(t *testing.T) { + cfg := makeRestAPIStoredConfigWithResilience(&api.Resilience{Timeout: ptrStr("7s")}, nil) + rdc, err := transformer.Transform(cfg) + require.NoError(t, err) + rt := rdc.Routes[routeKey].Timeout + require.NotNil(t, rt) + require.NotNil(t, rt.Timeout) + assert.Equal(t, 7*time.Second, *rt.Timeout) + assert.Nil(t, rt.IdleTimeout) + }) + + t.Run("no resilience leaves Timeout nil (global default applies)", func(t *testing.T) { + cfg := makeRestAPIStoredConfigWithResilience(nil, nil) + rdc, err := transformer.Transform(cfg) + require.NoError(t, err) + assert.Nil(t, rdc.Routes[routeKey].Timeout) + }) + + t.Run("0s is preserved as explicit disable", func(t *testing.T) { + cfg := makeRestAPIStoredConfigWithResilience(&api.Resilience{Timeout: ptrStr("0s")}, nil) + rdc, err := transformer.Transform(cfg) + require.NoError(t, err) + rt := rdc.Routes[routeKey].Timeout + require.NotNil(t, rt) + require.NotNil(t, rt.Timeout) + assert.Equal(t, time.Duration(0), *rt.Timeout) + }) +} + // findPolicyInChain returns true if any policy chain for the given route key // contains a policy with the given name. func findPolicyInChain(rdc *models.RuntimeDeployConfig, routeKey, policyName string) bool { diff --git a/gateway/gateway-controller/pkg/utils/llm_provider_transformer_test.go b/gateway/gateway-controller/pkg/utils/llm_provider_transformer_test.go index f5ed2e5706..6bbd5037f6 100644 --- a/gateway/gateway-controller/pkg/utils/llm_provider_transformer_test.go +++ b/gateway/gateway-controller/pkg/utils/llm_provider_transformer_test.go @@ -19,7 +19,10 @@ package utils import ( + "io" + "log/slog" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -8220,3 +8223,276 @@ func findOperation(ops []api.Operation, path string, method string) *api.Operati } return nil } + +// ============================================================================ +// Resilience route-mapping tests +// ============================================================================ + +// These tests pin the LLM resilience route-mapping rule: API-level resilience is attached to every +// traffic-forwarding route and never to the access-control deny routes. + +// In allow_all mode, the catch-all (and any policy-derived routes) carry the resilience block while +// the exception/deny routes (which only return a 404) are left untouched. +func TestTransformProvider_AllowAll_ResilienceSkipsDenyRoutes(t *testing.T) { + transformer, _ := setupTestTransformer(t) + + exceptions := []api.RouteException{ + {Path: "/admin", Methods: []api.RouteExceptionMethods{api.RouteExceptionMethodsGET, api.RouteExceptionMethodsPOST}}, + } + timeout := "30s" + provider := &api.LLMProviderConfiguration{ + ApiVersion: "gateway.api-platform.wso2.com/v1", + Kind: "LlmProvider", + Metadata: api.Metadata{Name: "openai-provider"}, + Spec: api.LLMProviderConfigData{ + DisplayName: "test", + Version: "v1.0", + Template: "openai", + Upstream: api.LLMProviderConfigData_Upstream{Url: stringPtr("https://api.example.com")}, + AccessControl: api.LLMAccessControl{ + Mode: api.AllowAll, + Exceptions: &exceptions, + }, + Resilience: &api.Resilience{Timeout: &timeout}, + }, + } + + result, err := transformer.Transform(provider, &api.RestAPI{}) + require.NoError(t, err) + + sawCatchAll := false + sawDeny := false + for _, op := range result.Spec.Operations { + if op.Path == "/admin" { // the deny/exception routes + sawDeny = true + assert.Nil(t, op.Resilience, "deny route %s %s must not carry resilience", op.Method, op.Path) + continue + } + // catch-all (traffic-forwarding) routes + sawCatchAll = true + require.NotNil(t, op.Resilience, "traffic route %s %s should carry resilience", op.Method, op.Path) + require.NotNil(t, op.Resilience.Timeout) + assert.Equal(t, "30s", *op.Resilience.Timeout) + } + assert.True(t, sawCatchAll, "expected at least one traffic-forwarding route") + assert.True(t, sawDeny, "expected at least one deny route") +} + +// In deny_all mode every created route is an allow-listed forwarding route, so all of them carry the +// resilience block. +func TestTransformProvider_DenyAll_ResilienceOnAllRoutes(t *testing.T) { + transformer, _ := setupTestTransformer(t) + + exceptions := []api.RouteException{ + {Path: "/chat/completions", Methods: []api.RouteExceptionMethods{api.RouteExceptionMethodsPOST}}, + {Path: "/models", Methods: []api.RouteExceptionMethods{api.RouteExceptionMethodsGET}}, + } + timeout := "45s" + idle := "0s" + provider := &api.LLMProviderConfiguration{ + ApiVersion: "gateway.api-platform.wso2.com/v1", + Kind: "LlmProvider", + Metadata: api.Metadata{Name: "openai-provider"}, + Spec: api.LLMProviderConfigData{ + DisplayName: "test", + Version: "v1.0", + Template: "openai", + Upstream: api.LLMProviderConfigData_Upstream{Url: stringPtr("https://api.example.com")}, + AccessControl: api.LLMAccessControl{ + Mode: api.DenyAll, + Exceptions: &exceptions, + }, + Resilience: &api.Resilience{Timeout: &timeout, IdleTimeout: &idle}, + }, + } + + result, err := transformer.Transform(provider, &api.RestAPI{}) + require.NoError(t, err) + require.NotEmpty(t, result.Spec.Operations) + + for _, op := range result.Spec.Operations { + require.NotNil(t, op.Resilience, "route %s %s should carry resilience", op.Method, op.Path) + require.NotNil(t, op.Resilience.Timeout) + assert.Equal(t, "45s", *op.Resilience.Timeout) + require.NotNil(t, op.Resilience.IdleTimeout) + assert.Equal(t, "0s", *op.Resilience.IdleTimeout) + } +} + +// With no resilience block on the source provider, no operation gets a resilience block (unchanged +// behavior — routes fall back to the gateway's global default timeout). +func TestTransformProvider_NoResilience_NotAttached(t *testing.T) { + transformer, _ := setupTestTransformer(t) + + provider := &api.LLMProviderConfiguration{ + ApiVersion: "gateway.api-platform.wso2.com/v1", + Kind: "LlmProvider", + Metadata: api.Metadata{Name: "openai-provider"}, + Spec: api.LLMProviderConfigData{ + DisplayName: "test", + Version: "v1.0", + Template: "openai", + Upstream: api.LLMProviderConfigData_Upstream{Url: stringPtr("https://api.example.com")}, + AccessControl: api.LLMAccessControl{Mode: api.AllowAll}, + }, + } + + result, err := transformer.Transform(provider, &api.RestAPI{}) + require.NoError(t, err) + require.NotEmpty(t, result.Spec.Operations) + + for _, op := range result.Spec.Operations { + assert.Nil(t, op.Resilience, "route %s %s should not carry resilience", op.Method, op.Path) + } +} + +// A proxy is always allow-all with no access control, so every generated route carries the +// resilience block. +func TestTransformProxy_ResilienceOnAllRoutes(t *testing.T) { + store := storage.NewConfigStore() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + db := newTestSQLiteStorage(t, logger) + + template := &models.StoredLLMProviderTemplate{ + UUID: "0000-db-template-id-0000-000000000001", + Configuration: api.LLMProviderTemplate{ + ApiVersion: api.LLMProviderTemplateApiVersionGatewayApiPlatformWso2Comv1, + Kind: api.LLMProviderTemplateKindLlmProviderTemplate, + Metadata: api.Metadata{Name: "openai"}, + Spec: api.LLMProviderTemplateData{DisplayName: "openai"}, + }, + } + require.NoError(t, db.SaveLLMProviderTemplate(template)) + + now := time.Now() + provider := &models.StoredConfig{ + UUID: "0000-db-provider-id-0000-000000000000", + Kind: string(api.LLMProviderConfigurationKindLlmProvider), + Handle: "db-provider", + DisplayName: "db-provider", + Version: "v1.0", + Configuration: api.RestAPI{ + ApiVersion: api.RestAPIApiVersionGatewayApiPlatformWso2Comv1, + Kind: api.RestAPIKindRestApi, + Metadata: api.Metadata{Name: "db-provider"}, + Spec: api.APIConfigData{ + DisplayName: "db-provider", + Version: "v1.0", + Context: "/db-provider", + Upstream: struct { + Main api.Upstream `json:"main" yaml:"main"` + Sandbox *api.Upstream `json:"sandbox,omitempty" yaml:"sandbox,omitempty"` + }{Main: api.Upstream{Url: stringPtr("https://api.openai.com")}}, + }, + }, + SourceConfiguration: api.LLMProviderConfiguration{ + ApiVersion: api.LLMProviderConfigurationApiVersionGatewayApiPlatformWso2Comv1, + Kind: api.LLMProviderConfigurationKindLlmProvider, + Metadata: api.Metadata{Name: "db-provider"}, + Spec: api.LLMProviderConfigData{ + DisplayName: "db-provider", + Version: "v1.0", + Context: stringPtr("/db-provider"), + Template: "openai", + Upstream: api.LLMProviderConfigData_Upstream{Url: stringPtr("https://api.openai.com")}, + AccessControl: api.LLMAccessControl{Mode: api.AllowAll}, + }, + }, + DesiredState: models.StateDeployed, + CreatedAt: now, + UpdatedAt: now, + } + require.NoError(t, db.SaveConfig(provider)) + + routerConfig := &config.RouterConfig{ListenerPort: 8080} + transformer := NewLLMProviderTransformer(store, db, routerConfig, newTestPolicyVersionResolver()) + + timeout := "75s" + proxy := &api.LLMProxyConfiguration{ + ApiVersion: api.LLMProxyConfigurationApiVersionGatewayApiPlatformWso2Comv1, + Kind: api.LLMProxyConfigurationKindLlmProxy, + Metadata: api.Metadata{Name: "db-proxy"}, + Spec: api.LLMProxyConfigData{ + DisplayName: "db-proxy", + Version: "v1.0", + Provider: api.LLMProxyProvider{Id: "db-provider"}, + Resilience: &api.Resilience{Timeout: &timeout}, + }, + } + + result, err := transformer.Transform(proxy, &api.RestAPI{}) + require.NoError(t, err) + require.NotEmpty(t, result.Spec.Operations) + + for _, op := range result.Spec.Operations { + require.NotNil(t, op.Resilience, "proxy route %s %s should carry resilience", op.Method, op.Path) + require.NotNil(t, op.Resilience.Timeout) + assert.Equal(t, "75s", *op.Resilience.Timeout) + } +} + +// ============================================================================ +// Upstream ref / connect-timeout threading tests +// ============================================================================ + +// The provider converter maps an upstream `ref` onto the derived RestAPI (instead of a direct url) +// and threads the upstreamDefinitions through, so the per-upstream connect timeout resolves the same +// way it does for RestApi. +func TestTransform_Provider_UpstreamRef_ThreadsDefinitions(t *testing.T) { + transformer, _ := setupTestTransformer(t) + + defs := &[]api.UpstreamDefinition{{ + Name: "openai-backend", + Timeout: &api.UpstreamTimeout{Connect: stringPtr("6s")}, + }} + provider := &api.LLMProviderConfiguration{ + ApiVersion: "gateway.api-platform.wso2.com/v1", + Kind: "LlmProvider", + Metadata: api.Metadata{Name: "openai-provider"}, + Spec: api.LLMProviderConfigData{ + DisplayName: "ref-provider", + Version: "v1.0", + Template: "openai", + UpstreamDefinitions: defs, + Upstream: api.LLMProviderConfigData_Upstream{Ref: stringPtr("openai-backend")}, + AccessControl: api.LLMAccessControl{Mode: api.AllowAll}, + }, + } + + res, err := transformer.Transform(provider, &api.RestAPI{}) + require.NoError(t, err) + + require.NotNil(t, res.Spec.Upstream.Main.Ref, "ref should be mapped onto the derived upstream") + assert.Equal(t, "openai-backend", *res.Spec.Upstream.Main.Ref) + assert.Nil(t, res.Spec.Upstream.Main.Url, "ref-based upstream must not also set url") + + require.NotNil(t, res.Spec.UpstreamDefinitions, "upstreamDefinitions must be threaded through") + require.Len(t, *res.Spec.UpstreamDefinitions, 1) + require.NotNil(t, (*res.Spec.UpstreamDefinitions)[0].Timeout) + assert.Equal(t, "6s", *(*res.Spec.UpstreamDefinitions)[0].Timeout.Connect) +} + +func TestTransform_Provider_UpstreamUrl_Unchanged(t *testing.T) { + transformer, _ := setupTestTransformer(t) + + provider := &api.LLMProviderConfiguration{ + ApiVersion: "gateway.api-platform.wso2.com/v1", + Kind: "LlmProvider", + Metadata: api.Metadata{Name: "openai-provider"}, + Spec: api.LLMProviderConfigData{ + DisplayName: "url-provider", + Version: "v1.0", + Template: "openai", + Upstream: api.LLMProviderConfigData_Upstream{Url: stringPtr("https://api.openai.com")}, + AccessControl: api.LLMAccessControl{Mode: api.AllowAll}, + }, + } + + res, err := transformer.Transform(provider, &api.RestAPI{}) + require.NoError(t, err) + + require.NotNil(t, res.Spec.Upstream.Main.Url) + assert.Equal(t, "https://api.openai.com", *res.Spec.Upstream.Main.Url) + assert.Nil(t, res.Spec.Upstream.Main.Ref) + assert.Nil(t, res.Spec.UpstreamDefinitions) +} diff --git a/gateway/gateway-controller/pkg/utils/llm_transformer.go b/gateway/gateway-controller/pkg/utils/llm_transformer.go index 0146d2ad64..52fe234c64 100644 --- a/gateway/gateway-controller/pkg/utils/llm_transformer.go +++ b/gateway/gateway-controller/pkg/utils/llm_transformer.go @@ -295,6 +295,9 @@ func (t *LLMProviderTransformer) transformProxy(proxy *api.LLMProxyConfiguration } } } + // A proxy is always allow-all with no access control, so there are no deny routes: + // attach API-level resilience to all generated routes. + applyResilienceToTrafficRoutes(ops, proxy.Spec.Resilience, nil) spec.Operations = ops // Global (api-level) policies: route into the derived RestAPI's spec.Policies so they are @@ -341,11 +344,19 @@ func (t *LLMProviderTransformer) transformProvider(provider *api.LLMProviderConf spec.Context = *provider.Spec.Context } - // Step 2) Upstreams: map provider.Spec.Upstreams to api.Upstreams - // Map provider upstream and vhost to API main upstream and vhost - spec.Upstream.Main = api.Upstream{ - Url: provider.Spec.Upstream.Url, + // Step 2) Upstreams: map provider upstream (direct url or upstreamDefinition ref) and vhost + // to the API main upstream. When a ref is used, carry the upstreamDefinitions through so the + // per-upstream connect timeout resolves the same way it does for RestApi. + if provider.Spec.Upstream.Ref != nil && strings.TrimSpace(*provider.Spec.Upstream.Ref) != "" { + spec.Upstream.Main = api.Upstream{ + Ref: provider.Spec.Upstream.Ref, + } + } else { + spec.Upstream.Main = api.Upstream{ + Url: provider.Spec.Upstream.Url, + } } + spec.UpstreamDefinitions = provider.Spec.UpstreamDefinitions if provider.Spec.Vhost != nil { spec.Vhosts = &struct { Main string `json:"main" yaml:"main"` @@ -388,6 +399,12 @@ func (t *LLMProviderTransformer) transformProvider(provider *api.LLMProviderConf var ops []api.Operation + // denyOpKeys tracks the routes created purely to deny traffic (the access-control + // exception routes, which carry the 404 respond policy). API-level resilience is NOT + // attached to these (they never reach an upstream). Populated in allow_all mode; empty + // in deny_all (where every created route is an allowed/forwarding route). + denyOpKeys := make(map[pathMethodKey]bool) + switch mode { case api.AllowAll: var denyPolicyVersion string @@ -427,6 +444,7 @@ func (t *LLMProviderTransformer) transformProvider(provider *api.LLMProviderConf for _, method := range methods { key := pathMethodKey{path: ex.Path, method: method} deniedPathMethods[key] = true + denyOpKeys[key] = true // Check if operation exists if _, exists := operationRegistry[key]; !exists { @@ -634,6 +652,8 @@ func (t *LLMProviderTransformer) transformProvider(provider *api.LLMProviderConf } } } + // Attach API-level resilience to every traffic-forwarding route, skipping the deny routes. + applyResilienceToTrafficRoutes(ops, provider.Spec.Resilience, denyOpKeys) spec.Operations = ops // Global (api-level) policies: route into the derived RestAPI's spec.Policies so they are @@ -658,6 +678,27 @@ func (t *LLMProviderTransformer) transformProvider(provider *api.LLMProviderConf return output, nil } +// applyResilienceToTrafficRoutes attaches the API-level resilience block to every operation that +// forwards traffic upstream, skipping access-control deny routes (which only return a canned 404 +// and never reach an upstream, so a timeout on them is meaningless). denyKeys identifies the deny +// routes by path+method; it is empty for deny_all and proxy (no deny routes exist there), so in +// those cases the block is applied to all routes. A nil resilience block is a no-op. +// +// Resilience is attached at the operation level (not the derived API level) on purpose: that keeps +// the deny routes on the gateway's global default timeout instead of inheriting the API-level value +// via the translator's per-field fallback. +func applyResilienceToTrafficRoutes(ops []api.Operation, resilience *api.Resilience, denyKeys map[pathMethodKey]bool) { + if resilience == nil { + return + } + for i := range ops { + if denyKeys[pathMethodKey{path: ops[i].Path, method: string(ops[i].Method)}] { + continue + } + ops[i].Resilience = resilience + } +} + // GetUpstreamAuthApikeyPolicyParams renders the policy params with given header and value func GetUpstreamAuthApikeyPolicyParams(header, value string) (map[string]interface{}, error) { rendered := fmt.Sprintf(constants.UPSTREAM_AUTH_APIKEY_POLICY_PARAMS, header, value) diff --git a/gateway/gateway-controller/pkg/utils/mcp_transformer.go b/gateway/gateway-controller/pkg/utils/mcp_transformer.go index 2e5b0c6451..7e11a29ac2 100644 --- a/gateway/gateway-controller/pkg/utils/mcp_transformer.go +++ b/gateway/gateway-controller/pkg/utils/mcp_transformer.go @@ -96,6 +96,45 @@ func addMCPSpecificOperations(mcpConfig *api.MCPProxyConfiguration, optionsRequi return operations } +// isMCPForwardingOperation reports whether an MCP-synthesized operation forwards traffic to the +// upstream (where a route/idle timeout is meaningful). The non-forwarding routes are answered +// locally by the policy engine and must be skipped: +// - any OPTIONS route -> CORS preflight, answered locally by the cors policy +// - the PRM path -> OAuth protected-resource metadata, synthesized by the mcp-auth policy +func isMCPForwardingOperation(op api.Operation) bool { + if op.Method == api.OperationMethodOPTIONS { + return false + } + if op.Path == constants.MCP_PRM_RESOURCE_PATH { + return false + } + return true +} + +// applyMCPResilience attaches the API-level resilience block to the MCP traffic-forwarding routes +// (GET/POST/DELETE on the MCP resource path), skipping the local-response routes (OPTIONS, PRM). +// +// Because MCP transports are long-lived streams (SSE / Streamable HTTP), a finite route timeout +// would sever a healthy stream. So — unlike REST/LLM, which fall back to the gateway's global route +// timeout — the MCP route timeout defaults to disabled ("0s") when the user does not set one; the +// idle timeout remains the liveness guard. An explicit resilience.timeout always wins. +func applyMCPResilience(ops []api.Operation, userRes *api.Resilience) { + disabled := "0s" + for i := range ops { + if !isMCPForwardingOperation(ops[i]) { + continue + } + res := &api.Resilience{Timeout: &disabled} + if userRes != nil { + if userRes.Timeout != nil { + res.Timeout = userRes.Timeout + } + res.IdleTimeout = userRes.IdleTimeout + } + ops[i].Resilience = res + } +} + // Transform converts an MCP proxy configuration (input) to an API configuration (output) func (t *MCPTransformer) Transform(input any, output *api.RestAPI) (*api.RestAPI, error) { mcpConfig, ok := input.(*api.MCPProxyConfiguration) @@ -114,9 +153,18 @@ func (t *MCPTransformer) Transform(input any, output *api.RestAPI) (*api.RestAPI apiData.Context = *mcpConfig.Spec.Context } - apiData.Upstream.Main = api.Upstream{ - Url: mcpConfig.Spec.Upstream.Url, + // Map the MCP backend (direct url or upstreamDefinition ref). When a ref is used, carry the + // upstreamDefinitions through so the per-upstream connect timeout resolves like it does for RestApi. + if mcpConfig.Spec.Upstream.Ref != nil && strings.TrimSpace(*mcpConfig.Spec.Upstream.Ref) != "" { + apiData.Upstream.Main = api.Upstream{ + Ref: mcpConfig.Spec.Upstream.Ref, + } + } else { + apiData.Upstream.Main = api.Upstream{ + Url: mcpConfig.Spec.Upstream.Url, + } } + apiData.UpstreamDefinitions = mcpConfig.Spec.UpstreamDefinitions // Process policies var policies []api.Policy @@ -140,6 +188,9 @@ func (t *MCPTransformer) Transform(input any, output *api.RestAPI) (*api.RestAPI // Add MCP-specific operations, conditionally including OPTIONS when CORS is enabled apiData.Operations = addMCPSpecificOperations(mcpConfig, optionsRequired) + // Attach the API-level resilience (route/idle timeout) to the traffic-forwarding routes only. + applyMCPResilience(apiData.Operations, mcpConfig.Spec.Resilience) + // Set upstream auth if present upstream := mcpConfig.Spec.Upstream if upstream.Auth != nil { diff --git a/gateway/gateway-controller/pkg/utils/mcp_transformer_test.go b/gateway/gateway-controller/pkg/utils/mcp_transformer_test.go index 2580a02c94..01eb7b9dc2 100644 --- a/gateway/gateway-controller/pkg/utils/mcp_transformer_test.go +++ b/gateway/gateway-controller/pkg/utils/mcp_transformer_test.go @@ -21,6 +21,8 @@ package utils import ( "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" api "github.com/wso2/api-platform/gateway/gateway-controller/pkg/api/management" "github.com/wso2/api-platform/gateway/gateway-controller/pkg/constants" ) @@ -478,3 +480,171 @@ func TestGetParamsOfPolicy_MCP(t *testing.T) { t.Error("Expected request.headers in params") } } + +// ============================================================================ +// Resilience route-mapping tests +// ============================================================================ + +// mcpOpResilience returns the resilience block attached to the operation matching method+path, +// or nil if the operation is absent or carries no resilience. +func mcpOpResilience(ops []api.Operation, method api.OperationMethod, path string) *api.Resilience { + for _, op := range ops { + if op.Method == method && op.Path == path { + return op.Resilience + } + } + return nil +} + +// When no resilience is set, the MCP forwarding routes (GET/POST/DELETE /mcp) default the route +// timeout to "0s" (disabled) — MCP is streaming, so a finite total cap would sever a live stream. +// The local-response routes (OPTIONS, PRM) must carry no resilience. See mcp-timeout-divergence.md. +func TestMCPTransform_Resilience_DefaultsRouteTimeoutDisabled(t *testing.T) { + context := "/everything" + latest := LATEST_SUPPORTED_MCP_SPEC_VERSION + in := &api.MCPProxyConfiguration{ + Spec: api.MCPProxyConfigData{ + DisplayName: "everything", + Version: "v1.0", + Context: &context, + SpecVersion: &latest, + Upstream: api.MCPProxyConfigData_Upstream{Url: stringPtr("http://backend:8080")}, + // A cors policy makes the transformer emit the OPTIONS routes so we can assert they are skipped. + Policies: &[]api.Policy{{Name: "cors", Version: "v1"}}, + }, + } + + res, err := (&MCPTransformer{}).Transform(in, &api.RestAPI{}) + require.NoError(t, err) + ops := res.Spec.Operations + + for _, m := range []api.OperationMethod{api.OperationMethodGET, api.OperationMethodPOST, api.OperationMethodDELETE} { + r := mcpOpResilience(ops, m, constants.MCP_RESOURCE_PATH) + require.NotNil(t, r, "forwarding %s %s should carry resilience", m, constants.MCP_RESOURCE_PATH) + require.NotNil(t, r.Timeout) + assert.Equal(t, "0s", *r.Timeout, "route timeout should default to disabled for MCP") + assert.Nil(t, r.IdleTimeout, "idle timeout should be left to the global default when unset") + } + + // Local-response routes must not carry a route timeout. + assert.Nil(t, mcpOpResilience(ops, api.OperationMethodOPTIONS, constants.MCP_RESOURCE_PATH), "OPTIONS /mcp is a local reply") + assert.Nil(t, mcpOpResilience(ops, api.OperationMethodGET, constants.MCP_PRM_RESOURCE_PATH), "PRM route is a local reply") + assert.Nil(t, mcpOpResilience(ops, api.OperationMethodOPTIONS, constants.MCP_PRM_RESOURCE_PATH), "OPTIONS PRM is a local reply") +} + +// An explicit resilience block overrides the disabled default and its idleTimeout is carried through, +// on the forwarding routes only. +func TestMCPTransform_Resilience_UserOverride(t *testing.T) { + context := "/everything" + latest := LATEST_SUPPORTED_MCP_SPEC_VERSION + in := &api.MCPProxyConfiguration{ + Spec: api.MCPProxyConfigData{ + DisplayName: "everything", + Version: "v1.0", + Context: &context, + SpecVersion: &latest, + Upstream: api.MCPProxyConfigData_Upstream{Url: stringPtr("http://backend:8080")}, + Resilience: &api.Resilience{Timeout: stringPtr("30s"), IdleTimeout: stringPtr("120s")}, + }, + } + + res, err := (&MCPTransformer{}).Transform(in, &api.RestAPI{}) + require.NoError(t, err) + ops := res.Spec.Operations + + r := mcpOpResilience(ops, api.OperationMethodGET, constants.MCP_RESOURCE_PATH) + require.NotNil(t, r) + require.NotNil(t, r.Timeout) + assert.Equal(t, "30s", *r.Timeout) + require.NotNil(t, r.IdleTimeout) + assert.Equal(t, "120s", *r.IdleTimeout) + + // The PRM route (present for spec >= 2025-06-18) is still skipped. + assert.Nil(t, mcpOpResilience(ops, api.OperationMethodGET, constants.MCP_PRM_RESOURCE_PATH)) +} + +// A user timeout with no idle override: timeout is the user value, idle stays unset (global default). +func TestMCPTransform_Resilience_TimeoutOnly(t *testing.T) { + context := "/everything" + latest := LATEST_SUPPORTED_MCP_SPEC_VERSION + in := &api.MCPProxyConfiguration{ + Spec: api.MCPProxyConfigData{ + DisplayName: "everything", + Version: "v1.0", + Context: &context, + SpecVersion: &latest, + Upstream: api.MCPProxyConfigData_Upstream{Url: stringPtr("http://backend:8080")}, + Resilience: &api.Resilience{Timeout: stringPtr("45s")}, + }, + } + + res, err := (&MCPTransformer{}).Transform(in, &api.RestAPI{}) + require.NoError(t, err) + + r := mcpOpResilience(res.Spec.Operations, api.OperationMethodPOST, constants.MCP_RESOURCE_PATH) + require.NotNil(t, r) + require.NotNil(t, r.Timeout) + assert.Equal(t, "45s", *r.Timeout) + assert.Nil(t, r.IdleTimeout) +} + +// ============================================================================ +// Upstream ref / connect-timeout threading tests +// ============================================================================ + +// The MCP converter maps an upstream `ref` onto the derived RestAPI and threads the +// upstreamDefinitions through, so the per-upstream connect timeout resolves like it does for RestApi. +func TestMCPTransform_UpstreamRef_ThreadsDefinitions(t *testing.T) { + context := "/everything" + latest := LATEST_SUPPORTED_MCP_SPEC_VERSION + defs := []api.UpstreamDefinition{{ + Name: "mcp-backend", + Timeout: &api.UpstreamTimeout{Connect: stringPtr("6s")}, + }} + in := &api.MCPProxyConfiguration{ + Spec: api.MCPProxyConfigData{ + DisplayName: "everything", + Version: "v1.0", + Context: &context, + SpecVersion: &latest, + UpstreamDefinitions: &defs, + Upstream: api.MCPProxyConfigData_Upstream{Ref: stringPtr("mcp-backend")}, + }, + } + + var out api.RestAPI + res, err := (&MCPTransformer{}).Transform(in, &out) + require.NoError(t, err) + + require.NotNil(t, res.Spec.Upstream.Main.Ref) + assert.Equal(t, "mcp-backend", *res.Spec.Upstream.Main.Ref) + assert.Nil(t, res.Spec.Upstream.Main.Url) + + require.NotNil(t, res.Spec.UpstreamDefinitions) + require.Len(t, *res.Spec.UpstreamDefinitions, 1) + require.NotNil(t, (*res.Spec.UpstreamDefinitions)[0].Timeout) + assert.Equal(t, "6s", *(*res.Spec.UpstreamDefinitions)[0].Timeout.Connect) +} + +func TestMCPTransform_UpstreamUrl_Unchanged(t *testing.T) { + context := "/everything" + latest := LATEST_SUPPORTED_MCP_SPEC_VERSION + in := &api.MCPProxyConfiguration{ + Spec: api.MCPProxyConfigData{ + DisplayName: "everything", + Version: "v1.0", + Context: &context, + SpecVersion: &latest, + Upstream: api.MCPProxyConfigData_Upstream{Url: stringPtr("http://backend:8080")}, + }, + } + + var out api.RestAPI + res, err := (&MCPTransformer{}).Transform(in, &out) + require.NoError(t, err) + + require.NotNil(t, res.Spec.Upstream.Main.Url) + assert.Equal(t, "http://backend:8080", *res.Spec.Upstream.Main.Url) + assert.Nil(t, res.Spec.Upstream.Main.Ref) + assert.Nil(t, res.Spec.UpstreamDefinitions) +} diff --git a/gateway/gateway-controller/pkg/xds/translator.go b/gateway/gateway-controller/pkg/xds/translator.go index c0123c5f9c..182f879db7 100644 --- a/gateway/gateway-controller/pkg/xds/translator.go +++ b/gateway/gateway-controller/pkg/xds/translator.go @@ -97,9 +97,11 @@ type Translator struct { } // resolvedTimeout represents parsed timeout values for an upstream. -// Currently only connect timeout is supported at the upstream definition level. +// Route and Idle come from the resilience block. type resolvedTimeout struct { Connect *time.Duration + Route *time.Duration + Idle *time.Duration } // NewTranslator creates a new translator @@ -223,6 +225,16 @@ func (t *Translator) translateRuntimeConfig(rdc *models.RuntimeDeployConfig) ([] return routes, clusters, nil } +// routeTimeoutOrDefault returns the per-route timeout when configured (including an +// explicit zero, which disables the timeout in Envoy), otherwise the global default +// expressed in milliseconds. +func (t *Translator) routeTimeoutOrDefault(v *time.Duration, defaultMs uint32) *durationpb.Duration { + if v != nil { + return durationpb.New(*v) + } + return durationpb.New(time.Duration(defaultMs) * time.Millisecond) +} + // createRouteFromRDC creates an Envoy route from a RuntimeDeployConfig Route. func (t *Translator) createRouteFromRDC(routeKey string, rdcRoute *models.Route, rdc *models.RuntimeDeployConfig) *route.Route { fullPath := rdcRoute.Path @@ -267,15 +279,17 @@ func (t *Translator) createRouteFromRDC(routeKey string, rdcRoute *models.Route, } } - // Build route action with timeouts + // Build route action with timeouts. Per-route resilience values (from the API/operation + // resilience block) take precedence; otherwise fall back to the global route defaults. + var routeResilienceTimeout, routeResilienceIdle *time.Duration + if rdcRoute.Timeout != nil { + routeResilienceTimeout = rdcRoute.Timeout.Timeout + routeResilienceIdle = rdcRoute.Timeout.IdleTimeout + } routeAction := &route.Route_Route{ Route: &route.RouteAction{ - Timeout: durationpb.New( - time.Duration(t.routerConfig.Upstream.Timeouts.RouteTimeoutMs) * time.Millisecond, - ), - IdleTimeout: durationpb.New( - time.Duration(t.routerConfig.Upstream.Timeouts.RouteIdleTimeoutMs) * time.Millisecond, - ), + Timeout: t.routeTimeoutOrDefault(routeResilienceTimeout, t.routerConfig.Upstream.Timeouts.RouteTimeoutMs), + IdleTimeout: t.routeTimeoutOrDefault(routeResilienceIdle, t.routerConfig.Upstream.Timeouts.RouteIdleTimeoutMs), }, } @@ -862,13 +876,25 @@ func (t *Translator) translateAPIConfig(cfg *models.StoredConfig, allConfigs []* } } + // Resolve API-level resilience timeouts once; operation-level values override per field. + apiTimeout, apiIdleTimeout, err := ResolveResilience(apiData.Resilience) + if err != nil { + return nil, nil, fmt.Errorf("invalid API-level resilience: %w", err) + } + for _, op := range apiData.Operations { // Determine if dynamic cluster selection should be used // When upstreamDefinitions exist, use cluster_header routing so policies can select the upstream useClusterHeader := apiData.UpstreamDefinitions != nil && len(*apiData.UpstreamDefinitions) > 0 + opTimeout, opIdleTimeout, err := ResolveResilience(op.Resilience) + if err != nil { + return nil, nil, fmt.Errorf("invalid resilience for operation %s %s: %w", op.Method, op.Path, err) + } + opTimeoutCfg := combineRouteResilience(mainTimeout, apiTimeout, apiIdleTimeout, opTimeout, opIdleTimeout) + r := t.createRoute(cfg.UUID, apiData.DisplayName, apiData.Version, apiData.Context, string(op.Method), op.Path, - mainClusterName, parsedMainURL.Path, effectiveMainVHost, cfg.Kind, templateHandle, providerName, apiData.Upstream.Main.HostRewrite, apiProjectID, mainTimeout, useClusterHeader, upstreamDefPaths) + mainClusterName, parsedMainURL.Path, effectiveMainVHost, cfg.Kind, templateHandle, providerName, apiData.Upstream.Main.HostRewrite, apiProjectID, opTimeoutCfg, useClusterHeader, upstreamDefPaths) mainRoutesList = append(mainRoutesList, r) } routesList = append(routesList, mainRoutesList...) @@ -894,8 +920,14 @@ func (t *Translator) translateAPIConfig(cfg *models.StoredConfig, allConfigs []* sbRoutesList := make([]*route.Route, 0) sbUseClusterHeader := apiData.UpstreamDefinitions != nil && len(*apiData.UpstreamDefinitions) > 0 for _, op := range apiData.Operations { + opTimeout, opIdleTimeout, err := ResolveResilience(op.Resilience) + if err != nil { + return nil, nil, fmt.Errorf("invalid resilience for operation %s %s: %w", op.Method, op.Path, err) + } + opTimeoutCfg := combineRouteResilience(sbTimeout, apiTimeout, apiIdleTimeout, opTimeout, opIdleTimeout) + r := t.createRoute(cfg.UUID, apiData.DisplayName, apiData.Version, apiData.Context, string(op.Method), op.Path, - sbClusterName, parsedSbURL.Path, effectiveSandboxVHost, cfg.Kind, templateHandle, providerName, apiData.Upstream.Sandbox.HostRewrite, apiProjectID, sbTimeout, sbUseClusterHeader, upstreamDefPaths) + sbClusterName, parsedSbURL.Path, effectiveSandboxVHost, cfg.Kind, templateHandle, providerName, apiData.Upstream.Sandbox.HostRewrite, apiProjectID, opTimeoutCfg, sbUseClusterHeader, upstreamDefPaths) sbRoutesList = append(sbRoutesList, r) } routesList = append(routesList, sbRoutesList...) @@ -1086,6 +1118,13 @@ func (t *Translator) createListener(virtualHosts []*route.VirtualHost, isHTTPS b HttpFilters: httpFilters, ServerHeaderTransformation: convertServerHeaderTransformation(t.routerConfig.HTTPListener.ServerHeaderTransformation), ServerName: t.routerConfig.HTTPListener.ServerHeaderValue, + // HCM-level (downstream) timeouts. Defaults match Envoy's documented defaults. + RequestTimeout: durationpb.New(t.routerConfig.HTTPListener.Timeouts.RequestTimeout), + RequestHeadersTimeout: durationpb.New(t.routerConfig.HTTPListener.Timeouts.RequestHeadersTimeout), + StreamIdleTimeout: durationpb.New(t.routerConfig.HTTPListener.Timeouts.StreamIdleTimeout), + CommonHttpProtocolOptions: &core.HttpProtocolOptions{ + IdleTimeout: durationpb.New(t.routerConfig.HTTPListener.Timeouts.IdleTimeout), + }, } // Add access logs if enabled @@ -1850,15 +1889,17 @@ func (t *Translator) createRoute(apiId, apiName, apiVersion, context, method, pa } } - // Currently the route level timeouts are configurable using the global configuration. + // Route-level timeouts: per-route resilience values (resilience block) take precedence, + // otherwise fall back to the global configuration defaults. + var routeTimeout, routeIdleTimeout *time.Duration + if timeoutCfg != nil { + routeTimeout = timeoutCfg.Route + routeIdleTimeout = timeoutCfg.Idle + } routeAction := &route.Route_Route{ Route: &route.RouteAction{ - Timeout: durationpb.New( - time.Duration(t.routerConfig.Upstream.Timeouts.RouteTimeoutMs) * time.Millisecond, - ), - IdleTimeout: durationpb.New( - time.Duration(t.routerConfig.Upstream.Timeouts.RouteIdleTimeoutMs) * time.Millisecond, - ), + Timeout: t.routeTimeoutOrDefault(routeTimeout, t.routerConfig.Upstream.Timeouts.RouteTimeoutMs), + IdleTimeout: t.routeTimeoutOrDefault(routeIdleTimeout, t.routerConfig.Upstream.Timeouts.RouteIdleTimeoutMs), }, } @@ -3204,6 +3245,68 @@ func parseTimeout(timeoutStr *string) (*time.Duration, error) { return &duration, nil } +// parseDurationAllowZero parses a duration string (e.g. "15s", "0s") into a *time.Duration. +// Unlike parseTimeout it accepts zero ("0s" means the timeout is explicitly disabled). The format +// is enforced against constants.ResilienceDurationPattern so this downstream parser stays +// consistent with the CRD admission controller and the management-API validator: compound +// ("1h30m"), negative ("-30s"), and unitless ("0") values are rejected. Returns nil for nil/empty. +func parseDurationAllowZero(timeoutStr *string) (*time.Duration, error) { + if timeoutStr == nil || strings.TrimSpace(*timeoutStr) == "" { + return nil, nil + } + + s := strings.TrimSpace(*timeoutStr) + if !constants.ResilienceDurationRegex.MatchString(s) { + return nil, fmt.Errorf("invalid timeout format %q: expected a single-unit duration like \"30s\", \"500ms\", or \"0s\" to disable", s) + } + + duration, err := time.ParseDuration(s) + if err != nil { + return nil, fmt.Errorf("invalid timeout format: %w", err) + } + + return &duration, nil +} + +// ResolveResilience parses a resilience block into route timeout and idle-timeout durations. +// A nil block, or unset fields, yield nil durations (meaning "use the global default"). +// "0s" yields a non-nil zero duration (meaning "explicitly disabled"). +func ResolveResilience(r *api.Resilience) (timeout *time.Duration, idleTimeout *time.Duration, err error) { + if r == nil { + return nil, nil, nil + } + if timeout, err = parseDurationAllowZero(r.Timeout); err != nil { + return nil, nil, fmt.Errorf("invalid resilience.timeout: %w", err) + } + if idleTimeout, err = parseDurationAllowZero(r.IdleTimeout); err != nil { + return nil, nil, fmt.Errorf("invalid resilience.idleTimeout: %w", err) + } + return timeout, idleTimeout, nil +} + +// combineRouteResilience returns a resolvedTimeout for a single route, preserving the +// upstream connect timeout from base and applying the effective route/idle timeouts +// (operation-level overriding API-level, per field). It returns base unchanged when no +// resilience is configured at either level. +func combineRouteResilience(base *resolvedTimeout, apiTimeout, apiIdle, opTimeout, opIdle *time.Duration) *resolvedTimeout { + effTimeout := opTimeout + if effTimeout == nil { + effTimeout = apiTimeout + } + effIdle := opIdle + if effIdle == nil { + effIdle = apiIdle + } + if effTimeout == nil && effIdle == nil { + return base + } + rt := resolvedTimeout{Route: effTimeout, Idle: effIdle} + if base != nil { + rt.Connect = base.Connect + } + return &rt +} + // resolveTimeoutFromDefinition converts an UpstreamDefinition's timeout block into a resolvedTimeout. // Returns nil if there is no timeout block or all fields are empty. func resolveTimeoutFromDefinition(def *api.UpstreamDefinition) (*resolvedTimeout, error) { diff --git a/gateway/gateway-controller/pkg/xds/translator_test.go b/gateway/gateway-controller/pkg/xds/translator_test.go index 80e6f87bb2..428ea8adf7 100644 --- a/gateway/gateway-controller/pkg/xds/translator_test.go +++ b/gateway/gateway-controller/pkg/xds/translator_test.go @@ -32,6 +32,7 @@ import ( core "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" listener "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" route "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + hcm "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" tlsv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" resource "github.com/envoyproxy/go-control-plane/pkg/resource/v3" "github.com/stretchr/testify/assert" @@ -789,6 +790,53 @@ func TestTranslator_WildcardUpstreamRewriteFromRDC(t *testing.T) { } } +// TestTranslator_RouteResilienceTimeoutsFromRDC verifies that per-route resilience +// timeouts on a models.Route flow into the Envoy RouteAction, with fallback to the +// global defaults (60s / 300s from testRouterConfig) when unset, and that an explicit +// 0s is preserved (disables the timeout). +func TestTranslator_RouteResilienceTimeoutsFromRDC(t *testing.T) { + logger := createTestLogger() + routerCfg := testRouterConfig() + cfg := testConfig() + translator := NewTranslator(logger, routerCfg, nil, cfg) + + dur := func(d time.Duration) *time.Duration { return &d } + + tests := []struct { + name string + timeout *models.RouteTimeout + wantTimeout time.Duration + wantIdle time.Duration + }{ + {name: "nil timeout uses global defaults", timeout: nil, wantTimeout: 60 * time.Second, wantIdle: 300 * time.Second}, + {name: "configured values applied", timeout: &models.RouteTimeout{Timeout: dur(2 * time.Second), IdleTimeout: dur(10 * time.Second)}, wantTimeout: 2 * time.Second, wantIdle: 10 * time.Second}, + {name: "timeout set, idle falls back", timeout: &models.RouteTimeout{Timeout: dur(3 * time.Second)}, wantTimeout: 3 * time.Second, wantIdle: 300 * time.Second}, + {name: "explicit 0s disables route timeout", timeout: &models.RouteTimeout{Timeout: dur(0)}, wantTimeout: 0, wantIdle: 300 * time.Second}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rdc := &models.RuntimeDeployConfig{ + UpstreamClusters: map[string]*models.UpstreamCluster{ + "main": {Endpoints: []models.Endpoint{{Host: "echo", Port: 80}}}, + }, + } + rdcRoute := &models.Route{ + Method: "GET", + Path: "/api/v1.0/items", + OperationPath: "/items", + AutoHostRewrite: true, + Timeout: tt.timeout, + Upstream: models.RouteUpstream{ClusterKey: "main"}, + } + r := translator.createRouteFromRDC("GET|/api/v1.0/items|", rdcRoute, rdc) + require.NotNil(t, r) + assert.Equal(t, tt.wantTimeout, r.GetRoute().GetTimeout().AsDuration(), "route timeout") + assert.Equal(t, tt.wantIdle, r.GetRoute().GetIdleTimeout().AsDuration(), "route idle timeout") + }) + } +} + // TestTranslator_MCPUpstreamRewriteFromRDC verifies the MCP "/mcp"-not-appended behavior on the // RuntimeDeployConfig path (createRouteFromRDC), which the policy/runtime xDS pipeline uses. func TestTranslator_MCPUpstreamRewriteFromRDC(t *testing.T) { @@ -1107,6 +1155,55 @@ func TestTranslator_ExtractProviderName_NilSourceConfig(t *testing.T) { assert.Equal(t, "", result) } +// extractHCM pulls the HttpConnectionManager out of the listener's first filter chain. +func extractHCM(t *testing.T, lis *listener.Listener) *hcm.HttpConnectionManager { + t.Helper() + require.NotEmpty(t, lis.GetFilterChains()) + require.NotEmpty(t, lis.GetFilterChains()[0].GetFilters()) + typedConfig := lis.GetFilterChains()[0].GetFilters()[0].GetTypedConfig() + require.NotNil(t, typedConfig) + manager := &hcm.HttpConnectionManager{} + require.NoError(t, typedConfig.UnmarshalTo(manager)) + return manager +} + +func TestTranslator_CreateListener_HCMTimeouts(t *testing.T) { + tests := []struct { + name string + timeouts config.HCMTimeouts + }{ + { + name: "configured values", + timeouts: config.HCMTimeouts{RequestTimeout: 30 * time.Second, RequestHeadersTimeout: 10 * time.Second, StreamIdleTimeout: 2 * time.Minute, IdleTimeout: 30 * time.Minute}, + }, + { + name: "envoy defaults flow through unchanged", + timeouts: config.HCMTimeouts{RequestTimeout: 0, RequestHeadersTimeout: 0, StreamIdleTimeout: 5 * time.Minute, IdleTimeout: time.Hour}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + logger := createTestLogger() + routerCfg := testRouterConfig() + routerCfg.HTTPListener.Timeouts = tt.timeouts + cfg := testConfig() + cfg.Router = *routerCfg + translator := NewTranslator(logger, routerCfg, nil, cfg) + + lis, _, err := translator.createListener(nil, false) + require.NoError(t, err) + + manager := extractHCM(t, lis) + assert.Equal(t, tt.timeouts.RequestTimeout, manager.GetRequestTimeout().AsDuration(), "request_timeout") + assert.Equal(t, tt.timeouts.RequestHeadersTimeout, manager.GetRequestHeadersTimeout().AsDuration(), "request_headers_timeout") + assert.Equal(t, tt.timeouts.StreamIdleTimeout, manager.GetStreamIdleTimeout().AsDuration(), "stream_idle_timeout") + require.NotNil(t, manager.GetCommonHttpProtocolOptions(), "common_http_protocol_options must be set") + assert.Equal(t, tt.timeouts.IdleTimeout, manager.GetCommonHttpProtocolOptions().GetIdleTimeout().AsDuration(), "idle_timeout") + }) + } +} + func TestTranslator_CreateAccessLogConfig_Disabled(t *testing.T) { // Note: createAccessLogConfig should only be called when access logs are enabled. // The check for enabled is done at the caller level. When called directly with disabled @@ -2527,3 +2624,40 @@ func TestTranslator_CreateDynamicFwdListenerForWebSubHub(t *testing.T) { assert.Equal(t, core.SocketAddress_TCP, listener.GetAddress().GetSocketAddress().GetProtocol()) }) } + +// parseDurationAllowZero must accept exactly what the CRD admission controller accepts +// (constants.ResilienceDurationPattern): single-unit durations including "0s" to disable, while +// rejecting compound, negative, and unitless values. +func TestParseDurationAllowZero_MatchesCRDPattern(t *testing.T) { + ptr := func(s string) *string { return &s } + + t.Run("accepts single-unit and zero", func(t *testing.T) { + for _, in := range []string{"30s", "500ms", "1m", "2h", "1.5s", "0s", "0ms"} { + d, err := parseDurationAllowZero(ptr(in)) + if err != nil { + t.Errorf("expected %q to be accepted, got error: %v", in, err) + continue + } + if d == nil { + t.Errorf("expected %q to yield a non-nil duration", in) + } + } + }) + + t.Run("nil and empty yield nil without error", func(t *testing.T) { + for _, in := range []*string{nil, ptr(""), ptr(" ")} { + d, err := parseDurationAllowZero(in) + if err != nil || d != nil { + t.Errorf("expected nil,nil for empty input, got %v,%v", d, err) + } + } + }) + + t.Run("rejects compound, negative, and unitless", func(t *testing.T) { + for _, in := range []string{"1h30m", "1m30s", "-30s", "-5s", "30", "0", "15seconds", "abc"} { + if _, err := parseDurationAllowZero(ptr(in)); err == nil { + t.Errorf("expected %q to be rejected, but it was accepted", in) + } + } + }) +} diff --git a/gateway/it/features/backend-timeout.feature b/gateway/it/features/backend-timeout.feature new file mode 100644 index 0000000000..d701f21f4d --- /dev/null +++ b/gateway/it/features/backend-timeout.feature @@ -0,0 +1,135 @@ +# -------------------------------------------------------------------- +# Copyright (c) 2026, WSO2 LLC. (https://www.wso2.com). +# +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# -------------------------------------------------------------------- + +@backend-timeout @resilience +Feature: Backend route timeouts via the resilience block + As an API developer + I want to configure the route timeout via a resilience block at the API and operation level + So that requests to slow backends are terminated by the gateway within the configured time + + # These scenarios exercise resilience.timeout (Envoy RouteAction.Timeout). The slow + # backend is httpbin (echo-backend) /delay/{n}, which sleeps n seconds before responding; + # when the route timeout is shorter than the delay, the gateway returns 504. + # resilience.idleTimeout maps to RouteAction.IdleTimeout and is covered by unit tests + # (it cannot be exercised deterministically over HTTP here). + + Background: + Given the gateway services are running + + # API-level resilience.timeout (2s) is shorter than the backend delay (5s), so the + # gateway must time the route out with 504 at ~2s instead of waiting for the backend. + Scenario: API-level resilience timeout terminates a slow backend + Given I authenticate using basic auth as "admin" + When I deploy this API configuration: + """ + apiVersion: gateway.api-platform.wso2.com/v1 + kind: RestApi + metadata: + name: backend-timeout-api-v1.0 + spec: + displayName: Backend-Timeout-API + version: v1.0 + context: /backend-timeout-api/$version + upstream: + main: + url: http://echo-backend:80 + resilience: + timeout: 2s + operations: + - method: GET + path: /get + - method: GET + path: /delay/5 + """ + Then the response should be successful + And I wait for the endpoint "http://localhost:8080/backend-timeout-api/v1.0/get" to be ready + And I record the current time as "request_start" + When I send a GET request to "http://localhost:8080/backend-timeout-api/v1.0/delay/5" + Then the response status code should be 504 + And the request should have taken at least "2" seconds since "request_start" + Given I authenticate using basic auth as "admin" + When I delete the API "backend-timeout-api-v1.0" + Then the response should be successful + + # Operation-level resilience overrides the API level: the API-level timeout (10s) is + # longer than the backend delay (5s) and would let the request succeed, but the + # operation-level timeout (2s) wins, so the gateway returns 504 at ~2s. + Scenario: Operation-level resilience timeout overrides the API level + Given I authenticate using basic auth as "admin" + When I deploy this API configuration: + """ + apiVersion: gateway.api-platform.wso2.com/v1 + kind: RestApi + metadata: + name: backend-timeout-override-v1.0 + spec: + displayName: Backend-Timeout-Override + version: v1.0 + context: /backend-timeout-override/$version + upstream: + main: + url: http://echo-backend:80 + resilience: + timeout: 10s + operations: + - method: GET + path: /get + - method: GET + path: /delay/5 + resilience: + timeout: 2s + """ + Then the response should be successful + And I wait for the endpoint "http://localhost:8080/backend-timeout-override/v1.0/get" to be ready + And I record the current time as "request_start" + When I send a GET request to "http://localhost:8080/backend-timeout-override/v1.0/delay/5" + Then the response status code should be 504 + And the request should have taken at least "2" seconds since "request_start" + Given I authenticate using basic auth as "admin" + When I delete the API "backend-timeout-override-v1.0" + Then the response should be successful + + # Without a resilience block the global route timeout default (60s) applies, so a + # backend that responds within it succeeds normally. + Scenario: No resilience block falls back to the global default and succeeds + Given I authenticate using basic auth as "admin" + When I deploy this API configuration: + """ + apiVersion: gateway.api-platform.wso2.com/v1 + kind: RestApi + metadata: + name: backend-timeout-default-v1.0 + spec: + displayName: Backend-Timeout-Default + version: v1.0 + context: /backend-timeout-default/$version + upstream: + main: + url: http://echo-backend:80 + operations: + - method: GET + path: /get + - method: GET + path: /delay/2 + """ + Then the response should be successful + And I wait for the endpoint "http://localhost:8080/backend-timeout-default/v1.0/get" to be ready + When I send a GET request to "http://localhost:8080/backend-timeout-default/v1.0/delay/2" + Then the response status code should be 200 + Given I authenticate using basic auth as "admin" + When I delete the API "backend-timeout-default-v1.0" + Then the response should be successful diff --git a/gateway/it/features/backend_timeout.feature b/gateway/it/features/backend_timeout.feature deleted file mode 100644 index 655afa375d..0000000000 --- a/gateway/it/features/backend_timeout.feature +++ /dev/null @@ -1,99 +0,0 @@ -# -------------------------------------------------------------------- -# Copyright (c) 2025, WSO2 LLC. (https://www.wso2.com). -# -# WSO2 LLC. licenses this file to you under the Apache License, -# Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -------------------------------------------------------------------- - -@backend-timeout -Feature: Backend timeout - As an API developer - I want backend timeout (upstreamDefinitions) to be enforced by the gateway - So that requests to slow or unreachable backends fail within the configured timeout - - Background: - Given the gateway services are running - - # Tests cluster connect_timeout: upstream does not accept TCP connection in time. - # Uses unreachable IP (192.0.2.1 per RFC 5737) so connect attempt hangs until connect_timeout. - Scenario: RestApi backend timeout using upstreamDefinitions - Given I authenticate using basic auth as "admin" - When I deploy this API configuration: - """ - apiVersion: gateway.api-platform.wso2.com/v1 - kind: RestApi - metadata: - name: timeout-api-v1.0 - spec: - displayName: Timeout-API - version: v1.0 - context: /timeout-api/$version - upstreamDefinitions: - - name: my-timeout-upstream - timeout: - connect: 6000ms - upstreams: - - url: http://192.0.2.1:80 - upstream: - main: - ref: my-timeout-upstream - operations: - - method: GET - path: / - """ - Then the response should be successful - And the response should be valid JSON - And the JSON response field "status" should be "success" - And I record the current time as "request_start" - When I send a GET request to "http://localhost:8080/timeout-api/v1.0/" - Then the response status code should be 503 - And the request should have taken at least "6" seconds since "request_start" - Given I authenticate using basic auth as "admin" - When I delete the API "timeout-api-v1.0" - Then the response should be successful - - # Global-default scenario: route timeout comes from it config (6s); elapsed-time assertion verifies configured global timeout. - Scenario: RestApi without upstream timeout uses global defaults - Given I authenticate using basic auth as "admin" - When I deploy this API configuration: - """ - apiVersion: gateway.api-platform.wso2.com/v1 - kind: RestApi - metadata: - name: timeout-api-global-v1.0 - spec: - displayName: Timeout-API-Global - version: v1.0 - context: /timeout-global/$version - upstreamDefinitions: - - name: my-timeout-upstream-global - upstreams: - - url: http://192.0.2.1:80 - upstream: - main: - ref: my-timeout-upstream-global - operations: - - method: GET - path: / - """ - Then the response should be successful - And the response should be valid JSON - And the JSON response field "status" should be "success" - And I record the current time as "request_start" - When I send a GET request to "http://localhost:8080/timeout-global/v1.0/" - Then the response status code should be 503 - And the request should have taken at least "5" seconds since "request_start" - Given I authenticate using basic auth as "admin" - When I delete the API "timeout-api-global-v1.0" - Then the response should be successful - diff --git a/gateway/it/features/llm-backend-timeout.feature b/gateway/it/features/llm-backend-timeout.feature new file mode 100644 index 0000000000..addc30347b --- /dev/null +++ b/gateway/it/features/llm-backend-timeout.feature @@ -0,0 +1,228 @@ +# -------------------------------------------------------------------- +# Copyright (c) 2026, WSO2 LLC. (https://www.wso2.com). +# +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# -------------------------------------------------------------------- + +@llm @backend-timeout @resilience +Feature: Backend route timeouts for LLM providers and proxies via the resilience block + As an API developer + I want to configure the route timeout via an API-level resilience block on LLM providers and proxies + So that requests to slow LLM backends are terminated by the gateway within the configured time + + # LLM kinds support resilience at the API level only + + Background: + Given the gateway services are running + + # allow_all creates a catch-all route for all traffic; the API-level resilience.timeout (2s) is + # shorter than the backend delay (5s), so the gateway times the route out with 504 at ~2s. + Scenario: LLM provider API-level resilience timeout terminates a slow backend + Given I authenticate using basic auth as "admin" + When I create this LLM provider: + """ + apiVersion: gateway.api-platform.wso2.com/v1 + kind: LlmProvider + metadata: + name: llm-timeout-provider + spec: + displayName: LLM Timeout Provider + version: v1.0 + template: openai + context: /llm-timeout + upstream: + url: http://echo-backend:80 + accessControl: + mode: allow_all + resilience: + timeout: 2s + """ + Then the response status code should be 201 + And I wait for the endpoint "http://localhost:8080/llm-timeout/get" to be ready + And I record the current time as "request_start" + When I send a GET request to "http://localhost:8080/llm-timeout/delay/5" + Then the response status code should be 504 + And the request should have taken at least "2" seconds since "request_start" + Given I authenticate using basic auth as "admin" + When I delete the LLM provider "llm-timeout-provider" + Then the response should be successful + + # deny_all creates routes only for the allow-listed exception paths; the resilience block must be + # attached to those (forwarding) routes too. The allow-listed /delay/5 route times out at ~2s. + Scenario: LLM provider deny_all attaches the timeout to allow-listed routes + Given I authenticate using basic auth as "admin" + When I create this LLM provider: + """ + apiVersion: gateway.api-platform.wso2.com/v1 + kind: LlmProvider + metadata: + name: llm-timeout-deny-provider + spec: + displayName: LLM Timeout Deny Provider + version: v1.0 + template: openai + context: /llm-timeout-deny + upstream: + url: http://echo-backend:80 + accessControl: + mode: deny_all + exceptions: + - path: /get + methods: [GET] + - path: /delay/5 + methods: [GET] + resilience: + timeout: 2s + """ + Then the response status code should be 201 + And I wait for the endpoint "http://localhost:8080/llm-timeout-deny/get" to be ready + And I record the current time as "request_start" + When I send a GET request to "http://localhost:8080/llm-timeout-deny/delay/5" + Then the response status code should be 504 + And the request should have taken at least "2" seconds since "request_start" + Given I authenticate using basic auth as "admin" + When I delete the LLM provider "llm-timeout-deny-provider" + Then the response should be successful + + # Without a resilience block the global route timeout default (60s) applies, so a backend that + # responds within it succeeds normally. + Scenario: LLM provider without a resilience block falls back to the global default and succeeds + Given I authenticate using basic auth as "admin" + When I create this LLM provider: + """ + apiVersion: gateway.api-platform.wso2.com/v1 + kind: LlmProvider + metadata: + name: llm-timeout-default-provider + spec: + displayName: LLM Timeout Default Provider + version: v1.0 + template: openai + context: /llm-timeout-default + upstream: + url: http://echo-backend:80 + accessControl: + mode: allow_all + """ + Then the response status code should be 201 + And I wait for the endpoint "http://localhost:8080/llm-timeout-default/get" to be ready + When I send a GET request to "http://localhost:8080/llm-timeout-default/delay/2" + Then the response status code should be 200 + Given I authenticate using basic auth as "admin" + When I delete the LLM provider "llm-timeout-default-provider" + Then the response should be successful + + # A proxy is always allow-all (catch-all). The proxy's own API-level resilience.timeout (2s) + # bounds the whole proxied call (client -> proxy route -> loopback -> provider route -> backend). + # The backing provider is left on the global default, so the 504 is attributable to the proxy. + Scenario: LLM proxy API-level resilience timeout terminates a slow backend + Given I authenticate using basic auth as "admin" + When I create this LLM provider: + """ + apiVersion: gateway.api-platform.wso2.com/v1 + kind: LlmProvider + metadata: + name: llm-timeout-proxy-provider + spec: + displayName: LLM Timeout Proxy Provider + version: v1.0 + template: openai + context: /llm-timeout-backing + upstream: + url: http://echo-backend:80 + accessControl: + mode: allow_all + """ + Then the response status code should be 201 + When I deploy this LLM proxy configuration: + """ + apiVersion: gateway.api-platform.wso2.com/v1 + kind: LlmProxy + metadata: + name: llm-timeout-proxy + spec: + displayName: LLM Timeout Proxy + version: v1.0 + context: /llm-timeout-proxy + provider: + id: llm-timeout-proxy-provider + resilience: + timeout: 2s + """ + Then the response status code should be 201 + And I wait for the endpoint "http://localhost:8080/llm-timeout-proxy/get" to be ready + And I record the current time as "request_start" + When I send a GET request to "http://localhost:8080/llm-timeout-proxy/delay/5" + Then the response status code should be 504 + And the request should have taken at least "2" seconds since "request_start" + Given I authenticate using basic auth as "admin" + When I send a DELETE request to the "gateway-controller" service at "/llm-proxies/llm-timeout-proxy" + Then the response should be successful + Given I authenticate using basic auth as "admin" + When I delete the LLM provider "llm-timeout-proxy-provider" + Then the response should be successful + + # Both the proxy and its backing provider set their own resilience.timeout. The request traverses + # both routes (proxy 6s -> loopback -> provider 2s -> backend /delay/10), so the shorter inner + # provider timeout (2s) must fire first and bound the whole call. The upper-bound assertion proves + # the 2s provider timeout won, not the 6s proxy timeout. + Scenario: LLM proxy and provider each set resilience and the shorter (provider) timeout wins + Given I authenticate using basic auth as "admin" + When I create this LLM provider: + """ + apiVersion: gateway.api-platform.wso2.com/v1 + kind: LlmProvider + metadata: + name: llm-timeout-both-provider + spec: + displayName: LLM Timeout Both Provider + version: v1.0 + template: openai + context: /llm-timeout-both-backing + upstream: + url: http://echo-backend:80 + accessControl: + mode: allow_all + resilience: + timeout: 2s + """ + Then the response status code should be 201 + When I deploy this LLM proxy configuration: + """ + apiVersion: gateway.api-platform.wso2.com/v1 + kind: LlmProxy + metadata: + name: llm-timeout-both-proxy + spec: + displayName: LLM Timeout Both Proxy + version: v1.0 + context: /llm-timeout-both + provider: + id: llm-timeout-both-provider + resilience: + timeout: 6s + """ + Then the response status code should be 201 + And I wait for the endpoint "http://localhost:8080/llm-timeout-both/get" to be ready + And I record the current time as "request_start" + When I send a GET request to "http://localhost:8080/llm-timeout-both/delay/10" + Then the response status code should be 504 + And the request should have taken at least "2" seconds since "request_start" + And the request should have taken at most "4" seconds since "request_start" + Given I authenticate using basic auth as "admin" + When I send a DELETE request to the "gateway-controller" service at "/llm-proxies/llm-timeout-both-proxy" + Then the response should be successful + Given I authenticate using basic auth as "admin" + When I delete the LLM provider "llm-timeout-both-provider" + Then the response should be successful diff --git a/gateway/it/features/upstream-connect-timeout.feature b/gateway/it/features/upstream-connect-timeout.feature new file mode 100644 index 0000000000..451ea0684d --- /dev/null +++ b/gateway/it/features/upstream-connect-timeout.feature @@ -0,0 +1,217 @@ +# -------------------------------------------------------------------- +# Copyright (c) 2025, WSO2 LLC. (https://www.wso2.com). +# +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# -------------------------------------------------------------------- + +@backend-timeout @timeouts +Feature: Timeouts + As an API developer + I want upstream (connect) and HTTP Connection Manager timeouts to be enforced by the gateway + So that requests to slow or unreachable backends, and slow downstream clients, + fail within the configured timeout + + # request_timeout, stream_idle_timeout and idle_timeout are not exercised here: + # small values would affect the whole shared suite + + Background: + Given the gateway services are running + + # Tests cluster connect_timeout: upstream does not accept TCP connection in time. + # Uses unreachable IP (192.0.2.1 per RFC 5737) so connect attempt hangs until connect_timeout. + Scenario: RestApi backend timeout using upstreamDefinitions + Given I authenticate using basic auth as "admin" + When I deploy this API configuration: + """ + apiVersion: gateway.api-platform.wso2.com/v1 + kind: RestApi + metadata: + name: timeout-api-v1.0 + spec: + displayName: Timeout-API + version: v1.0 + context: /timeout-api/$version + upstreamDefinitions: + - name: my-timeout-upstream + timeout: + connect: 6000ms + upstreams: + - url: http://192.0.2.1:8080 + upstream: + main: + ref: my-timeout-upstream + operations: + - method: GET + path: / + """ + Then the response should be successful + And the response should be valid JSON + And the JSON response field "status" should be "success" + And I wait for policy snapshot sync + And I record the current time as "request_start" + When I send a GET request to "http://localhost:8080/timeout-api/v1.0/" + Then the response status code should be 503 + And the request should have taken at least "6" seconds since "request_start" + Given I authenticate using basic auth as "admin" + When I delete the API "timeout-api-v1.0" + Then the response should be successful + + # Global-default scenario: route timeout comes from it config (6s); elapsed-time assertion verifies configured global timeout. + Scenario: RestApi without upstream timeout uses global defaults + Given I authenticate using basic auth as "admin" + When I deploy this API configuration: + """ + apiVersion: gateway.api-platform.wso2.com/v1 + kind: RestApi + metadata: + name: timeout-api-global-v1.0 + spec: + displayName: Timeout-API-Global + version: v1.0 + context: /timeout-global/$version + upstreamDefinitions: + - name: my-timeout-upstream-global + upstreams: + - url: http://192.0.2.1:8080 + upstream: + main: + ref: my-timeout-upstream-global + operations: + - method: GET + path: / + """ + Then the response should be successful + And the response should be valid JSON + And the JSON response field "status" should be "success" + And I wait for policy snapshot sync + And I record the current time as "request_start" + When I send a GET request to "http://localhost:8080/timeout-global/v1.0/" + Then the response status code should be 503 + And the request should have taken at least "5" seconds since "request_start" + Given I authenticate using basic auth as "admin" + When I delete the API "timeout-api-global-v1.0" + Then the response should be successful + + # Tests HCM request_headers_timeout (set to "5s" in it/test-config.toml). + # A raw client sends a partial request and never terminates the headers; the gateway + # must close the stream with 408 once request_headers_timeout elapses. + Scenario: HCM request_headers_timeout terminates a slow-header request + Given I authenticate using basic auth as "admin" + When I deploy this API configuration: + """ + apiVersion: gateway.api-platform.wso2.com/v1 + kind: RestApi + metadata: + name: headers-timeout-api-v1.0 + spec: + displayName: Headers-Timeout-API + version: v1.0 + context: /headers-timeout/$version + upstreamDefinitions: + - name: headers-timeout-upstream + upstreams: + - url: http://sample-backend:9080 + upstream: + main: + ref: headers-timeout-upstream + operations: + - method: GET + path: / + """ + Then the response should be successful + And the response should be valid JSON + And the JSON response field "status" should be "success" + And I record the current time as "request_start" + When I open a raw connection to "localhost:8080" and send incomplete request headers for path "/headers-timeout/v1.0/" + Then the raw response status code should be "408" + And the request should have taken at least "4" seconds since "request_start" + Given I authenticate using basic auth as "admin" + When I delete the API "headers-timeout-api-v1.0" + Then the response should be successful + + # LLM Provider connect_timeout via upstreamDefinitions ref: the provider's upstream references a + # definition whose only target is unreachable (192.0.2.1). The connect attempt hangs until the + # per-upstream connect timeout (6s), then the gateway returns 503. Proves the ref -> upstreamDefinition + # connect timeout works for LLM providers exactly as it does for RestApi. + Scenario: LLM provider backend connect timeout using upstreamDefinitions ref + Given I authenticate using basic auth as "admin" + When I create this LLM provider: + """ + apiVersion: gateway.api-platform.wso2.com/v1 + kind: LlmProvider + metadata: + name: llm-connect-timeout-provider + spec: + displayName: LLM Connect Timeout Provider + version: v1.0 + template: openai + context: /llm-connect-timeout + upstreamDefinitions: + - name: llm-unreachable-upstream + timeout: + connect: 6000ms + upstreams: + - url: http://192.0.2.1:8080 + upstream: + ref: llm-unreachable-upstream + accessControl: + mode: allow_all + """ + Then the response status code should be 201 + And I wait for policy snapshot sync + And I record the current time as "request_start" + When I send a GET request to "http://localhost:8080/llm-connect-timeout/get" + Then the response status code should be 503 + And the request should have taken at least "6" seconds since "request_start" + Given I authenticate using basic auth as "admin" + When I delete the LLM provider "llm-connect-timeout-provider" + Then the response should be successful + + # MCP connect_timeout via upstreamDefinitions ref: the MCP backend reference is unreachable, so the + # synthesized /mcp route's connect attempt hangs until the per-upstream connect timeout (6s) -> 503. + Scenario: MCP backend connect timeout using upstreamDefinitions ref + Given I authenticate using basic auth as "admin" + When I deploy this MCP configuration: + """ + apiVersion: gateway.api-platform.wso2.com/v1 + kind: Mcp + metadata: + name: mcp-connect-timeout + spec: + displayName: MCP Connect Timeout + version: v1.0 + context: /mcp-connect-timeout + specVersion: "2025-06-18" + upstreamDefinitions: + - name: mcp-unreachable-upstream + timeout: + connect: 6000ms + upstreams: + - url: http://192.0.2.1:3001 + upstream: + ref: mcp-unreachable-upstream + tools: [] + resources: [] + prompts: [] + """ + Then the response should be successful + And I wait for policy snapshot sync + And I record the current time as "request_start" + When I send a GET request to "http://localhost:8080/mcp-connect-timeout/mcp" + Then the response status code should be 503 + And the request should have taken at least "6" seconds since "request_start" + Given I authenticate using basic auth as "admin" + When I delete the MCP proxy "mcp-connect-timeout" + Then the response should be successful + diff --git a/gateway/it/steps_backend_timeout.go b/gateway/it/steps_backend_timeout.go deleted file mode 100644 index c5d8778acd..0000000000 --- a/gateway/it/steps_backend_timeout.go +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (c) 2025, WSO2 LLC. (https://www.wso2.com). - * - * WSO2 LLC. licenses this file to you under the Apache License, - * Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package it - -import ( - "fmt" - "strconv" - "time" - - "github.com/cucumber/godog" -) - -// elapsedTimeToleranceSeconds is the tolerance when asserting minimum elapsed time (e.g. clock skew, scheduling). -const elapsedTimeToleranceSeconds = 1 - -// RegisterBackendTimeoutSteps registers step definitions for backend timeout scenarios -func RegisterBackendTimeoutSteps(ctx *godog.ScenarioContext, state *TestState) { - ctx.Step(`^I record the current time as "([^"]*)"$`, func(key string) error { - state.SetContextValue(key, time.Now()) - return nil - }) - - ctx.Step(`^the request should have taken at least "(\d+)" seconds since "([^"]*)"$`, func(expectedSecondsStr, key string) error { - expectedSeconds, err := strconv.Atoi(expectedSecondsStr) - if err != nil { - return fmt.Errorf("expected seconds must be a number, got: %s", expectedSecondsStr) - } - val, ok := state.GetContextValue(key) - if !ok { - return fmt.Errorf("no start time recorded; record the current time as \"request_start\" before sending the request") - } - start, ok := val.(time.Time) - if !ok { - return fmt.Errorf("no start time recorded; record the current time as %q before sending the request", key) - } - elapsed := time.Since(start) - minElapsed := time.Duration(expectedSeconds-elapsedTimeToleranceSeconds) * time.Second - if elapsed < minElapsed { - return fmt.Errorf("request should have taken at least %d seconds (with %ds tolerance), but elapsed time was %s", - expectedSeconds, elapsedTimeToleranceSeconds, elapsed.Round(time.Millisecond)) - } - return nil - }) -} diff --git a/gateway/it/steps_timeouts.go b/gateway/it/steps_timeouts.go new file mode 100644 index 0000000000..f8bea39830 --- /dev/null +++ b/gateway/it/steps_timeouts.go @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2025, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package it + +import ( + "fmt" + "io" + "net" + "strconv" + "strings" + "time" + + "github.com/cucumber/godog" +) + +// elapsedTimeToleranceSeconds is the tolerance when asserting minimum elapsed time (e.g. clock skew, scheduling). +const elapsedTimeToleranceSeconds = 1 + +// rawResponseContextKey is where the slow-header raw response is stashed for later assertion. +const rawResponseContextKey = "raw_timeout_response" + +// RegisterTimeoutSteps registers step definitions for upstream and HCM timeout scenarios +func RegisterTimeoutSteps(ctx *godog.ScenarioContext, state *TestState) { + ctx.Step(`^I record the current time as "([^"]*)"$`, func(key string) error { + state.SetContextValue(key, time.Now()) + return nil + }) + + ctx.Step(`^the request should have taken at least "(\d+)" seconds since "([^"]*)"$`, func(expectedSecondsStr, key string) error { + expectedSeconds, err := strconv.Atoi(expectedSecondsStr) + if err != nil { + return fmt.Errorf("expected seconds must be a number, got: %s", expectedSecondsStr) + } + val, ok := state.GetContextValue(key) + if !ok { + return fmt.Errorf("no start time recorded; record the current time as \"request_start\" before sending the request") + } + start, ok := val.(time.Time) + if !ok { + return fmt.Errorf("no start time recorded; record the current time as %q before sending the request", key) + } + elapsed := time.Since(start) + minElapsed := time.Duration(expectedSeconds-elapsedTimeToleranceSeconds) * time.Second + if elapsed < minElapsed { + return fmt.Errorf("request should have taken at least %d seconds (with %ds tolerance), but elapsed time was %s", + expectedSeconds, elapsedTimeToleranceSeconds, elapsed.Round(time.Millisecond)) + } + return nil + }) + + ctx.Step(`^the request should have taken at most "(\d+)" seconds since "([^"]*)"$`, func(expectedSecondsStr, key string) error { + expectedSeconds, err := strconv.Atoi(expectedSecondsStr) + if err != nil { + return fmt.Errorf("expected seconds must be a number, got: %s", expectedSecondsStr) + } + val, ok := state.GetContextValue(key) + if !ok { + return fmt.Errorf("no start time recorded; record the current time as \"request_start\" before sending the request") + } + start, ok := val.(time.Time) + if !ok { + return fmt.Errorf("no start time recorded; record the current time as %q before sending the request", key) + } + elapsed := time.Since(start) + maxElapsed := time.Duration(expectedSeconds+elapsedTimeToleranceSeconds) * time.Second + if elapsed > maxElapsed { + return fmt.Errorf("request should have taken at most %d seconds (with %ds tolerance), but elapsed time was %s", + expectedSeconds, elapsedTimeToleranceSeconds, elapsed.Round(time.Millisecond)) + } + return nil + }) + + // Opens a raw TCP connection and sends a request line plus one header, but never + // terminates the header block (no final blank line). This forces the HCM + // request_headers_timeout to fire. The connection blocks until the gateway responds + // or closes it, and the raw response is stored for assertion. + ctx.Step(`^I open a raw connection to "([^"]*)" and send incomplete request headers for path "([^"]*)"$`, func(address, path string) error { + conn, err := net.DialTimeout("tcp", address, 10*time.Second) + if err != nil { + return fmt.Errorf("failed to connect to %s: %w", address, err) + } + defer conn.Close() + + // Request line + Host header, with NO terminating CRLF that ends the headers. + partial := fmt.Sprintf("GET %s HTTP/1.1\r\nHost: %s\r\n", path, address) + if _, err := conn.Write([]byte(partial)); err != nil { + return fmt.Errorf("failed to send partial request: %w", err) + } + + // Read until the gateway responds and closes, bounded by a deadline that is + // comfortably longer than request_headers_timeout so we capture the 408. + if err := conn.SetReadDeadline(time.Now().Add(30 * time.Second)); err != nil { + return fmt.Errorf("failed to set read deadline: %w", err) + } + raw, err := io.ReadAll(conn) + if err != nil && len(raw) == 0 { + return fmt.Errorf("failed to read response from gateway: %w", err) + } + state.SetContextValue(rawResponseContextKey, string(raw)) + return nil + }) + + ctx.Step(`^the raw response status code should be "([^"]*)"$`, func(expectedCode string) error { + val, ok := state.GetContextValue(rawResponseContextKey) + if !ok { + return fmt.Errorf("no raw response recorded; open a raw connection first") + } + raw, ok := val.(string) + if !ok { + return fmt.Errorf("recorded raw response has unexpected type %T", val) + } + statusLine := raw + if idx := strings.Index(raw, "\r\n"); idx >= 0 { + statusLine = raw[:idx] + } + if !strings.Contains(statusLine, expectedCode) { + return fmt.Errorf("expected raw response status line to contain %q, got %q", expectedCode, statusLine) + } + return nil + }) +} diff --git a/gateway/it/suite_test.go b/gateway/it/suite_test.go index 4cbcf0338e..d994bd8e6e 100644 --- a/gateway/it/suite_test.go +++ b/gateway/it/suite_test.go @@ -143,6 +143,9 @@ func getFeaturePaths() []string { "features/route-path-matching.feature", "features/secrets.feature", "features/template-functions.feature", + "features/upstream-connect-timeout.feature", + "features/backend-timeout.feature", + "features/llm-backend-timeout.feature", // Runs late: it restarts the gateway-controller (reject/reconnect scenario), so keep it // after features that assume an uninterrupted controller. Verifies the DP->CP artifact push. "features/dp-to-cp.feature", @@ -339,7 +342,7 @@ func InitializeScenario(ctx *godog.ScenarioContext) { RegisterMetricsSteps(ctx, testState, httpSteps) RegisterAuthSteps(ctx, testState, httpSteps) RegisterAPISteps(ctx, testState, httpSteps) - RegisterBackendTimeoutSteps(ctx, testState) + RegisterTimeoutSteps(ctx, testState) RegisterMCPSteps(ctx, testState, httpSteps, jwtSteps) RegisterLLMSteps(ctx, testState, httpSteps) RegisterJWTSteps(ctx, testState, httpSteps, jwtSteps) diff --git a/gateway/it/test-config.toml b/gateway/it/test-config.toml index 187d32390f..4833aee646 100644 --- a/gateway/it/test-config.toml +++ b/gateway/it/test-config.toml @@ -72,6 +72,10 @@ enabled = true enabled = true format = "text" +[router.http_listener.timeouts] +# This is the only HCM timeout set to a small, testable value here. +request_headers_timeout = "5s" + # ============================================================================= # POLICY ENGINE CONFIGURATION # ============================================================================= diff --git a/kubernetes/gateway-operator/api/v1alpha1/llmprovider_types.go b/kubernetes/gateway-operator/api/v1alpha1/llmprovider_types.go index 5d2fae5583..9b1c0be98e 100644 --- a/kubernetes/gateway-operator/api/v1alpha1/llmprovider_types.go +++ b/kubernetes/gateway-operator/api/v1alpha1/llmprovider_types.go @@ -107,6 +107,11 @@ type LLMProviderConfigData struct { // +kubebuilder:validation:Required AccessControl LLMAccessControl `json:"accessControl"` + // UpstreamDefinitions is the list of reusable upstream definitions (with optional + // connect timeout) that upstream.ref can reference. + // +optional + UpstreamDefinitions []UpstreamDefinition `json:"upstreamDefinitions,omitempty"` + // Upstream configures the LLM upstream. // +kubebuilder:validation:Required Upstream LLMProviderUpstream `json:"upstream"` @@ -128,6 +133,11 @@ type LLMProviderConfigData struct { // Policies is the list of policies applied to this LLM provider. // +optional Policies []LLMPolicy `json:"policies,omitempty"` + + // Resilience configures API-level backend/route timeouts applied to all routes + // generated for this LLM provider. Supported at the API level only. + // +optional + Resilience *Resilience `json:"resilience,omitempty"` } // LLMPolicyPath defines a path/methods combination together with policy diff --git a/kubernetes/gateway-operator/api/v1alpha1/llmproxy_types.go b/kubernetes/gateway-operator/api/v1alpha1/llmproxy_types.go index 046f492af5..62aac873c9 100644 --- a/kubernetes/gateway-operator/api/v1alpha1/llmproxy_types.go +++ b/kubernetes/gateway-operator/api/v1alpha1/llmproxy_types.go @@ -64,6 +64,11 @@ type LLMProxyConfigData struct { // Policies is the list of policies applied to this LLM proxy. // +optional Policies []LLMPolicy `json:"policies,omitempty"` + + // Resilience configures API-level backend/route timeouts applied to all routes + // generated for this LLM proxy. Supported at the API level only. + // +optional + Resilience *Resilience `json:"resilience,omitempty"` } //+kubebuilder:object:root=true diff --git a/kubernetes/gateway-operator/api/v1alpha1/mcp_types.go b/kubernetes/gateway-operator/api/v1alpha1/mcp_types.go index 8044dd8c79..bdb4f24884 100644 --- a/kubernetes/gateway-operator/api/v1alpha1/mcp_types.go +++ b/kubernetes/gateway-operator/api/v1alpha1/mcp_types.go @@ -156,6 +156,11 @@ type MCPProxyConfigData struct { // +kubebuilder:validation:Pattern=`^v?([0-9]+)(\.[0-9]+)?(\.[0-9]+)?(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$` Version string `json:"version"` + // UpstreamDefinitions is the list of reusable upstream definitions (with optional + // connect timeout) that upstream.ref can reference. + // +optional + UpstreamDefinitions []UpstreamDefinition `json:"upstreamDefinitions,omitempty"` + // Upstream is the MCP backend. // +kubebuilder:validation:Required Upstream MCPUpstream `json:"upstream"` @@ -193,6 +198,12 @@ type MCPProxyConfigData struct { // Policies are MCP proxy-level policies. // +optional Policies []Policy `json:"policies,omitempty"` + + // Resilience configures API-level backend/route timeouts applied to the traffic-forwarding + // routes generated for this MCP proxy. Supported at the API level only. Because MCP transports + // are long-lived streams, the route timeout defaults to disabled ("0s") for MCP when unset. + // +optional + Resilience *Resilience `json:"resilience,omitempty"` } //+kubebuilder:object:root=true diff --git a/kubernetes/gateway-operator/api/v1alpha1/restapi_types.go b/kubernetes/gateway-operator/api/v1alpha1/restapi_types.go index 6077eb981f..220d2f97c1 100644 --- a/kubernetes/gateway-operator/api/v1alpha1/restapi_types.go +++ b/kubernetes/gateway-operator/api/v1alpha1/restapi_types.go @@ -74,6 +74,16 @@ type APIConfigData struct { // +optional Policies []Policy `json:"policies,omitempty"` + // Resilience API-level backend/route timeout configuration applied to all operations + // unless overridden at the operation level + // +optional + Resilience *Resilience `json:"resilience,omitempty"` + + // UpstreamDefinitions is the list of reusable upstream definitions (with optional connect + // timeout) that upstream.ref can reference. + // +optional + UpstreamDefinitions []UpstreamDefinition `json:"upstreamDefinitions,omitempty"` + // Upstream API-level upstream configuration // +kubebuilder:validation:Required Upstream UpstreamConfig `json:"upstream"` @@ -125,6 +135,26 @@ type Operation struct { // Policies List of policies applied only to this operation (overrides or adds to API-level policies) // +optional Policies []Policy `json:"policies,omitempty"` + + // Resilience Operation-level backend/route timeout configuration (overrides API-level) + // +optional + Resilience *Resilience `json:"resilience,omitempty"` +} + +// Resilience defines backend/route timeout configuration (maps to Envoy RouteAction +// timeouts). Settable at the API level (applies to all routes) and/or the operation level +// (overrides the API level). "0s" disables a timeout; unset falls back to the gateway's +// global route timeout defaults. +type Resilience struct { + // Timeout Maximum time for the entire route (request to upstream response). "0s" disables. + // +optional + // +kubebuilder:validation:Pattern=`^\d+(\.\d+)?(ms|s|m|h)$` + Timeout *string `json:"timeout,omitempty"` + + // IdleTimeout Per-route stream idle timeout. "0s" disables. + // +optional + // +kubebuilder:validation:Pattern=`^\d+(\.\d+)?(ms|s|m|h)$` + IdleTimeout *string `json:"idleTimeout,omitempty"` } // OperationMethod HTTP method @@ -169,12 +199,69 @@ type Policy struct { Version string `json:"version"` } -// Upstream defines model for Upstream. +// Upstream defines model for Upstream. Exactly one of url or ref must be set: a direct backend URL, +// or a reference to a predefined upstreamDefinition (which can carry a per-upstream connect timeout). +// +kubebuilder:validation:XValidation:rule="has(self.url) != has(self.ref)",message="exactly one of url or ref must be set" type Upstream struct { - // Url Backend service URL (may include path prefix like /api/v2) + // Url Direct backend service URL (may include path prefix like /api/v2) + // +optional + // +kubebuilder:validation:Pattern=`^https?://[a-zA-Z0-9\-._~:/?#\[\]@!$&'()*+,;=%]+$` + Url *string `json:"url,omitempty"` + + // Ref Name of a predefined upstreamDefinition to route through. + // +optional + Ref *string `json:"ref,omitempty"` +} + +// UpstreamDefinition is a reusable upstream configuration with an optional connect timeout and +// load-balancing targets. Referenced from an upstream via its `ref` field. Shared by RestApi, +// LLM Provider, and MCP; mirrors the management-API UpstreamDefinition schema. +type UpstreamDefinition struct { + // Name Unique identifier for this upstream definition (referenced by upstream.ref). + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=100 + // +kubebuilder:validation:Pattern=`^[a-zA-Z0-9\-_]+$` + Name string `json:"name"` + + // BasePath Base path prefix prepended to all requests routed through this upstream (e.g. /api/v2). + // Must start with "/" and must not end with "/". Omit for root ("/"). + // +optional + // +kubebuilder:validation:Pattern=`^/[a-zA-Z0-9\-._~!$&'()*+,;=:@%/]*[^/]$` + BasePath *string `json:"basePath,omitempty"` + + // Timeout Optional timeout configuration for this upstream (connect timeout). + // +optional + Timeout *UpstreamTimeout `json:"timeout,omitempty"` + + // Upstreams List of backend targets with optional weights for load balancing. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinItems=1 + Upstreams []UpstreamTarget `json:"upstreams"` +} + +// UpstreamTimeout carries the per-upstream timeout configuration. Only the connect timeout is +// supported at the upstream-definition level. +type UpstreamTimeout struct { + // Connect Connection-establishment timeout duration (e.g. "5s", "500ms"). "0s" disables. + // +optional + // +kubebuilder:validation:Pattern=`^\d+(\.\d+)?(ms|s|m|h)$` + Connect *string `json:"connect,omitempty"` +} + +// UpstreamTarget is a single backend target within an UpstreamDefinition. +type UpstreamTarget struct { + // Url Backend URL (host and port only; path comes from the definition's basePath). // +kubebuilder:validation:Required // +kubebuilder:validation:Pattern=`^https?://[a-zA-Z0-9\-._~:/?#\[\]@!$&'()*+,;=%]+$` Url string `json:"url"` + + // Weight Relative weight for load balancing across multiple upstream targets. Reserved for + // future multi-target load balancing; not applied yet (only the first target is currently used). + // +optional + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=100 + Weight *int `json:"weight,omitempty"` } // Condition Types for RestApi diff --git a/kubernetes/gateway-operator/api/v1alpha1/zz_generated.deepcopy.go b/kubernetes/gateway-operator/api/v1alpha1/zz_generated.deepcopy.go index b6163d5fd5..62ce0b9bd5 100644 --- a/kubernetes/gateway-operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/kubernetes/gateway-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -43,6 +43,18 @@ func (in *APIConfigData) DeepCopyInto(out *APIConfigData) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Resilience != nil { + in, out := &in.Resilience, &out.Resilience + *out = new(Resilience) + (*in).DeepCopyInto(*out) + } + if in.UpstreamDefinitions != nil { + in, out := &in.UpstreamDefinitions, &out.UpstreamDefinitions + *out = make([]UpstreamDefinition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } in.Upstream.DeepCopyInto(&out.Upstream) if in.Vhosts != nil { in, out := &in.Vhosts, &out.Vhosts @@ -774,6 +786,13 @@ func (in *LLMPolicyPath) DeepCopy() *LLMPolicyPath { func (in *LLMProviderConfigData) DeepCopyInto(out *LLMProviderConfigData) { *out = *in in.AccessControl.DeepCopyInto(&out.AccessControl) + if in.UpstreamDefinitions != nil { + in, out := &in.UpstreamDefinitions, &out.UpstreamDefinitions + *out = make([]UpstreamDefinition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } in.Upstream.DeepCopyInto(&out.Upstream) if in.Context != nil { in, out := &in.Context, &out.Context @@ -797,6 +816,11 @@ func (in *LLMProviderConfigData) DeepCopyInto(out *LLMProviderConfigData) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Resilience != nil { + in, out := &in.Resilience, &out.Resilience + *out = new(Resilience) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LLMProviderConfigData. @@ -987,6 +1011,11 @@ func (in *LLMProxyConfigData) DeepCopyInto(out *LLMProxyConfigData) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Resilience != nil { + in, out := &in.Resilience, &out.Resilience + *out = new(Resilience) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LLMProxyConfigData. @@ -1282,6 +1311,13 @@ func (in *MCPPromptArgument) DeepCopy() *MCPPromptArgument { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MCPProxyConfigData) DeepCopyInto(out *MCPProxyConfigData) { *out = *in + if in.UpstreamDefinitions != nil { + in, out := &in.UpstreamDefinitions, &out.UpstreamDefinitions + *out = make([]UpstreamDefinition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } in.Upstream.DeepCopyInto(&out.Upstream) if in.Context != nil { in, out := &in.Context, &out.Context @@ -1331,6 +1367,11 @@ func (in *MCPProxyConfigData) DeepCopyInto(out *MCPProxyConfigData) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Resilience != nil { + in, out := &in.Resilience, &out.Resilience + *out = new(Resilience) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MCPProxyConfigData. @@ -1587,6 +1628,11 @@ func (in *Operation) DeepCopyInto(out *Operation) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Resilience != nil { + in, out := &in.Resilience, &out.Resilience + *out = new(Resilience) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Operation. @@ -1624,6 +1670,31 @@ func (in *Policy) DeepCopy() *Policy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Resilience) DeepCopyInto(out *Resilience) { + *out = *in + if in.Timeout != nil { + in, out := &in.Timeout, &out.Timeout + *out = new(string) + **out = **in + } + if in.IdleTimeout != nil { + in, out := &in.IdleTimeout, &out.IdleTimeout + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Resilience. +func (in *Resilience) DeepCopy() *Resilience { + if in == nil { + return nil + } + out := new(Resilience) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ResourceStatus) DeepCopyInto(out *ResourceStatus) { *out = *in @@ -2007,6 +2078,16 @@ func (in *SubscriptionSpec) DeepCopy() *SubscriptionSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Upstream) DeepCopyInto(out *Upstream) { *out = *in + if in.Url != nil { + in, out := &in.Url, &out.Url + *out = new(string) + **out = **in + } + if in.Ref != nil { + in, out := &in.Ref, &out.Ref + *out = new(string) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Upstream. @@ -2022,11 +2103,11 @@ func (in *Upstream) DeepCopy() *Upstream { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *UpstreamConfig) DeepCopyInto(out *UpstreamConfig) { *out = *in - out.Main = in.Main + in.Main.DeepCopyInto(&out.Main) if in.Sandbox != nil { in, out := &in.Sandbox, &out.Sandbox *out = new(Upstream) - **out = **in + (*in).DeepCopyInto(*out) } } @@ -2040,6 +2121,78 @@ func (in *UpstreamConfig) DeepCopy() *UpstreamConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UpstreamDefinition) DeepCopyInto(out *UpstreamDefinition) { + *out = *in + if in.BasePath != nil { + in, out := &in.BasePath, &out.BasePath + *out = new(string) + **out = **in + } + if in.Timeout != nil { + in, out := &in.Timeout, &out.Timeout + *out = new(UpstreamTimeout) + (*in).DeepCopyInto(*out) + } + if in.Upstreams != nil { + in, out := &in.Upstreams, &out.Upstreams + *out = make([]UpstreamTarget, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UpstreamDefinition. +func (in *UpstreamDefinition) DeepCopy() *UpstreamDefinition { + if in == nil { + return nil + } + out := new(UpstreamDefinition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UpstreamTarget) DeepCopyInto(out *UpstreamTarget) { + *out = *in + if in.Weight != nil { + in, out := &in.Weight, &out.Weight + *out = new(int) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UpstreamTarget. +func (in *UpstreamTarget) DeepCopy() *UpstreamTarget { + if in == nil { + return nil + } + out := new(UpstreamTarget) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UpstreamTimeout) DeepCopyInto(out *UpstreamTimeout) { + *out = *in + if in.Connect != nil { + in, out := &in.Connect, &out.Connect + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UpstreamTimeout. +func (in *UpstreamTimeout) DeepCopy() *UpstreamTimeout { + if in == nil { + return nil + } + out := new(UpstreamTimeout) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VhostConfig) DeepCopyInto(out *VhostConfig) { *out = *in diff --git a/kubernetes/gateway-operator/config/crd/bases/gateway.api-platform.wso2.com_llmproviders.yaml b/kubernetes/gateway-operator/config/crd/bases/gateway.api-platform.wso2.com_llmproviders.yaml index 8d48f6cbe8..3cc7e1366c 100644 --- a/kubernetes/gateway-operator/config/crd/bases/gateway.api-platform.wso2.com_llmproviders.yaml +++ b/kubernetes/gateway-operator/config/crd/bases/gateway.api-platform.wso2.com_llmproviders.yaml @@ -153,6 +153,21 @@ spec: - version type: object type: array + resilience: + description: |- + Resilience configures API-level backend/route timeouts applied to all routes + generated for this LLM provider. Supported at the API level only. + properties: + idleTimeout: + description: IdleTimeout Per-route stream idle timeout. "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + timeout: + description: Timeout Maximum time for the entire route (request + to upstream response). "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + type: object template: description: Template is the LlmProviderTemplate name to apply. type: string @@ -231,6 +246,68 @@ spec: description: Url is the direct backend URL. type: string type: object + upstreamDefinitions: + description: |- + UpstreamDefinitions is the list of reusable upstream definitions (with optional + connect timeout) that upstream.ref can reference. + items: + description: |- + UpstreamDefinition is a reusable upstream configuration with an optional connect timeout and + load-balancing targets. Referenced from an upstream via its `ref` field. Shared by RestApi, + LLM Provider, and MCP; mirrors the management-API UpstreamDefinition schema. + properties: + basePath: + description: |- + BasePath Base path prefix prepended to all requests routed through this upstream (e.g. /api/v2). + Must start with "/" and must not end with "/". Omit for root ("/"). + pattern: ^/[a-zA-Z0-9\-._~!$&'()*+,;=:@%/]*[^/]$ + type: string + name: + description: Name Unique identifier for this upstream definition + (referenced by upstream.ref). + maxLength: 100 + minLength: 1 + pattern: ^[a-zA-Z0-9\-_]+$ + type: string + timeout: + description: Timeout Optional timeout configuration for this + upstream (connect timeout). + properties: + connect: + description: Connect Connection-establishment timeout duration + (e.g. "5s", "500ms"). "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + type: object + upstreams: + description: Upstreams List of backend targets with optional + weights for load balancing. + items: + description: UpstreamTarget is a single backend target within + an UpstreamDefinition. + properties: + url: + description: Url Backend URL (host and port only; path + comes from the definition's basePath). + pattern: ^https?://[a-zA-Z0-9\-._~:/?#\[\]@!$&'()*+,;=%]+$ + type: string + weight: + description: |- + Weight Relative weight for load balancing across multiple upstream targets. Reserved for + future multi-target load balancing; not applied yet (only the first target is currently used). + maximum: 100 + minimum: 0 + type: integer + required: + - url + type: object + minItems: 1 + type: array + required: + - name + - upstreams + type: object + type: array version: description: Version is the semantic version of the LLM provider. pattern: ^v?([0-9]+)(\.[0-9]+)?(\.[0-9]+)?(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$ diff --git a/kubernetes/gateway-operator/config/crd/bases/gateway.api-platform.wso2.com_llmproxies.yaml b/kubernetes/gateway-operator/config/crd/bases/gateway.api-platform.wso2.com_llmproxies.yaml index dd5a2a5579..81da02ebd1 100644 --- a/kubernetes/gateway-operator/config/crd/bases/gateway.api-platform.wso2.com_llmproxies.yaml +++ b/kubernetes/gateway-operator/config/crd/bases/gateway.api-platform.wso2.com_llmproxies.yaml @@ -180,6 +180,21 @@ spec: required: - id type: object + resilience: + description: |- + Resilience configures API-level backend/route timeouts applied to all routes + generated for this LLM proxy. Supported at the API level only. + properties: + idleTimeout: + description: IdleTimeout Per-route stream idle timeout. "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + timeout: + description: Timeout Maximum time for the entire route (request + to upstream response). "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + type: object version: description: Version is the semantic version of the LLM proxy. pattern: ^v?([0-9]+)(\.[0-9]+)?(\.[0-9]+)?(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$ diff --git a/kubernetes/gateway-operator/config/crd/bases/gateway.api-platform.wso2.com_mcps.yaml b/kubernetes/gateway-operator/config/crd/bases/gateway.api-platform.wso2.com_mcps.yaml index e6d3402a6c..096fac435a 100644 --- a/kubernetes/gateway-operator/config/crd/bases/gateway.api-platform.wso2.com_mcps.yaml +++ b/kubernetes/gateway-operator/config/crd/bases/gateway.api-platform.wso2.com_mcps.yaml @@ -122,6 +122,22 @@ spec: - name type: object type: array + resilience: + description: |- + Resilience configures API-level backend/route timeouts applied to the traffic-forwarding + routes generated for this MCP proxy. Supported at the API level only. Because MCP transports + are long-lived streams, the route timeout defaults to disabled ("0s") for MCP when unset. + properties: + idleTimeout: + description: IdleTimeout Per-route stream idle timeout. "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + timeout: + description: Timeout Maximum time for the entire route (request + to upstream response). "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + type: object resources: description: Resources lists optional MCP resources exposed by this proxy. @@ -254,6 +270,68 @@ spec: description: Url is the direct backend URL. type: string type: object + upstreamDefinitions: + description: |- + UpstreamDefinitions is the list of reusable upstream definitions (with optional + connect timeout) that upstream.ref can reference. + items: + description: |- + UpstreamDefinition is a reusable upstream configuration with an optional connect timeout and + load-balancing targets. Referenced from an upstream via its `ref` field. Shared by RestApi, + LLM Provider, and MCP; mirrors the management-API UpstreamDefinition schema. + properties: + basePath: + description: |- + BasePath Base path prefix prepended to all requests routed through this upstream (e.g. /api/v2). + Must start with "/" and must not end with "/". Omit for root ("/"). + pattern: ^/[a-zA-Z0-9\-._~!$&'()*+,;=:@%/]*[^/]$ + type: string + name: + description: Name Unique identifier for this upstream definition + (referenced by upstream.ref). + maxLength: 100 + minLength: 1 + pattern: ^[a-zA-Z0-9\-_]+$ + type: string + timeout: + description: Timeout Optional timeout configuration for this + upstream (connect timeout). + properties: + connect: + description: Connect Connection-establishment timeout duration + (e.g. "5s", "500ms"). "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + type: object + upstreams: + description: Upstreams List of backend targets with optional + weights for load balancing. + items: + description: UpstreamTarget is a single backend target within + an UpstreamDefinition. + properties: + url: + description: Url Backend URL (host and port only; path + comes from the definition's basePath). + pattern: ^https?://[a-zA-Z0-9\-._~:/?#\[\]@!$&'()*+,;=%]+$ + type: string + weight: + description: |- + Weight Relative weight for load balancing across multiple upstream targets. Reserved for + future multi-target load balancing; not applied yet (only the first target is currently used). + maximum: 100 + minimum: 0 + type: integer + required: + - url + type: object + minItems: 1 + type: array + required: + - name + - upstreams + type: object + type: array version: description: Version is the MCP proxy semantic version. pattern: ^v?([0-9]+)(\.[0-9]+)?(\.[0-9]+)?(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$ diff --git a/kubernetes/gateway-operator/config/crd/bases/gateway.api-platform.wso2.com_restapis.yaml b/kubernetes/gateway-operator/config/crd/bases/gateway.api-platform.wso2.com_restapis.yaml index 3d1ad1a2aa..1193084898 100644 --- a/kubernetes/gateway-operator/config/crd/bases/gateway.api-platform.wso2.com_restapis.yaml +++ b/kubernetes/gateway-operator/config/crd/bases/gateway.api-platform.wso2.com_restapis.yaml @@ -98,6 +98,21 @@ spec: - version type: object type: array + resilience: + description: Resilience Operation-level backend/route timeout + configuration (overrides API-level) + properties: + idleTimeout: + description: IdleTimeout Per-route stream idle timeout. + "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + timeout: + description: Timeout Maximum time for the entire route (request + to upstream response). "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + type: object required: - method - path @@ -132,6 +147,21 @@ spec: - version type: object type: array + resilience: + description: |- + Resilience API-level backend/route timeout configuration applied to all operations + unless overridden at the operation level + properties: + idleTimeout: + description: IdleTimeout Per-route stream idle timeout. "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + timeout: + description: Timeout Maximum time for the entire route (request + to upstream response). "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + type: object upstream: description: Upstream API-level upstream configuration properties: @@ -139,29 +169,101 @@ spec: description: Main Upstream backend configuration for production traffic properties: + ref: + description: Ref Name of a predefined upstreamDefinition to + route through. + type: string url: - description: Url Backend service URL (may include path prefix - like /api/v2) + description: Url Direct backend service URL (may include path + prefix like /api/v2) pattern: ^https?://[a-zA-Z0-9\-._~:/?#\[\]@!$&'()*+,;=%]+$ type: string - required: - - url type: object + x-kubernetes-validations: + - message: exactly one of url or ref must be set + rule: has(self.url) != has(self.ref) sandbox: description: Sandbox Upstream backend configuration for sandbox/testing traffic properties: + ref: + description: Ref Name of a predefined upstreamDefinition to + route through. + type: string url: - description: Url Backend service URL (may include path prefix - like /api/v2) + description: Url Direct backend service URL (may include path + prefix like /api/v2) pattern: ^https?://[a-zA-Z0-9\-._~:/?#\[\]@!$&'()*+,;=%]+$ type: string - required: - - url type: object + x-kubernetes-validations: + - message: exactly one of url or ref must be set + rule: has(self.url) != has(self.ref) required: - main type: object + upstreamDefinitions: + description: |- + UpstreamDefinitions is the list of reusable upstream definitions (with optional connect + timeout) that upstream.ref can reference. + items: + description: |- + UpstreamDefinition is a reusable upstream configuration with an optional connect timeout and + load-balancing targets. Referenced from an upstream via its `ref` field. Shared by RestApi, + LLM Provider, and MCP; mirrors the management-API UpstreamDefinition schema. + properties: + basePath: + description: |- + BasePath Base path prefix prepended to all requests routed through this upstream (e.g. /api/v2). + Must start with "/" and must not end with "/". Omit for root ("/"). + pattern: ^/[a-zA-Z0-9\-._~!$&'()*+,;=:@%/]*[^/]$ + type: string + name: + description: Name Unique identifier for this upstream definition + (referenced by upstream.ref). + maxLength: 100 + minLength: 1 + pattern: ^[a-zA-Z0-9\-_]+$ + type: string + timeout: + description: Timeout Optional timeout configuration for this + upstream (connect timeout). + properties: + connect: + description: Connect Connection-establishment timeout duration + (e.g. "5s", "500ms"). "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + type: object + upstreams: + description: Upstreams List of backend targets with optional + weights for load balancing. + items: + description: UpstreamTarget is a single backend target within + an UpstreamDefinition. + properties: + url: + description: Url Backend URL (host and port only; path + comes from the definition's basePath). + pattern: ^https?://[a-zA-Z0-9\-._~:/?#\[\]@!$&'()*+,;=%]+$ + type: string + weight: + description: |- + Weight Relative weight for load balancing across multiple upstream targets. Reserved for + future multi-target load balancing; not applied yet (only the first target is currently used). + maximum: 100 + minimum: 0 + type: integer + required: + - url + type: object + minItems: 1 + type: array + required: + - name + - upstreams + type: object + type: array version: description: Version Semantic version of the API pattern: ^v?([0-9]+)(\.[0-9]+)?(\.[0-9]+)?(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$ diff --git a/kubernetes/gateway-operator/config/samples/api_v1_restapi.yaml b/kubernetes/gateway-operator/config/samples/api_v1_restapi.yaml index 750e88cf08..29286a9dbf 100644 --- a/kubernetes/gateway-operator/config/samples/api_v1_restapi.yaml +++ b/kubernetes/gateway-operator/config/samples/api_v1_restapi.yaml @@ -21,11 +21,18 @@ spec: # params: # issuers: # - WSO2KeyManager1 + # API-level resilience applies to all operations unless overridden per operation. + resilience: + timeout: 15s + idleTimeout: 0s operations: - method: GET path: /info - method: POST path: /submit + # Operation-level resilience overrides the API-level values for this route. + resilience: + timeout: 5s - method: GET path: /helloworld - method: GET @@ -63,4 +70,30 @@ spec: - method: GET path: /helloworld - method: GET - path: /heyhelloworld \ No newline at end of file + path: /heyhelloworld + + +--- +# Example 3: API routing through a reusable upstreamDefinition with a per-upstream connect timeout. +# The upstream references the definition by name (url XOR ref); the connect timeout bounds TCP +# connection establishment to the backend. +apiVersion: gateway.api-platform.wso2.com/v1alpha1 +kind: RestApi +metadata: + name: api-with-upstream-definition +spec: + displayName: test-api-upstreamdef + version: v1.0 + context: /orders + upstreamDefinitions: + - name: orders-backend + timeout: + connect: 6s + upstreams: + - url: http://orders.default.svc.cluster.local:8080 + upstream: + main: + ref: orders-backend + operations: + - method: GET + path: /info \ No newline at end of file diff --git a/kubernetes/gateway-operator/internal/controller/httproute_mapper.go b/kubernetes/gateway-operator/internal/controller/httproute_mapper.go index e760f3f97a..218142a131 100644 --- a/kubernetes/gateway-operator/internal/controller/httproute_mapper.go +++ b/kubernetes/gateway-operator/internal/controller/httproute_mapper.go @@ -257,7 +257,7 @@ func BuildAPIConfigFromHTTPRoute(ctx context.Context, c client.Client, route *ga DisplayName: displayName, Operations: ops, Upstream: apiv1.UpstreamConfig{ - Main: apiv1.Upstream{Url: backendURL}, + Main: apiv1.Upstream{Url: &backendURL}, }, Version: version, Policies: apiPolicies, diff --git a/kubernetes/gateway-operator/internal/controller/httproute_mapper_test.go b/kubernetes/gateway-operator/internal/controller/httproute_mapper_test.go index a2de552ecd..a82ad7d2c8 100644 --- a/kubernetes/gateway-operator/internal/controller/httproute_mapper_test.go +++ b/kubernetes/gateway-operator/internal/controller/httproute_mapper_test.go @@ -89,7 +89,7 @@ func TestBuildAPIConfigFromHTTPRoute(t *testing.T) { require.Len(t, spec.Operations, 1) // PathPrefix "/api/hello" must emit a prefix route ("/*" suffix), not an Exact path. require.Equal(t, "/api/hello/*", spec.Operations[0].Path) - require.Equal(t, "http://backend.default.svc.cluster.local:8080", spec.Upstream.Main.Url) + require.Equal(t, strPtr("http://backend.default.svc.cluster.local:8080"), spec.Upstream.Main.Url) } // TestBuildAPIConfigFromHTTPRoute_PathMatchType pins the issue #2021 fix: the mapper must honor @@ -563,7 +563,7 @@ func TestBuildAPIConfigFromHTTPRoute_CrossNamespaceReferenceGrant(t *testing.T) cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(svc, route, grant).Build() spec, err := BuildAPIConfigFromHTTPRoute(context.Background(), cl, route, "cluster.local", nil) require.NoError(t, err) - require.Equal(t, "http://backend.data.svc.cluster.local:8080", spec.Upstream.Main.Url) + require.Equal(t, strPtr("http://backend.data.svc.cluster.local:8080"), spec.Upstream.Main.Url) }) t.Run("name-scoped grant must match service", func(t *testing.T) { @@ -818,11 +818,11 @@ func TestBuildAPIConfigFromHTTPRoute_CustomClusterDomain(t *testing.T) { spec, err := BuildAPIConfigFromHTTPRoute(context.Background(), cl, route, "example.k8s.local", nil) require.NoError(t, err) - require.Equal(t, "http://backend.default.svc.example.k8s.local:8080", spec.Upstream.Main.Url) + require.Equal(t, strPtr("http://backend.default.svc.example.k8s.local:8080"), spec.Upstream.Main.Url) spec2, err := BuildAPIConfigFromHTTPRoute(context.Background(), cl, route, ".cluster.local.", nil) require.NoError(t, err) - require.Equal(t, "http://backend.default.svc.cluster.local:8080", spec2.Upstream.Main.Url) + require.Equal(t, strPtr("http://backend.default.svc.cluster.local:8080"), spec2.Upstream.Main.Url) } func TestDefaultHTTPRouteAPIHandle(t *testing.T) { diff --git a/kubernetes/gateway-operator/internal/gatewayclient/yaml_payload_test.go b/kubernetes/gateway-operator/internal/gatewayclient/yaml_payload_test.go index 40d3a915f8..53f9899ea8 100644 --- a/kubernetes/gateway-operator/internal/gatewayclient/yaml_payload_test.go +++ b/kubernetes/gateway-operator/internal/gatewayclient/yaml_payload_test.go @@ -17,7 +17,7 @@ func TestBuildRestAPIYAML_IncludesMetadataAnnotationsAndLabels(t *testing.T) { {Method: apiv1.OperationMethodGET, Path: "/"}, }, Upstream: apiv1.UpstreamConfig{ - Main: apiv1.Upstream{Url: "http://hello.default.svc.cluster.local:9080"}, + Main: apiv1.Upstream{Url: stringPtr("http://hello.default.svc.cluster.local:9080")}, }, } md := RestAPIPayloadMetadata{ @@ -43,3 +43,50 @@ func TestBuildRestAPIYAML_IncludesMetadataAnnotationsAndLabels(t *testing.T) { require.True(t, ok) require.Equal(t, "1234567890", annotations["project-id"]) } + +// A RestApi spec using upstreamDefinitions + upstream.main.ref must survive the verbatim +// marshal to the management-API payload, so the operator reaches parity with a direct +// management-API deploy (the connect timeout is carried on the definition). +func TestBuildRestAPIYAML_CarriesUpstreamDefinitionsAndRef(t *testing.T) { + spec := apiv1.APIConfigData{ + Context: "/hello", + DisplayName: "hello", + Version: "v1.0", + Operations: []apiv1.Operation{ + {Method: apiv1.OperationMethodGET, Path: "/"}, + }, + UpstreamDefinitions: []apiv1.UpstreamDefinition{{ + Name: "hello-backend", + Timeout: &apiv1.UpstreamTimeout{Connect: stringPtr("6s")}, + Upstreams: []apiv1.UpstreamTarget{{Url: "http://hello.default.svc.cluster.local:9080"}}, + }}, + Upstream: apiv1.UpstreamConfig{ + Main: apiv1.Upstream{Ref: stringPtr("hello-backend")}, + }, + } + md := RestAPIPayloadMetadata{Name: "api-handle"} + + b, err := BuildRestAPIYAML(apiv1.GroupVersion.String(), "RestApi", md, spec) + require.NoError(t, err) + + var got map[string]interface{} + require.NoError(t, yamlv3.Unmarshal(b, &got)) + specMap, ok := got["spec"].(map[string]interface{}) + require.True(t, ok) + + // upstream.main.ref survives; url is omitted when ref is used. + upstream := specMap["upstream"].(map[string]interface{}) + main := upstream["main"].(map[string]interface{}) + require.Equal(t, "hello-backend", main["ref"]) + _, hasURL := main["url"] + require.False(t, hasURL, "url should be omitted when ref is set") + + // upstreamDefinitions survive with the connect timeout. + defs, ok := specMap["upstreamDefinitions"].([]interface{}) + require.True(t, ok) + require.Len(t, defs, 1) + def0 := defs[0].(map[string]interface{}) + require.Equal(t, "hello-backend", def0["name"]) + timeout := def0["timeout"].(map[string]interface{}) + require.Equal(t, "6s", timeout["connect"]) +} diff --git a/kubernetes/helm/gateway-helm-chart/templates/gateway/gateway-config.yaml b/kubernetes/helm/gateway-helm-chart/templates/gateway/gateway-config.yaml index 1827bcdf94..52e0f0f898 100644 --- a/kubernetes/helm/gateway-helm-chart/templates/gateway/gateway-config.yaml +++ b/kubernetes/helm/gateway-helm-chart/templates/gateway/gateway-config.yaml @@ -143,6 +143,16 @@ data: route_idle_timeout_ms = {{ $router.upstream.timeouts.route_idle_timeout_ms }} connect_timeout_ms = {{ $router.upstream.timeouts.connect_timeout_ms }} + [router.http_listener] + server_header_transformation = {{ $router.http_listener.server_header_transformation | quote }} + server_header_value = {{ $router.http_listener.server_header_value | quote }} + + [router.http_listener.timeouts] + request_timeout = {{ $router.http_listener.timeouts.request_timeout | quote }} + request_headers_timeout = {{ $router.http_listener.timeouts.request_headers_timeout | quote }} + stream_idle_timeout = {{ $router.http_listener.timeouts.stream_idle_timeout | quote }} + idle_timeout = {{ $router.http_listener.timeouts.idle_timeout | quote }} + [router.policy_engine] mode = {{ $router.policy_engine.mode | quote }} host = {{ $router.policy_engine.host | default "" | quote }} diff --git a/kubernetes/helm/gateway-helm-chart/values.yaml b/kubernetes/helm/gateway-helm-chart/values.yaml index 467ad69fea..7666fcb89f 100644 --- a/kubernetes/helm/gateway-helm-chart/values.yaml +++ b/kubernetes/helm/gateway-helm-chart/values.yaml @@ -313,7 +313,25 @@ gateway: route_timeout_ms: 60000 route_idle_timeout_ms: 300000 connect_timeout_ms: 5000 - + + # HTTP Connection Manager (downstream) configuration: server-header handling and timeouts + http_listener: + # Server header handling: APPEND_IF_ABSENT | OVERWRITE | PASS_THROUGH + server_header_transformation: OVERWRITE + # Value written for the Server response header + server_header_value: "WSO2 API Platform" + + # Downstream (client-facing) timeouts. A value of "0s" disables the timeout. + timeouts: + # Max duration for the entire downstream request (0s = disabled) + request_timeout: "0s" + # Max duration to receive the complete request headers (0s = disabled) + request_headers_timeout: "0s" + # Idle timeout for a single HTTP stream/request + stream_idle_timeout: "5m" + # Idle timeout for the downstream connection + idle_timeout: "1h" + tracing_service_name: router # Policy Engine ext_proc filter configuration diff --git a/kubernetes/helm/operator-helm-chart/crds/gateway.api-platform.wso2.com_llmproviders.yaml b/kubernetes/helm/operator-helm-chart/crds/gateway.api-platform.wso2.com_llmproviders.yaml index 8d48f6cbe8..3cc7e1366c 100644 --- a/kubernetes/helm/operator-helm-chart/crds/gateway.api-platform.wso2.com_llmproviders.yaml +++ b/kubernetes/helm/operator-helm-chart/crds/gateway.api-platform.wso2.com_llmproviders.yaml @@ -153,6 +153,21 @@ spec: - version type: object type: array + resilience: + description: |- + Resilience configures API-level backend/route timeouts applied to all routes + generated for this LLM provider. Supported at the API level only. + properties: + idleTimeout: + description: IdleTimeout Per-route stream idle timeout. "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + timeout: + description: Timeout Maximum time for the entire route (request + to upstream response). "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + type: object template: description: Template is the LlmProviderTemplate name to apply. type: string @@ -231,6 +246,68 @@ spec: description: Url is the direct backend URL. type: string type: object + upstreamDefinitions: + description: |- + UpstreamDefinitions is the list of reusable upstream definitions (with optional + connect timeout) that upstream.ref can reference. + items: + description: |- + UpstreamDefinition is a reusable upstream configuration with an optional connect timeout and + load-balancing targets. Referenced from an upstream via its `ref` field. Shared by RestApi, + LLM Provider, and MCP; mirrors the management-API UpstreamDefinition schema. + properties: + basePath: + description: |- + BasePath Base path prefix prepended to all requests routed through this upstream (e.g. /api/v2). + Must start with "/" and must not end with "/". Omit for root ("/"). + pattern: ^/[a-zA-Z0-9\-._~!$&'()*+,;=:@%/]*[^/]$ + type: string + name: + description: Name Unique identifier for this upstream definition + (referenced by upstream.ref). + maxLength: 100 + minLength: 1 + pattern: ^[a-zA-Z0-9\-_]+$ + type: string + timeout: + description: Timeout Optional timeout configuration for this + upstream (connect timeout). + properties: + connect: + description: Connect Connection-establishment timeout duration + (e.g. "5s", "500ms"). "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + type: object + upstreams: + description: Upstreams List of backend targets with optional + weights for load balancing. + items: + description: UpstreamTarget is a single backend target within + an UpstreamDefinition. + properties: + url: + description: Url Backend URL (host and port only; path + comes from the definition's basePath). + pattern: ^https?://[a-zA-Z0-9\-._~:/?#\[\]@!$&'()*+,;=%]+$ + type: string + weight: + description: |- + Weight Relative weight for load balancing across multiple upstream targets. Reserved for + future multi-target load balancing; not applied yet (only the first target is currently used). + maximum: 100 + minimum: 0 + type: integer + required: + - url + type: object + minItems: 1 + type: array + required: + - name + - upstreams + type: object + type: array version: description: Version is the semantic version of the LLM provider. pattern: ^v?([0-9]+)(\.[0-9]+)?(\.[0-9]+)?(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$ diff --git a/kubernetes/helm/operator-helm-chart/crds/gateway.api-platform.wso2.com_llmproxies.yaml b/kubernetes/helm/operator-helm-chart/crds/gateway.api-platform.wso2.com_llmproxies.yaml index dd5a2a5579..81da02ebd1 100644 --- a/kubernetes/helm/operator-helm-chart/crds/gateway.api-platform.wso2.com_llmproxies.yaml +++ b/kubernetes/helm/operator-helm-chart/crds/gateway.api-platform.wso2.com_llmproxies.yaml @@ -180,6 +180,21 @@ spec: required: - id type: object + resilience: + description: |- + Resilience configures API-level backend/route timeouts applied to all routes + generated for this LLM proxy. Supported at the API level only. + properties: + idleTimeout: + description: IdleTimeout Per-route stream idle timeout. "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + timeout: + description: Timeout Maximum time for the entire route (request + to upstream response). "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + type: object version: description: Version is the semantic version of the LLM proxy. pattern: ^v?([0-9]+)(\.[0-9]+)?(\.[0-9]+)?(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$ diff --git a/kubernetes/helm/operator-helm-chart/crds/gateway.api-platform.wso2.com_mcps.yaml b/kubernetes/helm/operator-helm-chart/crds/gateway.api-platform.wso2.com_mcps.yaml index e6d3402a6c..096fac435a 100644 --- a/kubernetes/helm/operator-helm-chart/crds/gateway.api-platform.wso2.com_mcps.yaml +++ b/kubernetes/helm/operator-helm-chart/crds/gateway.api-platform.wso2.com_mcps.yaml @@ -122,6 +122,22 @@ spec: - name type: object type: array + resilience: + description: |- + Resilience configures API-level backend/route timeouts applied to the traffic-forwarding + routes generated for this MCP proxy. Supported at the API level only. Because MCP transports + are long-lived streams, the route timeout defaults to disabled ("0s") for MCP when unset. + properties: + idleTimeout: + description: IdleTimeout Per-route stream idle timeout. "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + timeout: + description: Timeout Maximum time for the entire route (request + to upstream response). "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + type: object resources: description: Resources lists optional MCP resources exposed by this proxy. @@ -254,6 +270,68 @@ spec: description: Url is the direct backend URL. type: string type: object + upstreamDefinitions: + description: |- + UpstreamDefinitions is the list of reusable upstream definitions (with optional + connect timeout) that upstream.ref can reference. + items: + description: |- + UpstreamDefinition is a reusable upstream configuration with an optional connect timeout and + load-balancing targets. Referenced from an upstream via its `ref` field. Shared by RestApi, + LLM Provider, and MCP; mirrors the management-API UpstreamDefinition schema. + properties: + basePath: + description: |- + BasePath Base path prefix prepended to all requests routed through this upstream (e.g. /api/v2). + Must start with "/" and must not end with "/". Omit for root ("/"). + pattern: ^/[a-zA-Z0-9\-._~!$&'()*+,;=:@%/]*[^/]$ + type: string + name: + description: Name Unique identifier for this upstream definition + (referenced by upstream.ref). + maxLength: 100 + minLength: 1 + pattern: ^[a-zA-Z0-9\-_]+$ + type: string + timeout: + description: Timeout Optional timeout configuration for this + upstream (connect timeout). + properties: + connect: + description: Connect Connection-establishment timeout duration + (e.g. "5s", "500ms"). "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + type: object + upstreams: + description: Upstreams List of backend targets with optional + weights for load balancing. + items: + description: UpstreamTarget is a single backend target within + an UpstreamDefinition. + properties: + url: + description: Url Backend URL (host and port only; path + comes from the definition's basePath). + pattern: ^https?://[a-zA-Z0-9\-._~:/?#\[\]@!$&'()*+,;=%]+$ + type: string + weight: + description: |- + Weight Relative weight for load balancing across multiple upstream targets. Reserved for + future multi-target load balancing; not applied yet (only the first target is currently used). + maximum: 100 + minimum: 0 + type: integer + required: + - url + type: object + minItems: 1 + type: array + required: + - name + - upstreams + type: object + type: array version: description: Version is the MCP proxy semantic version. pattern: ^v?([0-9]+)(\.[0-9]+)?(\.[0-9]+)?(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$ diff --git a/kubernetes/helm/operator-helm-chart/crds/gateway.api-platform.wso2.com_restapis.yaml b/kubernetes/helm/operator-helm-chart/crds/gateway.api-platform.wso2.com_restapis.yaml index 3d1ad1a2aa..1193084898 100644 --- a/kubernetes/helm/operator-helm-chart/crds/gateway.api-platform.wso2.com_restapis.yaml +++ b/kubernetes/helm/operator-helm-chart/crds/gateway.api-platform.wso2.com_restapis.yaml @@ -98,6 +98,21 @@ spec: - version type: object type: array + resilience: + description: Resilience Operation-level backend/route timeout + configuration (overrides API-level) + properties: + idleTimeout: + description: IdleTimeout Per-route stream idle timeout. + "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + timeout: + description: Timeout Maximum time for the entire route (request + to upstream response). "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + type: object required: - method - path @@ -132,6 +147,21 @@ spec: - version type: object type: array + resilience: + description: |- + Resilience API-level backend/route timeout configuration applied to all operations + unless overridden at the operation level + properties: + idleTimeout: + description: IdleTimeout Per-route stream idle timeout. "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + timeout: + description: Timeout Maximum time for the entire route (request + to upstream response). "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + type: object upstream: description: Upstream API-level upstream configuration properties: @@ -139,29 +169,101 @@ spec: description: Main Upstream backend configuration for production traffic properties: + ref: + description: Ref Name of a predefined upstreamDefinition to + route through. + type: string url: - description: Url Backend service URL (may include path prefix - like /api/v2) + description: Url Direct backend service URL (may include path + prefix like /api/v2) pattern: ^https?://[a-zA-Z0-9\-._~:/?#\[\]@!$&'()*+,;=%]+$ type: string - required: - - url type: object + x-kubernetes-validations: + - message: exactly one of url or ref must be set + rule: has(self.url) != has(self.ref) sandbox: description: Sandbox Upstream backend configuration for sandbox/testing traffic properties: + ref: + description: Ref Name of a predefined upstreamDefinition to + route through. + type: string url: - description: Url Backend service URL (may include path prefix - like /api/v2) + description: Url Direct backend service URL (may include path + prefix like /api/v2) pattern: ^https?://[a-zA-Z0-9\-._~:/?#\[\]@!$&'()*+,;=%]+$ type: string - required: - - url type: object + x-kubernetes-validations: + - message: exactly one of url or ref must be set + rule: has(self.url) != has(self.ref) required: - main type: object + upstreamDefinitions: + description: |- + UpstreamDefinitions is the list of reusable upstream definitions (with optional connect + timeout) that upstream.ref can reference. + items: + description: |- + UpstreamDefinition is a reusable upstream configuration with an optional connect timeout and + load-balancing targets. Referenced from an upstream via its `ref` field. Shared by RestApi, + LLM Provider, and MCP; mirrors the management-API UpstreamDefinition schema. + properties: + basePath: + description: |- + BasePath Base path prefix prepended to all requests routed through this upstream (e.g. /api/v2). + Must start with "/" and must not end with "/". Omit for root ("/"). + pattern: ^/[a-zA-Z0-9\-._~!$&'()*+,;=:@%/]*[^/]$ + type: string + name: + description: Name Unique identifier for this upstream definition + (referenced by upstream.ref). + maxLength: 100 + minLength: 1 + pattern: ^[a-zA-Z0-9\-_]+$ + type: string + timeout: + description: Timeout Optional timeout configuration for this + upstream (connect timeout). + properties: + connect: + description: Connect Connection-establishment timeout duration + (e.g. "5s", "500ms"). "0s" disables. + pattern: ^\d+(\.\d+)?(ms|s|m|h)$ + type: string + type: object + upstreams: + description: Upstreams List of backend targets with optional + weights for load balancing. + items: + description: UpstreamTarget is a single backend target within + an UpstreamDefinition. + properties: + url: + description: Url Backend URL (host and port only; path + comes from the definition's basePath). + pattern: ^https?://[a-zA-Z0-9\-._~:/?#\[\]@!$&'()*+,;=%]+$ + type: string + weight: + description: |- + Weight Relative weight for load balancing across multiple upstream targets. Reserved for + future multi-target load balancing; not applied yet (only the first target is currently used). + maximum: 100 + minimum: 0 + type: integer + required: + - url + type: object + minItems: 1 + type: array + required: + - name + - upstreams + type: object + type: array version: description: Version Semantic version of the API pattern: ^v?([0-9]+)(\.[0-9]+)?(\.[0-9]+)?(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$ diff --git a/kubernetes/helm/operator-helm-chart/values.yaml b/kubernetes/helm/operator-helm-chart/values.yaml index c0cf8f6689..9c132a2814 100644 --- a/kubernetes/helm/operator-helm-chart/values.yaml +++ b/kubernetes/helm/operator-helm-chart/values.yaml @@ -294,6 +294,24 @@ gateway: route_idle_timeout_ms: 300000 connect_timeout_ms: 5000 + # HTTP Connection Manager (downstream) configuration: server-header handling and timeouts + http_listener: + # Server header handling: APPEND_IF_ABSENT | OVERWRITE | PASS_THROUGH + server_header_transformation: OVERWRITE + # Value written for the Server response header + server_header_value: "WSO2 API Platform" + + # Downstream (client-facing) timeouts. A value of "0s" disables the timeout. + timeouts: + # Max duration for the entire downstream request (0s = disabled) + request_timeout: "0s" + # Max duration to receive the complete request headers (0s = disabled) + request_headers_timeout: "0s" + # Idle timeout for a single HTTP stream/request + stream_idle_timeout: "5m" + # Idle timeout for the downstream connection + idle_timeout: "1h" + # Policy Engine ext_proc filter configuration policy_engine: # Connection mode: "uds" (Unix domain socket, default) or "tcp"