Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 35 additions & 1 deletion projects/gateway2/translator/httproute/delegation.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,12 +39,46 @@ func flattenDelegatedRoutes(
outputs *[]*v1.Route,
routesVisited sets.Set[types.NamespacedName],
delegationChain *list.List,
routeBudget *int,
) error {
parentRoute, ok := parent.Object.(*gwv1.HTTPRoute)
if !ok {
return eris.Errorf("unsupported route type: %T", parent.Object)
}
parentRef := types.NamespacedName{Namespace: parentRoute.Namespace, Name: parentRoute.Name}

// Cycle detection prevents infinite recursion, but a non-cyclic delegation
// graph can still expand combinatorially. Bound the depth so a deep
// diamond/lattice cannot blow up exponentially. delegationChain holds the
// ancestors visited so far, so its length is the current delegation depth.
if delegationChain.Len() >= maxDelegationDepth {
msg := fmt.Sprintf("delegation depth limit (%d) exceeded at parent route %s; not expanding further delegated routes",
maxDelegationDepth, parentRef)
contextutils.LoggerFrom(ctx).Warn(msg)
parentReporter.SetCondition(reports.RouteCondition{
Type: gwv1.RouteConditionAccepted,
Status: metav1.ConditionFalse,
Reason: RouteReasonMaxDelegationDepthExceeded,
Message: msg,
})
return nil
}

// Bound the total number of flattened routes (catches wide/shallow lattices
// that the depth cap alone would miss).
if *routeBudget <= 0 {
msg := fmt.Sprintf("delegated route limit (%d) exceeded at parent route %s; not expanding further delegated routes",
maxDelegatedRoutes, parentRef)
contextutils.LoggerFrom(ctx).Warn(msg)
parentReporter.SetCondition(reports.RouteCondition{
Type: gwv1.RouteConditionAccepted,
Status: metav1.ConditionFalse,
Reason: RouteReasonMaxDelegatedRoutesExceeded,
Message: msg,
})
return nil
}

routesVisited.Insert(parentRef)
defer routesVisited.Delete(parentRef)

Expand Down Expand Up @@ -110,7 +144,7 @@ func flattenDelegatedRoutes(
}

translateGatewayHTTPRouteRulesUtil(
ctx, pluginRegistry, gwListener, child, reporter, baseReporter, outputs, routesVisited, hostnames, delegationChain)
ctx, pluginRegistry, gwListener, child, reporter, baseReporter, outputs, routesVisited, hostnames, delegationChain, routeBudget)
}

return nil
Expand Down
54 changes: 54 additions & 0 deletions projects/gateway2/translator/httproute/delegation_limits.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
package httproute

import (
"os"
"strconv"

gwv1 "sigs.k8s.io/gateway-api/apis/v1"
)

// Delegation expansion limits.
//
// Cycle detection (routesVisited) prevents *infinite* recursion, but a
// non-cyclic delegation graph can still expand combinatorially. The flattening
// enumerates every root->leaf PATH (the ancestor-scoped visited set is removed
// on unwind, by design, so a child reachable via multiple parents is expanded
// once per path). For a diamond/lattice graph this is exponential in depth and
// multiplicative in fan-out, so a moderate misconfiguration (often amplified by
// selector-based delegation under cluster churn) can materialize hundreds of
// millions of route objects and OOM the control plane.
//
// We cannot safely memoize/share a child subtree across parents: a delegatee's
// output depends on parent context (inherited hostnames, the delegation chain
// passed to plugins, and parent-policy override applied to child routes). So
// instead we bound the expansion and fail closed with a clear status condition.
//
// Both limits are generous enough that no legitimate config should hit them,
// and both are overridable via env var for operators with unusual topologies.
var (
// maxDelegationDepth bounds how many delegation hops deep the tree may be
// flattened. Real delegation trees are shallow (typically 1-3); the
// exponential-depth blowup needs many levels, so this is the primary guard.
maxDelegationDepth = envInt("GLOO_MAX_DELEGATION_DEPTH", 10)

// maxDelegatedRoutes bounds the total number of routes a single top-level
// HTTPRoute may flatten into. This catches wide/shallow lattices that a depth
// cap alone would miss. Set high enough to never trip a real config.
maxDelegatedRoutes = envInt("GLOO_MAX_DELEGATED_ROUTES", 100000)
)

// Implementation-specific status reasons (Gateway API permits custom,
// PascalCase reasons) used when an expansion limit is exceeded.
const (
RouteReasonMaxDelegationDepthExceeded gwv1.RouteConditionReason = "MaxDelegationDepthExceeded"
RouteReasonMaxDelegatedRoutesExceeded gwv1.RouteConditionReason = "MaxDelegatedRoutesExceeded"
)

func envInt(name string, def int) int {
if v := os.Getenv(name); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
return n
}
}
return def
}
121 changes: 121 additions & 0 deletions projects/gateway2/translator/httproute/delegation_limits_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
package httproute_test

import (
"context"
"fmt"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/ptr"
gwv1 "sigs.k8s.io/gateway-api/apis/v1"

"github.com/solo-io/gloo/projects/gateway2/query"
"github.com/solo-io/gloo/projects/gateway2/reports"
"github.com/solo-io/gloo/projects/gateway2/translator/httproute"
"github.com/solo-io/gloo/projects/gateway2/translator/plugins/registry"
"github.com/solo-io/gloo/projects/gateway2/wellknown"
)

var _ = Describe("delegation expansion limits", func() {
var (
ctx context.Context
pluginRegistry registry.PluginRegistry
gwListener gwv1.Listener
)

BeforeEach(func() {
ctx = context.Background()
pluginRegistry = registry.NewPluginRegistry(nil)
gwListener = gwv1.Listener{}
})

// delegatedBackendRef builds an HTTPRoute backendRef that points to another
// HTTPRoute (i.e. delegation).
delegatedBackendRef := func(name string) gwv1.HTTPBackendRef {
return gwv1.HTTPBackendRef{
BackendRef: gwv1.BackendRef{
BackendObjectReference: gwv1.BackendObjectReference{
Group: ptr.To(gwv1.Group(wellknown.GatewayGroup)),
Kind: ptr.To(gwv1.Kind(wellknown.HTTPRouteKind)),
Name: gwv1.ObjectName(name),
},
},
}
}

// route builds an HTTPRoute named name whose single rule has the given
// backendRefs.
route := func(name string, backendRefs ...gwv1.HTTPBackendRef) *gwv1.HTTPRoute {
return &gwv1.HTTPRoute{
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default"},
Spec: gwv1.HTTPRouteSpec{
Rules: []gwv1.HTTPRouteRule{{
Matches: []gwv1.HTTPRouteMatch{{
Path: &gwv1.HTTPPathMatch{
Type: ptr.To(gwv1.PathMatchPathPrefix),
Value: ptr.To("/"),
},
}},
BackendRefs: backendRefs,
}},
},
}
}

// buildChain builds a linear delegation chain of the given depth:
// r0 -> r1 -> ... -> r{depth}. Each RouteInfo's Children point to the next.
// Returns the root RouteInfo.
buildChain := func(depth int) *query.RouteInfo {
// Build leaf-up so each parent's Children contains its child RouteInfo.
var childInfo *query.RouteInfo
for i := depth; i >= 0; i-- {
name := fmt.Sprintf("r%d", i)
children := query.NewBackendMap[[]*query.RouteInfo]()
var hr *gwv1.HTTPRoute
if childInfo != nil {
childName := fmt.Sprintf("r%d", i+1)
ref := delegatedBackendRef(childName)
hr = route(name, ref)
children.Add(ref.BackendObjectReference, []*query.RouteInfo{childInfo})
} else {
// leaf has no backends; it still yields a (direct-response) route
hr = route(name)
}
childInfo = &query.RouteInfo{Object: hr, Children: children}
}
return childInfo
}

It("stops expanding and reports a condition when the delegation depth limit is exceeded", func() {
// A chain deeper than the default max depth.
root := buildChain(20)

rm := reports.NewReportMap()
baseReporter := reports.NewReporter(&rm)
parentRefReporter := baseReporter.Route(root.Object).ParentRef(&gwv1.ParentReference{Name: "gw"})

// Should not panic / hang, and should return a bounded set of routes.
routes := httproute.TranslateGatewayHTTPRouteRules(ctx, pluginRegistry, gwListener, root, parentRefReporter, baseReporter)
Expect(len(routes)).To(BeNumerically("<=", 1), "deep chain must not expand past the depth cap")

// Some route in the chain at the depth boundary should carry the
// MaxDelegationDepthExceeded condition (Accepted=False).
foundDepthCondition := false
for i := 0; i <= 20; i++ {
hr := route(fmt.Sprintf("r%d", i))
status := rm.BuildRouteStatus(ctx, hr, "")
if status == nil {
continue
}
for _, parent := range status.Parents {
for _, cond := range parent.Conditions {
if cond.Reason == string(httproute.RouteReasonMaxDelegationDepthExceeded) {
foundDepthCondition = true
}
}
}
}
Expect(foundDepthCondition).To(BeTrue(), "expected a MaxDelegationDepthExceeded condition at the depth boundary")
})
})
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,13 @@ func TranslateGatewayHTTPRouteRules(

delegationChain := list.New()

// routeBudget bounds the total number of routes this top-level HTTPRoute may
// flatten into, so a combinatorial (non-cyclic) delegation graph cannot OOM
// the control plane. See delegation_limits.go.
routeBudget := maxDelegatedRoutes

translateGatewayHTTPRouteRulesUtil(
ctx, pluginRegistry, gwListener, routeInfo, reporter, baseReporter, &finalRoutes, routesVisited, hostnames, delegationChain)
ctx, pluginRegistry, gwListener, routeInfo, reporter, baseReporter, &finalRoutes, routesVisited, hostnames, delegationChain, &routeBudget)
return finalRoutes
}

Expand All @@ -77,13 +82,21 @@ func translateGatewayHTTPRouteRulesUtil(
routesVisited sets.Set[types.NamespacedName],
hostnames []gwv1.Hostname,
delegationChain *list.List,
routeBudget *int,
) {
// Only HTTPRoute types should be translated.
route, ok := routeInfo.Object.(*gwv1.HTTPRoute)
if !ok {
return
}

// Stop expanding once the flattened-route budget is exhausted. This bounds
// the total work for a combinatorial delegation graph; the offending parent
// edge is flagged where the limit is detected (see flattenDelegatedRoutes).
if *routeBudget <= 0 {
return
}

for ruleIdx, rule := range route.Spec.Rules {
rule := rule
if rule.Matches == nil {
Expand All @@ -105,6 +118,7 @@ func translateGatewayHTTPRouteRulesUtil(
routesVisited,
hostnames,
delegationChain,
routeBudget,
)
for _, outputRoute := range outputRoutes {
// The above function will return a nil route if a matcher fails to apply plugins
Expand Down Expand Up @@ -132,6 +146,7 @@ func translateGatewayHTTPRouteRule(
routesVisited sets.Set[types.NamespacedName],
hostnames []gwv1.Hostname,
delegationChain *list.List,
routeBudget *int,
) []*v1.Route {
routes := make([]*v1.Route, len(rule.Matches))

Expand Down Expand Up @@ -171,6 +186,7 @@ func translateGatewayHTTPRouteRule(
&delegatedRoutes,
routesVisited,
delegationChain,
routeBudget,
)
}

Expand Down Expand Up @@ -225,6 +241,8 @@ func translateGatewayHTTPRouteRule(
if outputRoute.GetAction() != nil {
outputRoute.Matchers = []*matchers.Matcher{translateGlooMatcher(match)}
routes[idx] = outputRoute
// Count each finalized leaf route against the flattening budget.
*routeBudget--
}
}
return routes
Expand Down Expand Up @@ -321,6 +339,7 @@ func setRouteAction(
outputs *[]*v1.Route,
routesVisited sets.Set[types.NamespacedName],
delegationChain *list.List,
routeBudget *int,
) bool {
var weightedDestinations []*v1.WeightedDestination
backendRefs := rule.BackendRefs
Expand All @@ -344,6 +363,7 @@ func setRouteAction(
outputs,
routesVisited,
delegationChain,
routeBudget,
)
if err != nil {
query.ProcessBackendError(err, reporter)
Expand Down
Loading