diff --git a/go.mod b/go.mod index 46719c0ba..af8e3606a 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( go.uber.org/goleak v1.3.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.1 + go4.org/netipx v0.0.0-20231129151722-fdeea329fbba golang.org/x/sync v0.20.0 golang.org/x/text v0.37.0 k8s.io/api v0.35.0 @@ -191,7 +192,6 @@ require ( go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect golang.org/x/crypto v0.52.0 // indirect golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect golang.org/x/mod v0.35.0 // indirect diff --git a/pkg/registry/file/containerprofile_processor_test.go b/pkg/registry/file/containerprofile_processor_test.go index bacfb1629..af9fe4199 100644 --- a/pkg/registry/file/containerprofile_processor_test.go +++ b/pkg/registry/file/containerprofile_processor_test.go @@ -26,10 +26,10 @@ import ( ) func TestDeflateContainerProfileSpec_NetworkNeighborsCollapse(t *testing.T) { - const hostCount = 60 + const hostCount = 64 // a fully-observed /26 (10.0.0.0..10.0.0.63) newIngress := func() []softwarecomposition.NetworkNeighbor { ingress := make([]softwarecomposition.NetworkNeighbor, 0, hostCount) - for i := 1; i <= hostCount; i++ { + for i := 0; i < hostCount; i++ { ingress = append(ingress, softwarecomposition.NetworkNeighbor{ Identifier: fmt.Sprintf("external-%d", i), Type: "external", diff --git a/pkg/registry/file/dynamicpathdetector/types.go b/pkg/registry/file/dynamicpathdetector/types.go index 40e728777..7489b943c 100644 --- a/pkg/registry/file/dynamicpathdetector/types.go +++ b/pkg/registry/file/dynamicpathdetector/types.go @@ -25,11 +25,21 @@ const ( // NetworkNeighbor entries differing only by IP gets CIDR-collapsed. // NetworkCIDRFloorBits is the minimum CIDR prefix length (maximum breadth) // a single aggregated block may have. +// NetworkMaxCIDRSplitBits caps, PER prefix, how far a single cover block broader +// than the floor is split into floor-width children: up to 2^NetworkMaxCIDRSplitBits +// blocks (4096 here). A prefix whose split would exceed that is kept as-is rather +// than exploding the entry list. This bounds ONE block's fan-out (not the sum +// across a group — the whole neighborhood is separately capped by +// MaxNetworkNeighborhoodSize); it only bites when a held pass-through block is +// much broader than a tightened floor (e.g. a /16 under a /28 floor -> 4096 +// children; a /16 under a /24 floor is only 256). Not currently exposed as a +// CollapseConfiguration field. const ( OpenDynamicThreshold = 50 EndpointDynamicThreshold = 100 NetworkIPGroupThreshold = 50 NetworkCIDRFloorBits = 24 + NetworkMaxCIDRSplitBits = 12 ) // --- Collapse configuration --- diff --git a/pkg/registry/file/networkneighborhood_ipcollapse.go b/pkg/registry/file/networkneighborhood_ipcollapse.go index 347822d4f..8a21bd1c0 100644 --- a/pkg/registry/file/networkneighborhood_ipcollapse.go +++ b/pkg/registry/file/networkneighborhood_ipcollapse.go @@ -9,6 +9,7 @@ import ( "github.com/kubescape/storage/pkg/apis/softwarecomposition" "github.com/kubescape/storage/pkg/registry/file/dynamicpathdetector" + "go4.org/netipx" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -18,14 +19,16 @@ const ipCollapseFieldSep = "\x00" // into a small number of CIDR-bearing entries. Entries are grouped by // (Type, DNS, NamespaceSelector, PodSelector); within a group whose count of // aggregatable IPv4 host addresses exceeds settings.NetworkIPGroupThreshold, -// those hosts are replaced by covering CIDR block(s) no broader than -// settings.NetworkCIDRFloorBits. +// those hosts plus any already-collapsed pass-through CIDRs are replaced by +// covering CIDR blocks no broader than settings.NetworkCIDRFloorBits (see +// coverPrefixes). Output size is bounded by the number of distinct floor-length +// networks the workload actually reached, not by the host count: scattered hosts +// collapse to one block per floor network rather than one /32 apiece. // -// The pass is a fixpoint (AC10): already-collapsed CIDR values and the "*" -// sentinel / IPv6 values are treated as pass-through and are never re-parsed as -// host IPs or re-tightened, and collapsed output carries a deterministic -// Identifier, so a second run — whose groups now hold only CIDRs and thus have -// zero aggregatable hosts — leaves everything untouched. +// The pass is a fixpoint: collapsed blocks re-fed through the same aggregation +// converge to themselves, and the "*" sentinel / bare IPv6 values are +// pass-through held verbatim, so a second run — whose groups now hold only CIDRs +// and thus have zero aggregatable hosts — leaves everything untouched. func collapseIPGroups(entries []softwarecomposition.NetworkNeighbor, settings dynamicpathdetector.CollapseSettings) []softwarecomposition.NetworkNeighbor { if entries == nil { return nil @@ -78,8 +81,24 @@ func collapseIPGroups(entries []softwarecomposition.NetworkNeighbor, settings dy continue } - cidrs := aggregateHosts(hosts, floorBits) - values := append(cidrs, passthrough...) + // Split pass-through into already-collapsed CIDRs (folded into the cover) + // and non-CIDR sentinels ("*", bare IPv6, unparseable) held verbatim. + var cidrPass []netip.Prefix + var sentinels []string + for _, v := range passthrough { + if p, err := netip.ParsePrefix(v); err == nil { + cidrPass = append(cidrPass, p.Masked()) + } else { + sentinels = append(sentinels, v) + } + } + + // Exact minimal CIDR cover of the hosts plus already-held CIDRs, capped at + // the floor. Because it is an exact cover, incremental re-collapsing is a + // fixpoint and never accumulates duplicate or nested blocks — the bug that + // produced [52.216.0.0/26, 52.216.0.0/26, 52.216.0.0/27]. + values := coverPrefixes(hosts, cidrPass, floorBits) + values = append(values, sentinels...) sort.Strings(values) var dnsNames []string @@ -112,8 +131,11 @@ func collapseIPGroups(entries []softwarecomposition.NetworkNeighbor, settings dy // host addresses (deduped) and pass-through values held verbatim. An entry's // value comes from the singular IPAddress when set, otherwise from each element // of IPAddresses. CIDRs, the "*" sentinel, IPv6 and unparseable values are -// pass-through and are never fed to aggregation, which is what makes the pass a -// fixpoint on already-collapsed input. +// pass-through (already-collapsed CIDRs are folded back into the cover by the +// caller; the rest are held verbatim). Only IPv4 hosts are aggregated: policy +// generation (buildIPAddressesPeers) skips non-IPv4 entries, so collapsing IPv6 +// hosts into a CIDR would silently drop them — and any ports — from the derived +// NetworkPolicy. IPv6 is therefore kept as individual pass-through entries. func classifyGroupAddresses(entries []softwarecomposition.NetworkNeighbor) ([]netip.Addr, []string) { seenHost := map[netip.Addr]struct{}{} seenPass := map[string]struct{}{} @@ -149,24 +171,86 @@ func classifyGroupAddresses(entries []softwarecomposition.NetworkNeighbor) ([]ne return hosts, passthrough } -// aggregateHosts returns the CIDR block(s) covering the given IPv4 hosts. If the -// hosts share a common prefix at least as long as floorBits it is emitted as a -// single block; otherwise each host is bucketed into a floorBits-length prefix -// so no emitted block is ever broader than the floor. -func aggregateHosts(hosts []netip.Addr, floorBits int) []string { - if len(hosts) == 0 { +// coverPrefixes returns the set of CIDR strings covering the given IPv4 host +// addresses together with the group's already-collapsed pass-through CIDRs, with +// no block broader than floorBits and a bounded entry count. +// +// It works in two stages. First the raw hosts are aggregated to the floor +// (aggregateHostsToFloor): a group of hosts sharing a common prefix at least as +// long as the floor collapses to that single tight block — kept tighter than the +// floor when the traffic really is that tight, e.g. a fully-observed /26 — while +// scattered hosts are bucketed into their floor-length networks so the output is +// bounded by the number of distinct floor networks reached, not the host count. +// Second, those host blocks are folded together with the already-held CIDRs +// through netipx, which deduplicates, drops subsumed prefixes and merges adjacent +// siblings into a canonical set. That fold is what fixes incremental-learning +// garbage like [52.216.0.0/26, 52.216.0.0/26, 52.216.0.0/27] — the duplicate +// deduplicated and the nested /27 absorbed — and makes re-collapsing a fixpoint. +// +// Any IPv4 block still broader than the floor after the fold (a held block from a +// coarser prior floor, or floor networks that merged into a shorter parent) is +// split back into floorBits-wide children. IPv6 has no floor and is emitted as +// covered. The result is sorted. +func coverPrefixes(hosts []netip.Addr, cidrPass []netip.Prefix, floorBits int) []string { + if len(hosts) == 0 && len(cidrPass) == 0 { return nil } - if commonLen := commonPrefixLen(hosts); commonLen >= floorBits { - return []string{netip.PrefixFrom(hosts[0], commonLen).Masked().String()} + var b netipx.IPSetBuilder + for _, p := range aggregateHostsToFloor(hosts, floorBits) { + b.AddPrefix(p) + } + for _, p := range cidrPass { + b.AddPrefix(p.Masked()) + } + set, err := b.IPSet() + if err != nil || set == nil { + return nil } - seen := map[string]struct{}{} + var out []string - for _, addr := range hosts { - cidr := netip.PrefixFrom(addr, floorBits).Masked().String() - if _, ok := seen[cidr]; !ok { - seen[cidr] = struct{}{} - out = append(out, cidr) + for _, p := range set.Prefixes() { + // The floor is an IPv4 breadth cap; IPv6 covers are emitted as-is (a /24 + // floor is meaningless for v6, whose covers are already narrow). + if p.Addr().Is4() && p.Bits() < floorBits { + out = append(out, splitToFloor(p, floorBits)...) + continue + } + out = append(out, p.String()) + } + sort.Strings(out) + return out +} + +// aggregateHostsToFloor collapses raw host addresses into CIDR blocks no broader +// than floorBits. IPv4 hosts sharing a common prefix at least as long as the +// floor collapse to that single common block (kept tighter than the floor when +// the traffic is genuinely that tight, e.g. a fully-observed /26); otherwise each +// host is bucketed into its floor-length network, so scattered traffic yields at +// most one block per distinct floor network. IPv6 hosts — which the caller keeps +// out of aggregation, but which are handled defensively here — are emitted as +// individual host prefixes and never widened by the IPv4 floor. +func aggregateHostsToFloor(hosts []netip.Addr, floorBits int) []netip.Prefix { + var v4 []netip.Addr + var out []netip.Prefix + for _, h := range hosts { + if h.Is4() { + v4 = append(v4, h) + } else { + out = append(out, netip.PrefixFrom(h, h.BitLen()).Masked()) + } + } + if len(v4) == 0 { + return out + } + if commonLen := commonPrefixLen(v4); commonLen >= floorBits { + return append(out, netip.PrefixFrom(v4[0], commonLen).Masked()) + } + seen := make(map[netip.Prefix]struct{}, len(v4)) + for _, h := range v4 { + p := netip.PrefixFrom(h, floorBits).Masked() + if _, ok := seen[p]; !ok { + seen[p] = struct{}{} + out = append(out, p) } } return out @@ -201,6 +285,32 @@ func commonPrefixLen(addrs []netip.Addr) int { return common } +// splitToFloor divides a prefix broader than floorBits into its floorBits-wide +// children. If the fan-out would exceed 2^NetworkMaxCIDRSplitBits blocks the +// prefix is returned unsplit, trading a strictly-honored floor for a bounded +// entry count. +func splitToFloor(p netip.Prefix, floorBits int) []string { + shift := floorBits - p.Bits() + if shift <= 0 { + return []string{p.String()} + } + if shift > dynamicpathdetector.NetworkMaxCIDRSplitBits { + return []string{p.String()} + } + count := 1 << shift + out := make([]string, 0, count) + child := netip.PrefixFrom(p.Addr(), floorBits).Masked() + for i := 0; i < count; i++ { + out = append(out, child.String()) + next := netipx.RangeOfPrefix(child).To().Next() + if !next.IsValid() { + break + } + child = netip.PrefixFrom(next, floorBits).Masked() + } + return out +} + func neighborGroupKey(n softwarecomposition.NetworkNeighbor) string { return strings.Join([]string{ string(n.Type), diff --git a/pkg/registry/file/networkneighborhood_ipcollapse_bench_test.go b/pkg/registry/file/networkneighborhood_ipcollapse_bench_test.go new file mode 100644 index 000000000..f6dfdeaf2 --- /dev/null +++ b/pkg/registry/file/networkneighborhood_ipcollapse_bench_test.go @@ -0,0 +1,63 @@ +package file + +import ( + "fmt" + "net/netip" + "testing" + + "github.com/kubescape/storage/pkg/apis/softwarecomposition" + "github.com/kubescape/storage/pkg/registry/file/dynamicpathdetector" +) + +func cidrNeighbor(c string) softwarecomposition.NetworkNeighbor { + return softwarecomposition.NetworkNeighbor{ + Type: softwarecomposition.CommunicationTypeEgress, + DNS: "example.com", + IPAddresses: []string{c}, + } +} + +// BenchmarkCollapseIPGroups measures CPU/allocations of the full deflate path on +// a realistic incremental-learning snapshot: 200 contiguous hosts in a /24 plus +// two already-collapsed pass-through CIDRs from earlier saves, at a /16 and a +// /24 floor (the latter exercising the floor-cap split). +func BenchmarkCollapseIPGroups(b *testing.B) { + var in []softwarecomposition.NetworkNeighbor + for i := 0; i < 200; i++ { + in = append(in, hostNeighbor(fmt.Sprintf("52.216.%d.%d", i/256, i%256))) + } + in = append(in, cidrNeighbor("52.216.4.0/24"), cidrNeighbor("52.216.0.0/16")) + + for _, floor := range []int{16, 24} { + settings := dynamicpathdetector.CollapseSettings{NetworkIPGroupThreshold: 5, NetworkCIDRFloorBits: floor} + b.Run(fmt.Sprintf("floor%d", floor), func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = collapseIPGroups(in, settings) + } + }) + } +} + +// BenchmarkCoverPrefixes isolates the netipx exact-cover step: 256 scattered +// hosts across a /16 plus two pass-through CIDRs, at a /16 and a /24 floor. +func BenchmarkCoverPrefixes(b *testing.B) { + hosts := make([]netip.Addr, 0, 256) + for i := 0; i < 256; i++ { + hosts = append(hosts, netip.AddrFrom4([4]byte{52, 216, byte(i), byte((i * 7) % 256)})) + } + cidrPass := []netip.Prefix{ + netip.MustParsePrefix("52.216.4.0/24"), + netip.MustParsePrefix("52.216.128.0/17"), + } + for _, floor := range []int{16, 24} { + b.Run(fmt.Sprintf("floor%d", floor), func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = coverPrefixes(hosts, cidrPass, floor) + } + }) + } +} diff --git a/pkg/registry/file/networkneighborhood_ipcollapse_test.go b/pkg/registry/file/networkneighborhood_ipcollapse_test.go index beefdcda5..e8bc40025 100644 --- a/pkg/registry/file/networkneighborhood_ipcollapse_test.go +++ b/pkg/registry/file/networkneighborhood_ipcollapse_test.go @@ -44,23 +44,24 @@ func TestCollapseIPGroups_BelowThresholdUntouched(t *testing.T) { } func TestCollapseIPGroups_AboveThresholdSingleCoveringCIDR(t *testing.T) { - // 60 hosts spread across the full third octet of 10.1.0.0/16 (0..236, - // spanning the top bit) -> common prefix exactly the floor (16) -> one block. + // A fully-observed /24 (all 256 hosts) exact-covers to exactly one /24 block. var in []softwarecomposition.NetworkNeighbor - for i := 0; i < 60; i++ { - in = append(in, hostNeighbor(fmt.Sprintf("10.1.%d.0", i*4))) + for i := 0; i < 256; i++ { + in = append(in, hostNeighbor(fmt.Sprintf("10.1.5.%d", i))) } out := collapseIPGroups(in, testSettings()) require.Len(t, out, 1) - assert.Equal(t, []string{"10.1.0.0/16"}, out[0].IPAddresses) + assert.Equal(t, []string{"10.1.5.0/24"}, out[0].IPAddresses) assert.Empty(t, out[0].IPAddress) } -func TestCollapseIPGroups_AboveThresholdBroaderThanFloorBuckets(t *testing.T) { - // 60 hosts spread across many /16s -> common prefix broader than floor -> - // floor-bucket into distinct /16 blocks, none broader than the floor. +func TestCollapseIPGroups_ScatteredHostsBucketedToFloor(t *testing.T) { + // 60 lone hosts, each in its own /16, do not share a common prefix as long as + // the floor, so each is bucketed into its floor-length (/16) network. Output + // is one block per distinct floor network — bounded by the number of networks + // reached, not the host count — and no block is broader than the floor. var in []softwarecomposition.NetworkNeighbor for i := 0; i < 60; i++ { in = append(in, hostNeighbor(fmt.Sprintf("%d.%d.0.1", 10+i, i))) @@ -68,12 +69,12 @@ func TestCollapseIPGroups_AboveThresholdBroaderThanFloorBuckets(t *testing.T) { out := collapseIPGroups(in, testSettings()) - assert.Greater(t, len(out), 1) + require.Len(t, out, 60, "one bucket per distinct /16") for _, e := range out { require.Len(t, e.IPAddresses, 1) p, err := netip.ParsePrefix(e.IPAddresses[0]) require.NoError(t, err) - assert.GreaterOrEqual(t, p.Bits(), 16, "no emitted block may be broader than the floor") + assert.Equal(t, 16, p.Bits(), "each lone host is bucketed into its floor-length network") } } @@ -110,12 +111,12 @@ func TestCollapseIPGroups_DifferentSelectorsNotMerged(t *testing.T) { return &metav1.LabelSelector{MatchLabels: map[string]string{"app": v}} } var in []softwarecomposition.NetworkNeighbor - for i := 0; i < 60; i++ { + for i := 0; i < 64; i++ { // a full /26 per selector -> one exact block each e := hostNeighbor(fmt.Sprintf("10.3.0.%d", i)) e.PodSelector = sel("a") in = append(in, e) } - for i := 0; i < 60; i++ { + for i := 0; i < 64; i++ { e := hostNeighbor(fmt.Sprintf("10.3.0.%d", i)) e.PodSelector = sel("b") in = append(in, e) @@ -135,17 +136,20 @@ func TestCollapseIPGroups_DifferentSelectorsNotMerged(t *testing.T) { func TestCollapseIPGroups_RealWorldShapeOrdersOfMagnitude(t *testing.T) { var in []softwarecomposition.NetworkNeighbor - // ~500 IPs clustered in 100.68.x.x - for i := 0; i < 250; i++ { - in = append(in, hostNeighbor(fmt.Sprintf("100.68.%d.%d", i/256, i%256))) + // 256 IPs fully covering 100.68.0.0/24 + for i := 0; i < 256; i++ { + in = append(in, hostNeighbor(fmt.Sprintf("100.68.0.%d", i))) } - // ~250 IPs clustered in 16.15.183.x plus neighboring /24s - for i := 0; i < 250; i++ { - in = append(in, hostNeighbor(fmt.Sprintf("16.15.%d.%d", 180+i/256, i%256))) + // 256 IPs fully covering 16.15.180.0/24 + for i := 0; i < 256; i++ { + in = append(in, hostNeighbor(fmt.Sprintf("16.15.180.%d", i))) } out := collapseIPGroups(in, testSettings()) + // The two /24s fall in different /16s and share no common prefix as long as + // the /16 floor, so each is bucketed into its floor network: a handful of + // blocks (two /16s here), orders of magnitude below the host count. assert.Less(t, len(out), 10) assert.Less(t, len(out), len(in)/50) for _, e := range out { @@ -175,7 +179,8 @@ func TestCollapseIPGroups_Idempotent(t *testing.T) { DNS: "example.com", IPAddresses: []string{"*"}, }) - // IPv6 entry + // IPv6 entry — held as a pass-through value verbatim (IPv6 is not aggregated, + // since policy generation consumes only IPv4 collapsed entries) in = append(in, softwarecomposition.NetworkNeighbor{ Type: softwarecomposition.CommunicationTypeEgress, DNS: "example.com", @@ -187,7 +192,7 @@ func TestCollapseIPGroups_Idempotent(t *testing.T) { assert.Equal(t, once, twice, "collapseIPGroups must be a fixpoint") - // pass-through values survived + // pass-through + covered values survived var values []string for _, e := range once { values = append(values, e.IPAddresses...) @@ -233,16 +238,21 @@ func TestCollapseIPGroups_MultiBucketReplicatesDNSNamesAndPorts(t *testing.T) { } } -func TestCollapseIPGroups_IPv6PassThrough(t *testing.T) { +func TestCollapseIPGroups_IPv6NotAggregatedHeldPassThrough(t *testing.T) { + // IPv6 hosts are NOT aggregated into CIDRs: policy generation + // (buildIPAddressesPeers) consumes only IPv4 collapsed entries, so folding + // IPv6 hosts into an IPv6 CIDR would silently drop them — and their ports — + // from the derived NetworkPolicy. They are held as individual pass-through + // values instead, even when contiguous, while the co-located v4 group still + // collapses normally. var in []softwarecomposition.NetworkNeighbor + for i := 0; i < 256; i++ { + in = append(in, hostNeighbor(fmt.Sprintf("2606:4700:0:1::%x", i))) + } + in = append(in, hostNeighbor("2001:db8::42")) for i := 0; i < 60; i++ { in = append(in, hostNeighbor(fmt.Sprintf("10.5.0.%d", i))) } - in = append(in, softwarecomposition.NetworkNeighbor{ - Type: softwarecomposition.CommunicationTypeEgress, - DNS: "example.com", - IPAddress: "2001:db8::42", - }) out := collapseIPGroups(in, testSettings()) @@ -250,9 +260,117 @@ func TestCollapseIPGroups_IPv6PassThrough(t *testing.T) { for _, e := range out { values = append(values, e.IPAddresses...) } - assert.Contains(t, values, "2001:db8::42") + // v6 hosts survive verbatim, never merged into a /120 or /128 CIDR + assert.Contains(t, values, "2606:4700:0:1::0", "contiguous v6 hosts stay individual") + assert.Contains(t, values, "2001:db8::42", "lone v6 host stays verbatim") + assert.NotContains(t, values, "2606:4700:0:1::/120", "v6 must not be aggregated into a CIDR") + // the co-located IPv4 group still collapses (a fully-observed /26) + assert.Contains(t, values, "10.5.0.0/26", "co-located IPv4 group still collapses") +} + +func TestCoverPrefixes_IPv6ExactAndMerge(t *testing.T) { + // Two adjacent v6 /33 halves merge into the parent /32 (Cloudflare 2606:4700::/32), + // independent of any IPv4 floor. + got := coverPrefixes(nil, []netip.Prefix{ + netip.MustParsePrefix("2606:4700::/33"), + netip.MustParsePrefix("2606:4700:8000::/33"), + }, 24) + assert.Equal(t, []string{"2606:4700::/32"}, got) +} + +// TestCoverPrefixes_RealCloudRangesDedupAndMerge feeds netipx the kind of messy, +// overlapping, non-aggregated CIDR lists cloud providers publish (a subsumed +// range, two adjacent siblings that merge, and disjoint blocks across families) +// and asserts the minimal exact cover. +func TestCoverPrefixes_RealCloudRangesDedupAndMerge(t *testing.T) { + pass := []netip.Prefix{ + // AWS S3 us-east-1: 52.216.0.0/15 subsumes the more specific 52.216.4.0/24 + netip.MustParsePrefix("52.216.0.0/15"), + netip.MustParsePrefix("52.216.4.0/24"), + // Cloudflare: 104.16.0.0/13 subsumes 104.16.0.0/14 + netip.MustParsePrefix("104.16.0.0/13"), + netip.MustParsePrefix("104.16.0.0/14"), + // Cloudflare v6 siblings that merge to a /31 + netip.MustParsePrefix("2606:4700::/32"), + netip.MustParsePrefix("2606:4701::/32"), + } + // Permissive floor (/8) so the cap does not split these broad blocks — this + // isolates the dedup/merge behavior (the floor cap has its own test). + got := coverPrefixes(nil, pass, 8) + // sorted lexicographically (the collapse output order) + assert.Equal(t, []string{ + "104.16.0.0/13", + "2606:4700::/31", + "52.216.0.0/15", + }, got) } func TestCollapseIPGroups_NilInput(t *testing.T) { assert.Nil(t, collapseIPGroups(nil, testSettings())) } + +func TestCollapseIPGroups_IncrementalReCollapseDeduplicatesAndAbsorbs(t *testing.T) { + // Regression for the incremental-learning garbage [/26, /26, /27]: a group + // that already holds collapsed CIDRs from earlier saves (a /27 and a /26) + // plus freshly observed hosts that re-aggregate to 52.216.0.0/26 must + // converge to exactly one 52.216.0.0/26 — the duplicate /26 deduplicated and + // the nested /27 absorbed — instead of accumulating all three entries. + settings := dynamicpathdetector.CollapseSettings{ + NetworkIPGroupThreshold: 5, + NetworkCIDRFloorBits: 16, + } + cidr := func(c string) softwarecomposition.NetworkNeighbor { + return softwarecomposition.NetworkNeighbor{ + Type: softwarecomposition.CommunicationTypeEgress, + DNS: "example.com", + IPAddresses: []string{c}, + } + } + in := []softwarecomposition.NetworkNeighbor{ + cidr("52.216.0.0/27"), + cidr("52.216.0.0/26"), + } + for _, h := range []string{"52.216.0.1", "52.216.0.10", "52.216.0.20", "52.216.0.40", "52.216.0.55", "52.216.0.60"} { + in = append(in, hostNeighbor(h)) + } + + out := collapseIPGroups(in, settings) + + var cidrs []string + for _, e := range out { + cidrs = append(cidrs, e.IPAddresses...) + assert.Empty(t, e.IPAddress) + } + assert.Equal(t, []string{"52.216.0.0/26"}, cidrs, "must converge to a single covering /26, not [/26 /26 /27]") +} + +func TestCoverPrefixes_HostsCollapseToCommonPrefixWhenTighterThanFloor(t *testing.T) { + // Hosts sharing a common prefix at least as long as the floor collapse to that + // single common block. {.1,.2,.3} share a /30, which is tighter than the /16 + // floor, so they aggregate to 52.216.0.0/30 (bounding the entry count to one + // rather than emitting a /32 and a /31). The block is capped at the floor, but + // the workload's own common prefix is honored when it is already narrower. + hosts := []netip.Addr{ + netip.MustParseAddr("52.216.0.1"), + netip.MustParseAddr("52.216.0.2"), + netip.MustParseAddr("52.216.0.3"), + } + got := coverPrefixes(hosts, nil, 16) + assert.Equal(t, []string{"52.216.0.0/30"}, got) +} + +func TestCoverPrefixes_MergesAdjacentSiblings(t *testing.T) { + // The two /25 halves of a /24 merge into the single parent /24. + got := coverPrefixes(nil, []netip.Prefix{ + netip.MustParsePrefix("10.0.0.0/25"), + netip.MustParsePrefix("10.0.0.128/25"), + }, 16) + assert.Equal(t, []string{"10.0.0.0/24"}, got) +} + +func TestCoverPrefixes_FloorCapSplitsBroadBlock(t *testing.T) { + // A pass-through /22 under a /24 floor splits into its four /24 children; + // none is broader than the floor. + got := coverPrefixes(nil, []netip.Prefix{netip.MustParsePrefix("10.9.0.0/22")}, 24) + assert.Equal(t, []string{"10.9.0.0/24", "10.9.1.0/24", "10.9.2.0/24", "10.9.3.0/24"}, got) +} diff --git a/pkg/registry/file/networkneighborhood_processor_test.go b/pkg/registry/file/networkneighborhood_processor_test.go index 67d8eba77..6d5e0fc7c 100644 --- a/pkg/registry/file/networkneighborhood_processor_test.go +++ b/pkg/registry/file/networkneighborhood_processor_test.go @@ -133,9 +133,9 @@ func TestNetworkNeighborhoodProcessor_PreSave(t *testing.T) { } func TestNetworkNeighborhoodProcessor_PreSave_IPCollapse(t *testing.T) { - const hostCount = 60 + const hostCount = 64 // a fully-observed /26 (10.0.0.0..10.0.0.63) ingress := make([]softwarecomposition.NetworkNeighbor, 0, hostCount) - for i := 1; i <= hostCount; i++ { + for i := 0; i < hostCount; i++ { ingress = append(ingress, softwarecomposition.NetworkNeighbor{ Identifier: fmt.Sprintf("external-%d", i), Type: "external",