turbine: accelerate 32+32 FEC generation and recovery

cmd/repair-sim/main.go

@@ -0,0 +1,137 @@
+// repair-sim runs deterministic, single-node Turbine repair scenarios.
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/Overclock-Validator/mithril/pkg/turbine/repairsim"
+)
+
+type environment struct {
+	GoVersion string `json:"go_version"`
+	GOOS      string `json:"goos"`
+	GOARCH    string `json:"goarch"`
+	CPU       string `json:"cpu"`
+}
+
+type report struct {
+	Environment          environment            `json:"environment"`
+	Ledger               repairsim.LedgerConfig `json:"ledger"`
+	Network              repairsim.Config       `json:"network"`
+	LedgerGenerationWall time.Duration          `json:"ledger_generation_wall_ns"`
+	Result               repairsim.Result       `json:"result"`
+}
+
+func main() {
+	var (
+		scenarioFlag = flag.String("scenario", string(repairsim.ScenarioNearTip), "near-tip or deep-catchup")
+		slots        = flag.Int("slots", 200, "number of deterministic slots")
+		fecSets      = flag.Int("fec-sets", 4, "FEC sets generated per slot")
+		entries      = flag.Int("entries", 0, "entries per slot (0 derives an exact FEC count)")
+		seed         = flag.Int64("seed", 1, "deterministic content and network seed")
+		availability = flag.String("availability", "", "complete, near-loss, sparse, or mixed")
+		repair       = flag.Bool("repair", true, "enable repair requests")
+		latency      = flag.Duration("repair-latency", 20*time.Millisecond, "synthetic one-way response latency")
+		jitter       = flag.Duration("repair-jitter", 2*time.Millisecond, "deterministic +/- response jitter")
+		loss         = flag.Float64("packet-loss", 0, "repair response loss probability [0,1]")
+		duplicates   = flag.Float64("duplicates", 0.02, "duplicate response probability [0,1]")
+		bandwidth    = flag.Int64("repair-bandwidth", 100*1024*1024, "synthetic repair bytes/sec (0 is unlimited)")
+		concurrent   = flag.Int("max-concurrent", 256, "maximum outstanding repair shreds")
+		corrupt      = flag.Int("corrupt-responses", 0, "corrupt the first N repair responses")
+		naturalLate  = flag.Bool("natural-late", true, "schedule selected late live shreds during repair")
+		spoolDir     = flag.String("spool-dir", "", "persistent shred-spool directory (empty uses a temporary directory)")
+		cpuLabel     = flag.String("cpu-label", "", "explicit CPU label when platform discovery is unavailable")
+		output       = flag.String("output", "", "write JSON to this file instead of stdout")
+		includeTrace = flag.Bool("trace", true, "include the logical event trace in JSON")
+	)
+	flag.Parse()
+
+	scenario := repairsim.Scenario(*scenarioFlag)
+	network := repairsim.DefaultConfig(scenario)
+	network.Availability = repairsim.Availability(*availability)
+	network.RepairEnabled = *repair
+	network.RepairLatency = *latency
+	network.RepairJitter = *jitter
+	network.PacketLoss = *loss
+	network.DuplicateProbability = *duplicates
+	network.BandwidthBytesPerSec = *bandwidth
+	network.MaxConcurrent = *concurrent
+	network.CorruptResponses = *corrupt
+	network.NaturalLateShreds = *naturalLate
+	network.CollectTrace = *includeTrace
+	network.Seed = *seed
+	network.SpoolDir = *spoolDir
+	if network.Availability == "" {
+		network.Availability = repairsim.DefaultConfig(scenario).Availability
+	}
+
+	ledgerCfg := repairsim.LedgerConfig{
+		StartSlot:      10_000,
+		Slots:          *slots,
+		FECsPerSlot:    *fecSets,
+		EntriesPerSlot: *entries,
+		Seed:           *seed,
+		ShredVersion:   1,
+		ReferenceTick:  63,
+	}
+	started := time.Now()
+	ledger, err := repairsim.GenerateLedger(ledgerCfg)
+	if err != nil {
+		fatalf("generate ledger: %v", err)
+	}
+	generationWall := time.Since(started)
+	result, err := repairsim.Run(ledger, network)
+	if err != nil {
+		fatalf("run simulation: %v", err)
+	}
+	cpu := *cpuLabel
+	if cpu == "" {
+		cpu = cpuModel()
+	}
+	report := report{
+		Environment: environment{GoVersion: runtime.Version(), GOOS: runtime.GOOS, GOARCH: runtime.GOARCH, CPU: cpu},
+		Ledger:      ledger.Config, Network: network, LedgerGenerationWall: generationWall, Result: result,
+	}
+	encoded, err := json.MarshalIndent(report, "", "  ")
+	if err != nil {
+		fatalf("marshal report: %v", err)
+	}
+	encoded = append(encoded, '\n')
+	if *output == "" {
+		_, _ = os.Stdout.Write(encoded)
+		return
+	}
+	if err := os.WriteFile(*output, encoded, 0o644); err != nil {
+		fatalf("write %s: %v", *output, err)
+	}
+}
+
+func cpuModel() string {
+	if runtime.GOOS == "linux" {
+		if data, err := os.ReadFile("/proc/cpuinfo"); err == nil {
+			for _, line := range strings.Split(string(data), "\n") {
+				if key, value, ok := strings.Cut(line, ":"); ok && strings.TrimSpace(key) == "model name" {
+					return strings.TrimSpace(value)
+				}
+			}
+		}
+	}
+	if runtime.GOOS == "darwin" {
+		if out, err := exec.Command("sysctl", "-n", "machdep.cpu.brand_string").Output(); err == nil {
+			return strings.TrimSpace(string(out))
+		}
+	}
+	return "unknown"
+}
+
+func fatalf(format string, args ...any) {
+	_, _ = fmt.Fprintf(os.Stderr, format+"\n", args...)
+	os.Exit(1)
+}

docs/erasure_recovery_experiments.md

@@ -0,0 +1,185 @@
+# Erasure recovery experiments
+
+Status: `SlotAssembler` dispatches the fixed 32+32, exactly-one-missing-data
+case to the direct recovery path. Reduced multi-missing and all-coding paths
+remain experimental and are not used by production dispatch.
+
+The end-to-end deterministic harness that drives production repair selection,
+assembly, storage, and completion is documented in [repair_sim.md](repair_sim.md).
+
+This document separates two repair regimes that have different objectives. It
+also records the fixed 32 data + 32 coding Reed-Solomon contract used by the
+synthetic implementation in `pkg/turbine/internal/rsrecover`.
+
+## Regimes
+
+Near-tip repair minimizes the time until replay receives a particular blocking
+data shred. Its primary candidate is direct recovery when exactly one data
+shred is absent and at least one coding shred is present.
+
+Catch-up repair minimizes useful recovered-data time across many incomplete FEC
+sets. Its candidate constructs only the reduced system induced by the missing
+data columns and produces only missing data outputs.
+
+Slot age alone should not select the regime. A future policy experiment should
+consume observed Turbine progress:
+
+- number and fraction of incomplete FEC sets;
+- missing data shreds per FEC set;
+- whether new shreds are still arriving;
+- time or scheduling intervals since the last useful arrival;
+- number of FEC sets that have crossed the recovery threshold;
+- data-heavy versus coding-heavy availability.
+
+A progressing slot with one or two holes remains a near-tip workload even if it
+is not the newest slot. A stalled slot with many incomplete FEC sets is a
+catch-up workload even if wall-clock age is modest. Any eventual selector needs
+hysteresis so bursty arrivals cannot oscillate the algorithm on every packet.
+
+## Matrix contract
+
+The experiment uses the systematic generator over `GF(256)/0x11d`:
+
+```text
+V[x,j] = x^j
+A      = V[0:32,0:32]
+G      = V * A^-1
+G      = [I_32; C]
+```
+
+For the fixed 32+32 shape, exhaustive scalar tests confirm all 1,024 entries:
+
+```text
+C[r,c] = 0xa5 / (0x20 xor r xor c)
+```
+
+They also confirm `C*C=I`. Recovery tests remain differential against
+`github.com/klauspost/reedsolomon`; the closed form is not the sole oracle.
+
+## Candidates
+
+### Near tip: direct one-data recovery
+
+For missing data position `m` and available coding position `r`:
+
+```text
+D_m = C[r,m]^-1 * P_r
+    + sum(i != m, C[r,m]^-1 * C[r,i] * D_i)
+```
+
+This prepares one 32-source coefficient row and writes one destination. It does
+not construct or invert a general 32x32 matrix.
+
+Production uses a process-wide table containing every missing-data and
+coding-row combination. This removes per-call plan construction while keeping
+the same equation. The table is exhaustively differential-tested against the
+general decoder across all 32 x 32 combinations.
+
+### Catch up: reduced missing-data system
+
+For missing data columns `M` and selected coding rows `R`, substitute every
+known data shard and solve:
+
+```text
+B[i,j] = C[R[i],M[j]]
+B * D_M = adjusted_coding_rows
+```
+
+The experiment uses the Cauchy closed form to construct `B^-1` in `O(m^2)`,
+expands direct rows over 32 selected sources, and proves each row satisfies the
+requested systematic generator row before byte processing. An independently
+implemented Gauss-Jordan inverse is the setup fallback.
+
+The byte kernel is intentionally portable and uses `reedsolomon.LowLevel`.
+This isolates algorithm and plan costs; it is not evidence that a portable
+kernel will beat the dependency's generated AVX2/GFNI kernels on amd64.
+
+### Catch up edge: all coding rows
+
+When all data rows are missing and every coding row is present, `C*C=I` means
+the existing optimized encoder can apply `C` to the coding rows and recover the
+data directly. This is kept as a separate synthetic arm. It is simpler than a
+general decoder, but a cached reduced-system plan may still have a faster byte
+kernel; hardware decides between them.
+
+## Synthetic coverage
+
+The tests cover:
+
+- every missing-data position with every coding-row choice for the direct path;
+- every pair of missing data positions;
+- deterministic mixed patterns at 2, 4, 8, 16, 24, and 32 missing data shreds;
+- exactly-threshold and one-below-threshold availability;
+- changed availability between setup and execution;
+- destination failure atomicity;
+- coefficient mutation detection;
+- Cauchy inverses against independent Gauss-Jordan inversion;
+- recovered bytes against the existing general decoder.
+
+Run:
+
+```bash
+go test ./pkg/turbine/internal/rsrecover
+go test -run '^$' \
+  -bench '^(BenchmarkRecoverOneData|BenchmarkRecoverDataSubset)$' \
+  -benchmem -benchtime=2s -count=6 \
+  ./pkg/turbine/internal/rsrecover
+```
+
+Benchmark result interpretation must keep these cases separate:
+
+- `prepare`: cold or changing erasure pattern;
+- `execute`: prepared/repeated pattern;
+- `prepare-and-execute`: first useful output for a new pattern;
+- general cache off: existing decoder with changing pattern cost;
+- general cache on: existing decoder after the inversion is cached.
+
+No production dispatch threshold should be chosen from an Apple benchmark.
+Final crossover decisions require the pinned amd64 target and synthetic arrival
+traces for progressing, stalled, and bursty slots.
+
+## Zen 5 production gate
+
+The direct one-data path was measured on a Ryzen 7 9700X with Go 1.26.4,
+`GOMAXPROCS=1`, and one pinned physical core. Medians below are from seven
+sequential one-second samples unless otherwise noted.
+
+| Benchmark | General path | Direct one-data path | Change |
+| --- | ---: | ---: | ---: |
+| one-missing `SlotAssembler` boundary | 10.73 us/FEC | 2.82 us/FEC | -73.7% (3.8x) |
+| near-tip repair simulation | 3.0295 ms/op | 2.8659 ms/op | -5.40% |
+| deep-mixed repair simulation | 4.0343 ms/op | 4.0240 ms/op | -0.26% |
+| deep-sparse repair simulation | 10.8489 ms/op | 10.8327 ms/op | -0.15% |
+
+The production dispatch is intentionally narrow. The deep scenarios do not
+enter it and remain effectively neutral, while the near-tip workload benefits
+from repeated exactly-one-missing recoveries. The one-missing boundary also
+dropped from 144 to 5 allocations per operation.
+
+## Preliminary Apple M4 Pro diagnostic
+
+These single-sample medians use 987-byte shards and exist only to reject or
+retain candidates before the amd64 gate. Times are microseconds per FEC set.
+
+| Missing data | Specialized first use | Specialized prepared | General uncached | General cached |
+|---:|---:|---:|---:|---:|
+| 1 | 1.97 | 1.07 | 8.36 | 2.31 |
+| 2 | 4.19 | 2.10 | 10.78 | 3.39 |
+| 4 | 8.84 | 4.19 | 17.00 | 5.91 |
+| 8 | 18.94 | 8.24 | 25.54 | 10.06 |
+| 16 | 39.90 | 16.44 | 44.93 | 19.78 |
+| 24 | 62.14 | 24.71 | 63.43 | 28.55 |
+| 32 | 79.50 | 32.76 | 89.53 | 37.53 |
+
+The all-coding involution arm measured approximately 36.5 microseconds,
+compared with 82.7 microseconds for an uncached general decode and 37.5
+microseconds for its cached form. Its main possible value is avoiding plan
+setup; the prepared reduced-system byte path was faster on this machine.
+
+The current interpretation is deliberately conditional:
+
+- direct one-data recovery is strong enough to require an amd64 prototype;
+- reduced-system first use wins through most of the tested range, but the
+  24-missing crossover is within noise on this machine;
+- prepared reduced-system execution wins at every tested width;
+- none of these figures establishes a production policy or Zen 5 result.