diff --git a/bft-sharding-compose.yml b/bft-sharding-compose.yml index fabb2ee..2ce8716 100644 --- a/bft-sharding-compose.yml +++ b/bft-sharding-compose.yml @@ -217,6 +217,7 @@ services: CONCURRENCY_LIMIT: "10000" ENABLE_DOCS: "true" ENABLE_CORS: "true" + MALLOC_ARENA_MAX: "${MALLOC_ARENA_MAX:-2}" MONGODB_URI: "mongodb://mongodb-shard0:27017/aggregator?replicaSet=rs0&directConnection=true" MONGODB_DATABASE: "aggregator_bft_shard_0" @@ -263,7 +264,7 @@ services: SMT_ROCKSDB_SUBCOMPACTIONS: "${SMT_ROCKSDB_SUBCOMPACTIONS:-4}" SMT_ROCKSDB_BLOOM_BITS: "${SMT_ROCKSDB_BLOOM_BITS:-10}" SMT_ROCKSDB_MEMTABLE_MB: "${SMT_ROCKSDB_MEMTABLE_MB:-64}" - SMT_MATERIALIZE_WORKERS: "${SMT_MATERIALIZE_WORKERS:-64}" + SMT_MATERIALIZE_WORKERS: "${SMT_MATERIALIZE_WORKERS:-16}" SMT_STARTUP_REPLAY_LIMIT_BLOCKS: "${SMT_STARTUP_REPLAY_LIMIT_BLOCKS:-100}" SHARDING_MODE: "bft-shard" diff --git a/bft-sharding-ha-compose.yml b/bft-sharding-ha-compose.yml index db45986..bbd0c90 100644 --- a/bft-sharding-ha-compose.yml +++ b/bft-sharding-ha-compose.yml @@ -222,6 +222,7 @@ services: CONCURRENCY_LIMIT: "10000" ENABLE_DOCS: "true" ENABLE_CORS: "true" + MALLOC_ARENA_MAX: "${MALLOC_ARENA_MAX:-2}" MONGODB_URI: "mongodb://mongodb-shard0:27017/aggregator?replicaSet=rs0&directConnection=true" MONGODB_DATABASE: "aggregator_bft_shard_0" @@ -271,7 +272,7 @@ services: SMT_ROCKSDB_SUBCOMPACTIONS: "${SMT_ROCKSDB_SUBCOMPACTIONS:-4}" SMT_ROCKSDB_BLOOM_BITS: "${SMT_ROCKSDB_BLOOM_BITS:-10}" SMT_ROCKSDB_MEMTABLE_MB: "${SMT_ROCKSDB_MEMTABLE_MB:-64}" - SMT_MATERIALIZE_WORKERS: "${SMT_MATERIALIZE_WORKERS:-64}" + SMT_MATERIALIZE_WORKERS: "${SMT_MATERIALIZE_WORKERS:-16}" SMT_STARTUP_REPLAY_LIMIT_BLOCKS: "${SMT_STARTUP_REPLAY_LIMIT_BLOCKS:-100}" SHARDING_MODE: "bft-shard" diff --git a/cmd/performance-test/main.go b/cmd/performance-test/main.go index b03bdb6..f24b2b0 100644 --- a/cmd/performance-test/main.go +++ b/cmd/performance-test/main.go @@ -8,8 +8,11 @@ import ( "encoding/hex" "encoding/json" "fmt" + "io" "log" "math" + "net/http" + "net/url" "os" "path/filepath" "runtime" @@ -43,6 +46,7 @@ const ( defaultStartupProbeWait = 60 * time.Second startupProbeInterval = 250 * time.Millisecond defaultBufferSeconds = 10 + defaultProofReadyMetric = "aggregator_proof_readiness_seconds" ) // Sharding modes for routing generated state IDs to shard endpoints. @@ -63,11 +67,14 @@ var ( proofRetryDelay = getEnvDuration("PROOF_RETRY_DELAY", defaultProofRetryDelay) proofInitialDelay = getEnvDuration("PROOF_INITIAL_DELAY", defaultProofInitialDelay) startupProbeWait = getEnvDuration("STARTUP_PROBE_WAIT", defaultStartupProbeWait) + requestWireFormat = strings.ToLower(strings.TrimSpace(os.Getenv("CERTIFICATION_REQUEST_WIRE_FORMAT"))) commitmentBuffer = getCommitmentBufferSize() verifyProofs = getEnvBool("VERIFY_PROOFS", true) + verifyProofCrypto = getEnvBool("VERIFY_PROOF_CRYPTO", true) shardingMode = getShardingMode() shardTargets = getEnvShardTargets() enableH2C = os.Getenv("ENABLE_H2C") != "false" + proofReadyMetric = getEnvString("SERVER_PROOF_READINESS_METRIC", defaultProofReadyMetric) ) func getShardingMode() string { @@ -122,6 +129,13 @@ func getEnvBool(key string, defaultVal bool) bool { return defaultVal } +func getEnvString(key string, defaultVal string) string { + if val := strings.TrimSpace(os.Getenv(key)); val != "" { + return val + } + return defaultVal +} + func waitForStartingBlock(sc *ShardClient, timeout time.Duration) (int64, error) { deadline := time.Now().Add(timeout) var lastErr error @@ -324,6 +338,284 @@ func buildShardClients(aggregatorURL, authHeader string, metrics *Metrics) []*Sh return clients } +type proofReadinessHistogramSnapshot struct { + metricName string + buckets map[float64]float64 + targets int +} + +func metricsEndpoint(target string) (string, error) { + parsed, err := url.Parse(target) + if err != nil { + return "", err + } + if parsed.Scheme == "" || parsed.Host == "" { + return "", fmt.Errorf("target URL must include scheme and host: %q", target) + } + parsed.Path = "/metrics" + parsed.RawPath = "" + parsed.RawQuery = "" + parsed.Fragment = "" + return parsed.String(), nil +} + +func scrapeProofReadinessHistogram(ctx context.Context, shardClients []*ShardClient, authHeader string) (*proofReadinessHistogramSnapshot, error) { + if len(shardClients) == 0 { + return nil, fmt.Errorf("no shard clients configured") + } + + snapshot := &proofReadinessHistogramSnapshot{ + metricName: proofReadyMetric, + buckets: make(map[float64]float64), + } + var errors []string + for _, sc := range shardClients { + endpoint, err := metricsEndpoint(sc.url) + if err != nil { + errors = append(errors, fmt.Sprintf("%s: %v", sc.url, err)) + continue + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + errors = append(errors, fmt.Sprintf("%s: %v", endpoint, err)) + continue + } + if authHeader != "" { + req.Header.Set("Authorization", authHeader) + } + + client := newHTTPClient(sc.url, nil) + resp, err := client.Do(req) + if err != nil { + errors = append(errors, fmt.Sprintf("%s: %v", endpoint, err)) + continue + } + body, readErr := io.ReadAll(resp.Body) + closeErr := resp.Body.Close() + if readErr != nil { + errors = append(errors, fmt.Sprintf("%s: %v", endpoint, readErr)) + continue + } + if closeErr != nil { + errors = append(errors, fmt.Sprintf("%s: %v", endpoint, closeErr)) + continue + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + errors = append(errors, fmt.Sprintf("%s: HTTP %d", endpoint, resp.StatusCode)) + continue + } + + buckets, err := parseProofReadinessBuckets(body, proofReadyMetric) + if err != nil { + errors = append(errors, fmt.Sprintf("%s: %v", endpoint, err)) + continue + } + for le, value := range buckets { + snapshot.buckets[le] += value + } + snapshot.targets++ + } + + if snapshot.targets == 0 { + if len(errors) == 0 { + return nil, fmt.Errorf("no %s_bucket metrics found", proofReadyMetric) + } + return nil, fmt.Errorf("no %s_bucket metrics found (%s)", proofReadyMetric, strings.Join(errors, "; ")) + } + if len(errors) > 0 { + fmt.Printf("Warning: skipped %d metrics target(s): %s\n", len(errors), strings.Join(errors, "; ")) + } + return snapshot, nil +} + +func parseProofReadinessBuckets(body []byte, metricName string) (map[float64]float64, error) { + bucketMetric := metricName + "_bucket" + buckets := make(map[float64]float64) + scanner := bufio.NewScanner(bytes.NewReader(body)) + scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + if !strings.HasPrefix(line, bucketMetric+"{") && !strings.HasPrefix(line, bucketMetric+" ") { + continue + } + + le, ok := parseHistogramLE(line) + if !ok { + continue + } + fields := strings.Fields(line) + if len(fields) < 2 { + continue + } + value, err := strconv.ParseFloat(fields[1], 64) + if err != nil { + continue + } + buckets[le] += value + } + if err := scanner.Err(); err != nil { + return nil, err + } + if len(buckets) == 0 { + return nil, fmt.Errorf("metric %s not present", bucketMetric) + } + return buckets, nil +} + +func parseHistogramLE(line string) (float64, bool) { + idx := strings.Index(line, `le="`) + if idx < 0 { + return 0, false + } + rest := line[idx+len(`le="`):] + end := strings.IndexByte(rest, '"') + if end < 0 { + return 0, false + } + raw := rest[:end] + if raw == "+Inf" { + return math.Inf(1), true + } + le, err := strconv.ParseFloat(raw, 64) + return le, err == nil +} + +func proofReadinessHistogramDelta(before, after *proofReadinessHistogramSnapshot) (*proofReadinessHistogramSnapshot, error) { + if before == nil || after == nil { + return nil, fmt.Errorf("missing before/after histogram snapshot") + } + if before.metricName != after.metricName { + return nil, fmt.Errorf("metric changed from %s to %s", before.metricName, after.metricName) + } + delta := &proofReadinessHistogramSnapshot{ + metricName: after.metricName, + buckets: make(map[float64]float64), + targets: after.targets, + } + for le, afterValue := range after.buckets { + value := afterValue - before.buckets[le] + if value < 0 { + return nil, fmt.Errorf("histogram bucket %v decreased from %.0f to %.0f", le, before.buckets[le], afterValue) + } + delta.buckets[le] = value + } + return delta, nil +} + +func histogramTotal(buckets map[float64]float64) float64 { + if total, ok := buckets[math.Inf(1)]; ok { + return total + } + var total float64 + for le, value := range buckets { + if !math.IsInf(le, 1) && value > total { + total = value + } + } + return total +} + +func histogramCountLE(buckets map[float64]float64, target float64) float64 { + var bestLE float64 + var bestValue float64 + found := false + for le, value := range buckets { + if math.IsInf(le, 1) || le > target { + continue + } + if !found || le > bestLE { + bestLE = le + bestValue = value + found = true + } + } + if !found { + return 0 + } + return bestValue +} + +func histogramQuantile(q float64, buckets map[float64]float64) float64 { + total := histogramTotal(buckets) + if total <= 0 { + return math.NaN() + } + rank := q * total + bounds := make([]float64, 0, len(buckets)) + for le := range buckets { + if !math.IsInf(le, 1) { + bounds = append(bounds, le) + } + } + sort.Float64s(bounds) + + var prevBound float64 + var prevCount float64 + for _, bound := range bounds { + count := buckets[bound] + if count >= rank { + bucketCount := count - prevCount + if bucketCount <= 0 { + return bound + } + fraction := (rank - prevCount) / bucketCount + return prevBound + (bound-prevBound)*fraction + } + prevBound = bound + prevCount = count + } + if len(bounds) > 0 { + return bounds[len(bounds)-1] + } + return math.NaN() +} + +func formatSeconds(seconds float64) string { + if math.IsNaN(seconds) { + return "n/a" + } + if math.IsInf(seconds, 1) { + return "+Inf" + } + return time.Duration(seconds * float64(time.Second)).Truncate(time.Millisecond).String() +} + +func printServerProofReadinessHistogram(before, after *proofReadinessHistogramSnapshot, beforeErr, afterErr error) { + fmt.Printf("\nSERVER PROOF READINESS HISTOGRAM:\n") + if beforeErr != nil { + fmt.Printf(" unavailable before test: %v\n", beforeErr) + return + } + if afterErr != nil { + fmt.Printf(" unavailable after test: %v\n", afterErr) + return + } + + delta, err := proofReadinessHistogramDelta(before, after) + if err != nil { + fmt.Printf(" unavailable: %v\n", err) + return + } + + total := histogramTotal(delta.buckets) + if total <= 0 { + fmt.Printf(" no samples in test window for %s_bucket\n", delta.metricName) + return + } + within1s := histogramCountLE(delta.buckets, 1.0) + fmt.Printf(" Source: direct /metrics scrape, %s_bucket, %d target(s)\n", delta.metricName, delta.targets) + fmt.Printf(" Samples: %.0f\n", total) + fmt.Printf(" <=1s: %.0f/%.0f (%.1f%%)\n", within1s, total, within1s/total*100) + fmt.Printf(" Global server proof readiness: p50 %s, p95 %s, p99 %s\n", + formatSeconds(histogramQuantile(0.50, delta.buckets)), + formatSeconds(histogramQuantile(0.95, delta.buckets)), + formatSeconds(histogramQuantile(0.99, delta.buckets))) +} + func selectShardIndex(stateID api.StateID, shardClients []*ShardClient) int { shardCount := len(shardClients) if shardCount <= 1 { @@ -472,7 +764,7 @@ func generateCommitmentRequest() *api.CertificationRequest { } for { - calculated, err := api.CreateStateID(ownerPredicate, sourceStateHash) + calculated, err := createStateIDForWireFormat(ownerPredicate, sourceStateHash) if err != nil { panic(fmt.Sprintf("Failed to create state ID: %v", err)) } @@ -512,6 +804,49 @@ func generateCommitmentRequest() *api.CertificationRequest { } } +// Wire-format hooks allow optional local overrides of request/response +// encoding. When no hook is registered, the standard JSON/CBOR path is used. +var ( + stateIDWireFormatHook func(ownerPredicate api.Predicate, sourceStateHash api.SourceStateHash) (api.StateID, bool, error) + encodeParamsHook func(method string, params interface{}) (interface{}, bool, error) + decodeProofHook func(respBytes []byte) (*api.GetInclusionProofResponseV2, bool, error) +) + +func createStateIDForWireFormat(ownerPredicate api.Predicate, sourceStateHash api.SourceStateHash) (api.StateID, error) { + if stateIDWireFormatHook != nil { + if id, handled, err := stateIDWireFormatHook(ownerPredicate, sourceStateHash); handled { + return id, err + } + } + return api.CreateStateID(ownerPredicate, sourceStateHash) +} + +func encodeJSONRPCParams(method string, params interface{}) (interface{}, error) { + if encodeParamsHook != nil { + if out, handled, err := encodeParamsHook(method, params); handled { + return out, err + } + } + return params, nil +} + +func decodeProofResponse(result interface{}) (*api.GetInclusionProofResponseV2, error) { + respBytes, err := json.Marshal(result) + if err != nil { + return nil, fmt.Errorf("marshal proof response: %w", err) + } + if decodeProofHook != nil { + if resp, handled, err := decodeProofHook(respBytes); handled { + return resp, err + } + } + var proofResp api.GetInclusionProofResponseV2 + if err := json.Unmarshal(respBytes, &proofResp); err != nil { + return nil, err + } + return &proofResp, nil +} + func startCommitmentGenerator(ctx context.Context, total, bufferSize, workers, progressEvery int, label string) <-chan *api.CertificationRequest { if bufferSize < 1 { bufferSize = 1 @@ -809,17 +1144,8 @@ func verifyProofJob(ctx context.Context, shardClients []*ShardClient, metrics *M continue } - var proofResp api.GetInclusionProofResponseV2 - respBytes, err := json.Marshal(resp.Result) + proofResp, err := decodeProofResponse(resp.Result) if err != nil { - metrics.recordError(fmt.Sprintf("Failed to marshal proof response: %v", err)) - atomic.AddInt64(&metrics.proofFailed, 1) - if sm := metrics.shard(shardIdx); sm != nil { - sm.proofFailed.Add(1) - } - return - } - if err := json.Unmarshal(respBytes, &proofResp); err != nil { metrics.recordError(fmt.Sprintf("Failed to parse proof response: %v", err)) atomic.AddInt64(&metrics.proofFailed, 1) if sm := metrics.shard(shardIdx); sm != nil { @@ -853,6 +1179,14 @@ func verifyProofJob(ctx context.Context, shardClients []*ShardClient, metrics *M } metrics.addProofLatency(totalLatency) + if !verifyProofCrypto { + atomic.AddInt64(&metrics.proofVerified, 1) + if sm := metrics.shard(shardIdx); sm != nil { + sm.proofVerified.Add(1) + } + return + } + if err := proofverify.VerifyInclusionProofLocal(proofResp.InclusionProof, job.request); err != nil { if attempt < proofMaxRetries-1 { if !sleepOrDone(ctx, proofRetryDelay) { @@ -1087,15 +1421,16 @@ func parseAggregatorRoundLogs(path string, start, end time.Time) ([]aggregatorRo if err := json.Unmarshal(line, &raw); err != nil { continue } - if raw.Msg != "PERF: Round completed" { - continue - } timestamp, err := time.Parse(time.RFC3339Nano, raw.Time) if err != nil { continue } - if timestamp.Before(start) || timestamp.After(end) { + if timestamp.Before(start) || timestamp.After(end) || raw.Block == "" { + continue + } + + if raw.Msg != "Round completed" { continue } @@ -1156,6 +1491,30 @@ func parseAggregatorRoundLogs(path string, start, end time.Time) ([]aggregatorRo if err != nil { continue } + finalizeSmtCommitCollect, hasFinalizeSmtCommitCollect, err := parseOptionalLogDuration(raw.FinalizeSmtCommitCollect) + if err != nil { + continue + } + finalizeSmtCommitTombstone, hasFinalizeSmtCommitTombstone, err := parseOptionalLogDuration(raw.FinalizeSmtCommitTombstone) + if err != nil { + continue + } + finalizeSmtCommitBatchBuild, hasFinalizeSmtCommitBatchBuild, err := parseOptionalLogDuration(raw.FinalizeSmtCommitBatchBuild) + if err != nil { + continue + } + finalizeSmtCommitRootHash, hasFinalizeSmtCommitRootHash, err := parseOptionalLogDuration(raw.FinalizeSmtCommitRootHash) + if err != nil { + continue + } + finalizeSmtCommitEngineWrite, hasFinalizeSmtCommitEngineWrite, err := parseOptionalLogDuration(raw.FinalizeSmtCommitEngineWrite) + if err != nil { + continue + } + finalizeSmtCommitCacheUpdate, hasFinalizeSmtCommitCacheUpdate, err := parseOptionalLogDuration(raw.FinalizeSmtCommitCacheUpdate) + if err != nil { + continue + } finalizeSetFinalized, hasFinalizeSetFinalized, err := parseOptionalLogDuration(raw.FinalizeSetFinalized) if err != nil { continue @@ -1164,7 +1523,12 @@ func parseAggregatorRoundLogs(path string, start, end time.Time) ([]aggregatorRo if err != nil { continue } - hasFinalizationBreakdown := hasFinalizeScan || hasFinalizeConvert || hasFinalizeStoreBlock || hasFinalizeStoreBlockDoc || hasFinalizeStoreBlockRecords || hasFinalizeStoreData || hasFinalizeStoreSmt || hasFinalizeStoreRecords || hasFinalizeLockWait || hasFinalizeSmtCommit || hasFinalizeSetFinalized || hasFinalizeAck + hasFinalizationBreakdown := hasFinalizeScan || hasFinalizeConvert || hasFinalizeStoreBlock || hasFinalizeStoreBlockDoc || hasFinalizeStoreBlockRecords || hasFinalizeStoreData || hasFinalizeStoreSmt || hasFinalizeStoreRecords || hasFinalizeLockWait || hasFinalizeSmtCommit || hasFinalizeSmtCommitCollect || hasFinalizeSmtCommitTombstone || hasFinalizeSmtCommitBatchBuild || hasFinalizeSmtCommitRootHash || hasFinalizeSmtCommitEngineWrite || hasFinalizeSmtCommitCacheUpdate || hasFinalizeSetFinalized || hasFinalizeAck + + proposalToProofReadyDur, hasProposalToProofReady, err := parseOptionalLogDuration(raw.ProposalToProofReady) + if err != nil { + continue + } medianDur, hasMedian, err := parseOptionalLogDuration(raw.ProofReadyMedian) if err != nil { @@ -1183,34 +1547,45 @@ func parseAggregatorRoundLogs(path string, start, end time.Time) ([]aggregatorRo continue } - summaries = append(summaries, aggregatorRoundSummary{ - Timestamp: timestamp, - Block: raw.Block, - Commitments: raw.Commitments, - RoundTime: roundDur, - Processing: procDur, - BftWait: bftDur, - Finalization: finalDur, - HasFinalizationBreakdown: hasFinalizationBreakdown, - FinalizeScan: finalizeScan, - FinalizeConvert: finalizeConvert, - FinalizeStoreBlock: finalizeStoreBlock, - FinalizeStoreBlockDoc: finalizeStoreBlockDoc, - FinalizeStoreBlockRecords: finalizeStoreBlockRecords, - FinalizeStoreData: finalizeStoreData, - FinalizeStoreSmt: finalizeStoreSmt, - FinalizeStoreRecords: finalizeStoreRecords, - FinalizeLockWait: finalizeLockWait, - FinalizeSmtCommit: finalizeSmtCommit, - FinalizeSetFinalized: finalizeSetFinalized, - FinalizeAck: finalizeAck, - HasProofReady: hasProofReady, - ProofMedian: medianDur, - ProofP95: p95Dur, - ProofP99: p99Dur, - RedisTotal: raw.RedisTotal, - RedisPending: raw.RedisPending, - }) + summary := aggregatorRoundSummary{ + Timestamp: timestamp, + Block: raw.Block, + Commitments: raw.Commitments, + RoundTime: roundDur, + Processing: procDur, + BftWait: bftDur, + Finalization: finalDur, + HasFinalizationBreakdown: hasFinalizationBreakdown, + FinalizeScan: finalizeScan, + FinalizeConvert: finalizeConvert, + FinalizeStoreBlock: finalizeStoreBlock, + FinalizeStoreBlockDoc: finalizeStoreBlockDoc, + FinalizeStoreBlockRecords: finalizeStoreBlockRecords, + FinalizeStoreData: finalizeStoreData, + FinalizeStoreSmt: finalizeStoreSmt, + FinalizeStoreRecords: finalizeStoreRecords, + FinalizeLockWait: finalizeLockWait, + FinalizeSmtCommit: finalizeSmtCommit, + FinalizeSmtCommitCollect: finalizeSmtCommitCollect, + FinalizeSmtCommitTombstone: finalizeSmtCommitTombstone, + FinalizeSmtCommitBatchBuild: finalizeSmtCommitBatchBuild, + FinalizeSmtCommitRootHash: finalizeSmtCommitRootHash, + FinalizeSmtCommitEngineWrite: finalizeSmtCommitEngineWrite, + FinalizeSmtCommitCacheUpdate: finalizeSmtCommitCacheUpdate, + FinalizeSmtCommitNodeWrites: raw.FinalizeSmtCommitNodeWrites, + FinalizeSmtCommitNodeDeletes: raw.FinalizeSmtCommitNodeDeletes, + FinalizeSetFinalized: finalizeSetFinalized, + FinalizeAck: finalizeAck, + HasProposalToProofReady: hasProposalToProofReady, + ProposalToProofReady: proposalToProofReadyDur, + HasProofReady: hasProofReady, + ProofMedian: medianDur, + ProofP95: p95Dur, + ProofP99: p99Dur, + RedisTotal: raw.RedisTotal, + RedisPending: raw.RedisPending, + } + summaries = append(summaries, summary) } if err := scanner.Err(); err != nil { @@ -1220,11 +1595,131 @@ func parseAggregatorRoundLogs(path string, start, end time.Time) ([]aggregatorRo return summaries, nil } +func parseAggregatorPrecollectorLogs(path string, start, end time.Time) ([]aggregatorPrecollectorSummary, error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + + if end.Before(start) { + start, end = end, start + } + + scanner := bufio.NewScanner(file) + buf := make([]byte, 0, 1024*1024) + scanner.Buffer(buf, 10*1024*1024) + + var summaries []aggregatorPrecollectorSummary + for scanner.Scan() { + line := bytes.TrimSpace(scanner.Bytes()) + if len(line) == 0 { + continue + } + + var raw aggregatorLogRaw + if err := json.Unmarshal(line, &raw); err != nil { + continue + } + if raw.Msg != "Precollector prepared" && raw.Msg != "Precollector advanced" { + continue + } + + timestamp, err := time.Parse(time.RFC3339Nano, raw.Time) + if err != nil { + continue + } + if timestamp.Before(start) || timestamp.After(end) { + continue + } + + advanceFlush, _, err := parseOptionalLogDuration(raw.AdvanceFlush) + if err != nil { + continue + } + flushTotal, _, err := parseOptionalLogDuration(raw.FlushTotal) + if err != nil { + continue + } + flushMax, _, err := parseOptionalLogDuration(raw.FlushMax) + if err != nil { + continue + } + stageTotal, _, err := parseOptionalLogDuration(raw.StageTotal) + if err != nil { + continue + } + stageMax, _, err := parseOptionalLogDuration(raw.StageMax) + if err != nil { + continue + } + fork, _, err := parseOptionalLogDuration(raw.Fork) + if err != nil { + continue + } + total, _, err := parseOptionalLogDuration(raw.Total) + if err != nil { + continue + } + + summaries = append(summaries, aggregatorPrecollectorSummary{ + Timestamp: timestamp, + Prepared: raw.Msg == "Precollector prepared", + Advance: raw.Msg == "Precollector advanced", + Commitments: raw.Commitments, + Leaves: raw.Leaves, + PendingAtAdvance: raw.PendingAtAdvance, + TailMerged: raw.TailMerged, + TailRemaining: raw.TailRemaining, + AlreadyPrepared: raw.AlreadyPrepared, + AdvanceFlush: advanceFlush, + FlushCalls: raw.FlushCalls, + FlushAdded: raw.FlushAdded, + FlushTotal: flushTotal, + FlushMax: flushMax, + StageCalls: raw.StageCalls, + StageAdded: raw.StageAdded, + StageTotal: stageTotal, + StageMax: stageMax, + Fork: fork, + Total: total, + }) + } + + if err := scanner.Err(); err != nil { + return summaries, err + } + return summaries, nil +} + type aggregatorLogSource struct { label string path string } +type aggregatorPrecollectorSummary struct { + Timestamp time.Time + Prepared bool + Advance bool + Commitments int + Leaves int + PendingAtAdvance int + TailMerged int + TailRemaining int + AlreadyPrepared bool + AdvanceFlush time.Duration + FlushCalls int + FlushAdded int + FlushTotal time.Duration + FlushMax time.Duration + StageCalls int + StageAdded int + StageTotal time.Duration + StageMax time.Duration + Fork time.Duration + Total time.Duration +} + func parseAggregatorLogSourcesOverride(raw string) []aggregatorLogSource { entries := strings.Split(raw, ",") sources := make([]aggregatorLogSource, 0, len(entries)) @@ -1339,6 +1834,9 @@ func printFinalizationBreakdownSummary(label string, entries []aggregatorRoundSu var scanSum, convertSum, storeBlockSum, storeBlockDocSum, storeBlockRecordsSum, storeDataSum time.Duration var storeSmtSum, storeRecordsSum, lockWaitSum time.Duration var smtCommitSum, setFinalizedSum, ackSum time.Duration + var smtCommitCollectSum, smtCommitTombstoneSum, smtCommitBatchBuildSum time.Duration + var smtCommitRootHashSum, smtCommitEngineWriteSum, smtCommitCacheUpdateSum time.Duration + var smtCommitNodeWritesSum, smtCommitNodeDeletesSum int for _, entry := range withBreakdown { scanSum += entry.FinalizeScan convertSum += entry.FinalizeConvert @@ -1350,12 +1848,21 @@ func printFinalizationBreakdownSummary(label string, entries []aggregatorRoundSu storeRecordsSum += entry.FinalizeStoreRecords lockWaitSum += entry.FinalizeLockWait smtCommitSum += entry.FinalizeSmtCommit + smtCommitCollectSum += entry.FinalizeSmtCommitCollect + smtCommitTombstoneSum += entry.FinalizeSmtCommitTombstone + smtCommitBatchBuildSum += entry.FinalizeSmtCommitBatchBuild + smtCommitRootHashSum += entry.FinalizeSmtCommitRootHash + smtCommitEngineWriteSum += entry.FinalizeSmtCommitEngineWrite + smtCommitCacheUpdateSum += entry.FinalizeSmtCommitCacheUpdate + smtCommitNodeWritesSum += entry.FinalizeSmtCommitNodeWrites + smtCommitNodeDeletesSum += entry.FinalizeSmtCommitNodeDeletes setFinalizedSum += entry.FinalizeSetFinalized ackSum += entry.FinalizeAck } count := time.Duration(len(withBreakdown)) - fmt.Printf("%s finalization breakdown: scan=%v convert=%v storeBlock=%v (blockDoc=%v blockRecords=%v) storeData=%v (smt=%v records=%v) lockWait=%v smtCommit=%v setFinalized=%v ack=%v (%d rounds)\n", + intCount := len(withBreakdown) + fmt.Printf("%s finalization breakdown: scan=%v convert=%v storeBlock=%v (blockDoc=%v blockRecords=%v) storeData=%v (smt=%v records=%v) lockWait=%v smtCommit=%v (collect=%v tombstone=%v batchBuild=%v rootHash=%v engineWrite=%v cacheUpdate=%v nodeWrites=%d nodeDeletes=%d) setFinalized=%v ack=%v (%d rounds)\n", prefix, (scanSum / count).Truncate(time.Millisecond), (convertSum / count).Truncate(time.Millisecond), @@ -1367,11 +1874,118 @@ func printFinalizationBreakdownSummary(label string, entries []aggregatorRoundSu (storeRecordsSum / count).Truncate(time.Millisecond), (lockWaitSum / count).Truncate(time.Millisecond), (smtCommitSum / count).Truncate(time.Millisecond), + (smtCommitCollectSum / count).Truncate(time.Millisecond), + (smtCommitTombstoneSum / count).Truncate(time.Millisecond), + (smtCommitBatchBuildSum / count).Truncate(time.Millisecond), + (smtCommitRootHashSum / count).Truncate(time.Millisecond), + (smtCommitEngineWriteSum / count).Truncate(time.Millisecond), + (smtCommitCacheUpdateSum / count).Truncate(time.Millisecond), + smtCommitNodeWritesSum/intCount, + smtCommitNodeDeletesSum/intCount, (setFinalizedSum / count).Truncate(time.Millisecond), (ackSum / count).Truncate(time.Millisecond), len(withBreakdown)) } +func printPrecollectorBreakdownSummary(label string, entries []aggregatorPrecollectorSummary) { + prefix := "Average" + if label != "" { + prefix = label + " average" + } + if len(entries) == 0 { + return + } + + var prepareCount, advanceCount int + var prepareTotal, prepareFlushTotal, prepareStageTotal, prepareFork time.Duration + var prepareCommitments, prepareLeaves, prepareFlushCalls, prepareFlushAdded, prepareStageCalls, prepareStageAdded int + var advanceTotal, advanceFlush, advanceFlushTotal, advanceFlushMax, advanceStageTotal, advanceStageMax, advanceFork time.Duration + var advanceCommitments, advanceLeaves, advancePending, advanceTailMerged, advanceTailRemaining int + var advanceFlushCalls, advanceFlushAdded, advanceStageCalls, advanceStageAdded, alreadyPrepared int + + for _, entry := range entries { + if entry.Prepared { + prepareCount++ + prepareTotal += entry.Total + prepareFlushTotal += entry.FlushTotal + prepareStageTotal += entry.StageTotal + prepareFork += entry.Fork + prepareCommitments += entry.Commitments + prepareLeaves += entry.Leaves + prepareFlushCalls += entry.FlushCalls + prepareFlushAdded += entry.FlushAdded + prepareStageCalls += entry.StageCalls + prepareStageAdded += entry.StageAdded + } + if entry.Advance { + advanceCount++ + advanceTotal += entry.Total + advanceFlush += entry.AdvanceFlush + advanceFlushTotal += entry.FlushTotal + if entry.FlushMax > advanceFlushMax { + advanceFlushMax = entry.FlushMax + } + advanceStageTotal += entry.StageTotal + if entry.StageMax > advanceStageMax { + advanceStageMax = entry.StageMax + } + advanceFork += entry.Fork + advanceCommitments += entry.Commitments + advanceLeaves += entry.Leaves + advancePending += entry.PendingAtAdvance + advanceTailMerged += entry.TailMerged + advanceTailRemaining += entry.TailRemaining + advanceFlushCalls += entry.FlushCalls + advanceFlushAdded += entry.FlushAdded + advanceStageCalls += entry.StageCalls + advanceStageAdded += entry.StageAdded + if entry.AlreadyPrepared { + alreadyPrepared++ + } + } + } + + if prepareCount > 0 { + count := time.Duration(prepareCount) + fmt.Printf("%s precollector prepare: total=%v snapshotAdd/materialize=%v durableStage=%v fork=%v commitments=%.0f leaves=%.0f flushCalls=%.1f flushAdded=%.0f stageCalls=%.1f stageAdded=%.0f (%d prepares)\n", + prefix, + (prepareTotal / count).Truncate(time.Millisecond), + (prepareFlushTotal / count).Truncate(time.Millisecond), + (prepareStageTotal / count).Truncate(time.Millisecond), + (prepareFork / count).Truncate(time.Millisecond), + float64(prepareCommitments)/float64(prepareCount), + float64(prepareLeaves)/float64(prepareCount), + float64(prepareFlushCalls)/float64(prepareCount), + float64(prepareFlushAdded)/float64(prepareCount), + float64(prepareStageCalls)/float64(prepareCount), + float64(prepareStageAdded)/float64(prepareCount), + prepareCount) + } + if advanceCount > 0 { + count := time.Duration(advanceCount) + fmt.Printf("%s precollector advance: total=%v advanceFlush=%v cumulativeSnapshotAdd/materialize=%v flushMax=%v cumulativeDurableStage=%v stageMax=%v fork=%v commitments=%.0f pendingAtAdvance=%.0f tailMerged=%.0f tailRemaining=%.0f alreadyPrepared=%d/%d flushCalls=%.1f flushAdded=%.0f stageCalls=%.1f stageAdded=%.0f (%d advances)\n", + prefix, + (advanceTotal / count).Truncate(time.Millisecond), + (advanceFlush / count).Truncate(time.Millisecond), + (advanceFlushTotal / count).Truncate(time.Millisecond), + advanceFlushMax.Truncate(time.Millisecond), + (advanceStageTotal / count).Truncate(time.Millisecond), + advanceStageMax.Truncate(time.Millisecond), + (advanceFork / count).Truncate(time.Millisecond), + float64(advanceCommitments)/float64(advanceCount), + float64(advancePending)/float64(advanceCount), + float64(advanceTailMerged)/float64(advanceCount), + float64(advanceTailRemaining)/float64(advanceCount), + alreadyPrepared, + advanceCount, + float64(advanceFlushCalls)/float64(advanceCount), + float64(advanceFlushAdded)/float64(advanceCount), + float64(advanceStageCalls)/float64(advanceCount), + float64(advanceStageAdded)/float64(advanceCount), + advanceCount) + } +} + func printAggregatorAverages(label string, entries []aggregatorRoundSummary) { prefix := "Average" if label != "" { @@ -1383,8 +1997,10 @@ func printAggregatorAverages(label string, entries []aggregatorRoundSummary) { } var roundSum, finalSum, procSum, bftSum time.Duration + var proposalToProofReadySum time.Duration var proofMedSum, proofP95Sum, proofP99Sum time.Duration totalCommitments := 0 + proposalToProofReadyCount := 0 proofCount := 0 for _, entry := range entries { roundSum += entry.RoundTime @@ -1392,6 +2008,10 @@ func printAggregatorAverages(label string, entries []aggregatorRoundSummary) { procSum += entry.Processing bftSum += entry.BftWait totalCommitments += entry.Commitments + if entry.HasProposalToProofReady { + proposalToProofReadySum += entry.ProposalToProofReady + proposalToProofReadyCount++ + } if entry.HasProofReady { proofMedSum += entry.ProofMedian proofP95Sum += entry.ProofP95 @@ -1420,6 +2040,15 @@ func printAggregatorAverages(label string, entries []aggregatorRoundSummary) { fmt.Printf("%s commitments per round: %.0f\n", prefix, avgCommit) fmt.Printf("%s processing time: %v (%.1f%% of round time)\n", prefix, avgProcessing.Truncate(time.Millisecond), procPct) fmt.Printf("%s BFT wait: %v (%.1f%% of round time)\n", prefix, avgBft.Truncate(time.Millisecond), bftPct) + if proposalToProofReadyCount > 0 { + proposalToProofReadyCountDuration := time.Duration(proposalToProofReadyCount) + fmt.Printf("%s proposal to proof-ready: %v (%d rounds)\n", + prefix, + (proposalToProofReadySum / proposalToProofReadyCountDuration).Truncate(time.Millisecond), + proposalToProofReadyCount) + } else { + fmt.Printf("%s proposal to proof-ready: n/a (no proposal-to-proof-ready rounds in window)\n", prefix) + } if proofCount > 0 { proofCountDuration := time.Duration(proofCount) fmt.Printf("%s proof readiness: median %v, p95 %v, p99 %v (%d rounds)\n", @@ -1478,6 +2107,7 @@ func reportAggregatorServerStats(start, end time.Time, shardClients []*ShardClie foundSources := 0 combined := make([]aggregatorRoundSummary, 0) + combinedPrecollector := make([]aggregatorPrecollectorSummary, 0) readErrors := make([]string, 0) noDataSources := make([]aggregatorLogSource, 0) for _, source := range sources { @@ -1496,9 +2126,16 @@ func reportAggregatorServerStats(start, end time.Time, shardClients []*ShardClie continue } + precollectorSummaries, err := parseAggregatorPrecollectorLogs(source.path, start, end) + if err != nil { + readErrors = append(readErrors, fmt.Sprintf("failed to read precollector logs from %s (%s): %v", source.path, source.label, err)) + } + foundSources++ combined = append(combined, summaries...) + combinedPrecollector = append(combinedPrecollector, precollectorSummaries...) printAggregatorServerStatsSummary(fmt.Sprintf("%s (%s)", source.label, source.path), summaries) + printPrecollectorBreakdownSummary(fmt.Sprintf("%s (%s)", source.label, source.path), precollectorSummaries) } if foundSources == 0 { @@ -1525,6 +2162,7 @@ func reportAggregatorServerStats(start, end time.Time, shardClients []*ShardClie if foundSources > 1 { printAggregatorServerStatsSummary("combined", combined) + printPrecollectorBreakdownSummary("combined", combinedPrecollector) } } @@ -1573,9 +2211,16 @@ func main() { fmt.Printf("Proof scheduling: exact per-submission timer (PROOF_WORKERS ignored, value=%d)\n", proofWorkerCount) fmt.Printf("Proof initial delay: %v\n", proofInitialDelay) fmt.Printf("Proof retry delay: %v\n", proofRetryDelay) + fmt.Printf("Server proof-readiness metric: %s_bucket (direct /metrics scrape)\n", proofReadyMetric) + if !verifyProofCrypto { + fmt.Printf("Proof crypto verification: disabled\n") + } } else { fmt.Printf("Proof verification: disabled\n") } + if requestWireFormat != "" { + fmt.Printf("Certification request wire format: %s\n", requestWireFormat) + } fmt.Printf("HTTP client pool size: %d\n", httpClientPoolSize) if enableH2C { fmt.Printf("H2C: enabled (HTTP/2 cleartext for plain HTTP)\n") @@ -1704,6 +2349,13 @@ func main() { var wg sync.WaitGroup var submissionWg sync.WaitGroup // Track outstanding submission requests + metricsScrapeCtx, metricsScrapeCancel := context.WithTimeout(context.Background(), 10*time.Second) + proofReadyBefore, proofReadyBeforeErr := scrapeProofReadinessHistogram(metricsScrapeCtx, shardClients, authHeader) + metricsScrapeCancel() + if proofReadyBeforeErr != nil { + fmt.Printf("Server proof-readiness histogram before test unavailable: %v\n", proofReadyBeforeErr) + } + // Record when submission actually starts metrics.submissionStartTime = time.Now() @@ -1762,6 +2414,10 @@ func main() { } proofCancel() + metricsScrapeCtx, metricsScrapeCancel = context.WithTimeout(context.Background(), 10*time.Second) + proofReadyAfter, proofReadyAfterErr := scrapeProofReadinessHistogram(metricsScrapeCtx, shardClients, authHeader) + metricsScrapeCancel() + // Stop submission phase and get counts fmt.Printf("\n----------------------------------------\n") successful := atomic.LoadInt64(&metrics.successfulRequests) @@ -1871,6 +2527,8 @@ func main() { } } + printServerProofReadinessHistogram(proofReadyBefore, proofReadyAfter, proofReadyBeforeErr, proofReadyAfterErr) + fmt.Printf("========================================\n") printShardFinalReport(metrics, shardClients) diff --git a/cmd/performance-test/types.go b/cmd/performance-test/types.go index 248bdfe..d9359f4 100644 --- a/cmd/performance-test/types.go +++ b/cmd/performance-test/types.go @@ -139,60 +139,95 @@ func (rr *RequestRateCounters) IncProofCompleted() { rr.proofCompleted.Add(1) } func (rr *RequestRateCounters) IncProofRetries() { rr.proofRetries.Add(1) } type aggregatorLogRaw struct { - Time string `json:"time"` - Msg string `json:"msg"` - Block string `json:"block"` - Commitments int `json:"commitments"` - RoundTime string `json:"roundTime"` - Processing string `json:"processing"` - BftWait string `json:"bftWait"` - Finalization string `json:"finalization"` - FinalizeScan string `json:"finalizeScan"` - FinalizeConvert string `json:"finalizeConvert"` - FinalizeStoreBlock string `json:"finalizeStoreBlock"` - FinalizeStoreBlockDoc string `json:"finalizeStoreBlockDoc"` - FinalizeStoreBlockRecords string `json:"finalizeStoreBlockRecords"` - FinalizeStoreData string `json:"finalizeStoreData"` - FinalizeStoreSmt string `json:"finalizeStoreSmt"` - FinalizeStoreRecords string `json:"finalizeStoreRecords"` - FinalizeLockWait string `json:"finalizeLockWait"` - FinalizeSmtCommit string `json:"finalizeSmtCommit"` - FinalizeSetFinalized string `json:"finalizeSetFinalized"` - FinalizeAck string `json:"finalizeAck"` - ProofReadyMedian string `json:"proofReadyMedian"` - ProofReadyP95 string `json:"proofReadyP95"` - ProofReadyP99 string `json:"proofReadyP99"` - RedisTotal int `json:"redisTotal"` - RedisPending int `json:"redisPending"` + Time string `json:"time"` + Msg string `json:"msg"` + Block string `json:"block"` + Commitments int `json:"commitments"` + Leaves int `json:"leaves"` + RoundTime string `json:"roundTime"` + Processing string `json:"processing"` + BftWait string `json:"bftWait"` + Finalization string `json:"finalization"` + FinalizeScan string `json:"finalizeScan"` + FinalizeConvert string `json:"finalizeConvert"` + FinalizeStoreBlock string `json:"finalizeStoreBlock"` + FinalizeStoreBlockDoc string `json:"finalizeStoreBlockDoc"` + FinalizeStoreBlockRecords string `json:"finalizeStoreBlockRecords"` + FinalizeStoreData string `json:"finalizeStoreData"` + FinalizeStoreSmt string `json:"finalizeStoreSmt"` + FinalizeStoreRecords string `json:"finalizeStoreRecords"` + FinalizeLockWait string `json:"finalizeLockWait"` + FinalizeSmtCommit string `json:"finalizeSmtCommit"` + FinalizeSmtCommitCollect string `json:"finalizeSmtCommitCollect"` + FinalizeSmtCommitTombstone string `json:"finalizeSmtCommitTombstone"` + FinalizeSmtCommitBatchBuild string `json:"finalizeSmtCommitBatchBuild"` + FinalizeSmtCommitRootHash string `json:"finalizeSmtCommitRootHash"` + FinalizeSmtCommitEngineWrite string `json:"finalizeSmtCommitEngineWrite"` + FinalizeSmtCommitCacheUpdate string `json:"finalizeSmtCommitCacheUpdate"` + FinalizeSmtCommitNodeWrites int `json:"finalizeSmtCommitNodeWrites"` + FinalizeSmtCommitNodeDeletes int `json:"finalizeSmtCommitNodeDeletes"` + FinalizeSetFinalized string `json:"finalizeSetFinalized"` + FinalizeAck string `json:"finalizeAck"` + ProposalToProofReady string `json:"proposalToProofReady"` + ProofReadyMedian string `json:"proofReadyMedian"` + ProofReadyP95 string `json:"proofReadyP95"` + ProofReadyP99 string `json:"proofReadyP99"` + PendingAtAdvance int `json:"pendingAtAdvance"` + TailMerged int `json:"tailMerged"` + TailRemaining int `json:"tailRemaining"` + AlreadyPrepared bool `json:"alreadyPrepared"` + AdvanceFlush string `json:"advanceFlush"` + FlushCalls int `json:"flushCalls"` + FlushAdded int `json:"flushAdded"` + FlushTotal string `json:"flushTotal"` + FlushMax string `json:"flushMax"` + StageCalls int `json:"stageCalls"` + StageAdded int `json:"stageAdded"` + StageTotal string `json:"stageTotal"` + StageMax string `json:"stageMax"` + Fork string `json:"fork"` + Total string `json:"total"` + RedisTotal int `json:"redisTotal"` + RedisPending int `json:"redisPending"` } type aggregatorRoundSummary struct { - Timestamp time.Time - Block string - Commitments int - RoundTime time.Duration - Processing time.Duration - BftWait time.Duration - Finalization time.Duration - HasFinalizationBreakdown bool - FinalizeScan time.Duration - FinalizeConvert time.Duration - FinalizeStoreBlock time.Duration - FinalizeStoreBlockDoc time.Duration - FinalizeStoreBlockRecords time.Duration - FinalizeStoreData time.Duration - FinalizeStoreSmt time.Duration - FinalizeStoreRecords time.Duration - FinalizeLockWait time.Duration - FinalizeSmtCommit time.Duration - FinalizeSetFinalized time.Duration - FinalizeAck time.Duration - HasProofReady bool - ProofMedian time.Duration - ProofP95 time.Duration - ProofP99 time.Duration - RedisTotal int - RedisPending int + Timestamp time.Time + Block string + Commitments int + RoundTime time.Duration + Processing time.Duration + BftWait time.Duration + Finalization time.Duration + HasFinalizationBreakdown bool + FinalizeScan time.Duration + FinalizeConvert time.Duration + FinalizeStoreBlock time.Duration + FinalizeStoreBlockDoc time.Duration + FinalizeStoreBlockRecords time.Duration + FinalizeStoreData time.Duration + FinalizeStoreSmt time.Duration + FinalizeStoreRecords time.Duration + FinalizeLockWait time.Duration + FinalizeSmtCommit time.Duration + FinalizeSmtCommitCollect time.Duration + FinalizeSmtCommitTombstone time.Duration + FinalizeSmtCommitBatchBuild time.Duration + FinalizeSmtCommitRootHash time.Duration + FinalizeSmtCommitEngineWrite time.Duration + FinalizeSmtCommitCacheUpdate time.Duration + FinalizeSmtCommitNodeWrites int + FinalizeSmtCommitNodeDeletes int + FinalizeSetFinalized time.Duration + FinalizeAck time.Duration + HasProposalToProofReady bool + ProposalToProofReady time.Duration + HasProofReady bool + ProofMedian time.Duration + ProofP95 time.Duration + ProofP99 time.Duration + RedisTotal int + RedisPending int } func (m *Metrics) addProofLatency(latency time.Duration) { @@ -558,10 +593,15 @@ func (c *JSONRPCClient) call(method string, params interface{}) (*JSONRPCRespons func (c *JSONRPCClient) callWithContext(ctx context.Context, method string, params interface{}) (*JSONRPCResponse, error) { id := atomic.AddInt64(&c.requestID, 1) + encodedParams, err := encodeJSONRPCParams(method, params) + if err != nil { + return nil, fmt.Errorf("failed to encode JSON-RPC params: %w", err) + } + request := JSONRPCRequest{ JSONRPC: "2.0", Method: method, - Params: params, + Params: encodedParams, ID: int(id), } diff --git a/docker-compose.yml b/docker-compose.yml index 1b0cb0c..26d7199 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -146,6 +146,7 @@ services: CONCURRENCY_LIMIT: "1000" ENABLE_DOCS: "true" ENABLE_CORS: "true" + MALLOC_ARENA_MAX: "${MALLOC_ARENA_MAX:-2}" # Database Configuration MONGODB_URI: "mongodb://mongodb:27017/aggregator?replicaSet=rs0&directConnection=true" @@ -174,7 +175,7 @@ services: SMT_ROCKSDB_SUBCOMPACTIONS: "${SMT_ROCKSDB_SUBCOMPACTIONS:-4}" SMT_ROCKSDB_BLOOM_BITS: "${SMT_ROCKSDB_BLOOM_BITS:-10}" SMT_ROCKSDB_MEMTABLE_MB: "${SMT_ROCKSDB_MEMTABLE_MB:-64}" - SMT_MATERIALIZE_WORKERS: "${SMT_MATERIALIZE_WORKERS:-64}" + SMT_MATERIALIZE_WORKERS: "${SMT_MATERIALIZE_WORKERS:-16}" SMT_STARTUP_REPLAY_LIMIT_BLOCKS: "${SMT_STARTUP_REPLAY_LIMIT_BLOCKS:-100}" # High Availability Configuration diff --git a/docs/disk-backed-smt-performance.md b/docs/disk-backed-smt-performance.md new file mode 100644 index 0000000..7ec71d9 --- /dev/null +++ b/docs/disk-backed-smt-performance.md @@ -0,0 +1,125 @@ +# Disk-Backed SMT E2E Performance + +This document records end-to-end BFT-shard performance for the Go aggregator +with RocksDB-backed SMT and durable proposals. + +## Scope + +The BFT-sharding stack starts two shards because the local setup expects two +shards. The performance client sends traffic only to shard 0, so the rows below +measure one loaded shard: + +```text +SHARD_TARGETS=http://localhost:3001:0 +``` + +All checkpoint rows are single-loaded-shard measurements at `1000/s`. They are +short checkpoint probes. + +## Config + +The runs used the disk-backed BFT-shard stack with RocksDB as the SMT backend. +All components run co-located on a single host. + +| Setting | Value | +|---|---:| +| Host CPU | AMD Ryzen 9 5900XT, 16 cores / 32 threads | +| Disk | local NVMe SSD | +| Active shard load | shard 0 only | +| Required stack shards | `NUM_SHARDS=2` | +| Mongo groups | `MONGO_GROUPS=2` | +| SMT backend | `rocksdb` | +| Go build tags | `rocksdb` | +| RocksDB cache | `SMT_ROCKSDB_CACHE_MB=1024` | +| RocksDB background jobs | `SMT_ROCKSDB_BG_JOBS=8` | +| RocksDB subcompactions | `SMT_ROCKSDB_SUBCOMPACTIONS=4` | +| RocksDB bloom bits | `SMT_ROCKSDB_BLOOM_BITS=10` | +| RocksDB memtable | `SMT_ROCKSDB_MEMTABLE_MB=64` | +| RocksDB library | static RocksDB 8.10 | +| `PRECOLLECTOR_GRACE_PERIOD` | `0s` | +| `MAX_COMMITMENTS_PER_ROUND` | `20000` | +| `COLLECT_MINI_BATCH_SIZE` | `500` | +| Mongo insert chunk size | `200` | +| Mongo insert chunk workers | `8` | +| Redis ack batch size | `10000` | +| Submission workers | `300` | +| Proof workers | `300` | +| HTTP client pool | `48` | + +Each checkpoint probe used: + +| Setting | Value | +|---|---:| +| Target RPS | `1000` | +| Duration | `5m` | +| `ROOT_BLOCK_RATE` | `350ms` | +| `PROOF_INITIAL_DELAY` | `1s` | +| `PROOF_RETRY_DELAY` | `500ms` | + +## Metrics + +`Pre-BFT materialize` and `Pre-BFT record stage (Mongo)` happen before sending +the BFT certification request. `Finalization total` is post-UC work, and `SMT +commit` is part of that finalization total. + +Client proof latency includes the configured first poll delay +(`PROOF_INITIAL_DELAY=1s`), so client p50 near `1.005s` means most proofs were +ready on the first poll. + +## Results + +### Checkpoint Matrix + +These rows track one loaded shard at fixed `1000/s` as the RocksDB SMT grows. +Each checkpoint is a `5m` probe with full submission and proof verification +success unless noted otherwise. + +| Checkpoint | Tree size | Commitments / round | Server proofReady p50 | Server proofReady p95 | Server proofReady p99 | `<=1s` ready | Client p50 | Client p95 | BFT wait | Pre-BFT materialize | Pre-BFT record stage (Mongo) | Finalization total | SMT commit inside finalization | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| Clean | ~303k | 532 | 841ms | 1.149s | 1.189s | 80.0% | 1.005s | 1.506s | 490ms | 8ms | 12ms | 17ms | 14ms | +| 5M | 5.06M | 532 | 814ms | 1.130s | 1.186s | 85.5% | 1.005s | 1.507s | 479ms | 13ms | 11ms | 24ms | 20ms | +| 10M | 10.05M | 537 | 848ms | 1.163s | 1.280s | 78.2% | 1.005s | 1.507s | 468ms | 17ms | 18ms | 28ms | 23ms | +| 15M | 15.71M | 532 | 840ms | 1.149s | 1.189s | 80.2% | 1.005s | 1.507s | 474ms | 18ms | 9ms | 26ms | 22ms | +| 20M | 21.97M | 563 | 882ms | 1.172s | 1.247s | 71.1% | 1.006s | 1.508s | 470ms | 24ms | 19ms | 43ms | 35ms | +| 25M | 26.77M | 559 | 877ms | 1.179s | 1.485s | 71.6% | 1.006s | 1.509s | 464ms | 27ms | 22ms | 40ms | 33ms | +| 30M | 30.44M | 558 | 873ms | 1.176s | 1.334s | 72.6% | 1.006s | 1.508s | 465ms | 26ms | 20ms | 40ms | 32ms | +| 40M | 40.37M | 568 | 867ms | 1.183s | 1.505s | 73.5% | 1.006s | 1.509s | 460ms | 32ms | 25ms | 44ms | 36ms | +| 50M | 50.33M | 565 | 889ms | 1.187s | 1.566s | 69.4% | 1.006s | 1.509s | 452ms | 36ms | 27ms | 42ms | 34ms | +| 60M | 60.32M | 567 | 884ms | 1.189s | 1.666s | 70.4% | 1.006s | 1.509s | 453ms | 33ms | 30ms | 44ms | 36ms | + +### Single-Shard Scaling Matrix + +These runs increase target load on one active shard using the low-latency BFT cadence. + +| Tree size | Target RPS | Duration | Achieved RPS | Submitted / verified | Server proofReady p50 | Server proofReady p95 | Server proofReady p99 | `<=1s` ready | Client p50 | Client p95 | Client p99 | BFT wait | Finalization total | SMT commit inside finalization | Commitments / round | Result | +|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---| +| ~606k | 2,000 | 5m | 1,998.99 | 599,700 / 599,700 | 870ms | 1.166s | 1.197s | 73.9% | 1.006s | 1.507s | 1.512s | 484ms | 31ms | 22ms | 1,109 | pass | +| ~1.51M | 3,000 | 5m | 2,999.02 | 899,700 / 899,700 | 888ms | 1.177s | 1.379s | 69.8% | 1.005s | 1.511s | 1.519s | 452ms | 41ms | 31ms | 1,685 | pass | +| ~2.73M | 4,000 | 5m | 3,999.04 | 1,199,700 / 1,199,700 | 1.027s | 1.546s | 1.838s | 46.4% | 1.015s | 1.516s | 2.017s | 477ms | 55ms | 42ms | 2,651 | pass | +| ~4.24M | 5,000 | 5m | 4,997.57 | 1,496,389 / 1,496,389 | 1.250s | 2.141s | 2.996s | 22.1% | 1.506s | 2.513s | 3.150s | 488ms | 94ms | 73ms | 4,095 | degraded | + +### Multi-Shard Scaling + +Fresh-tree `5m` scaling checks with `ROOT_BLOCK_RATE=350ms`, +`PRECOLLECTOR_GRACE_PERIOD=0s`, `PROOF_INITIAL_DELAY=2s`, and +`PROOF_RETRY_DELAY=1s`. + +| Active shards | Mongo groups | Target RPS | Achieved RPS | Submitted / verified | Server proofReady p50 | Server proofReady p95 | Server proofReady p99 | `<=1s` ready | Client p50 | Client p95 | Client p99 | Finalization total | SMT commit inside finalization | Commitments / round / shard | Aggregator CPU avg | Result | +|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---| +| 4 | 2 | 4,000 | 3,999.15 | 1,199,700 / 1,199,700 | 844ms | 1.159s | 1.196s | 77.9% | 2.002s | 2.005s | 2.008s | 24ms | 15ms | 548 | 369% total | pass | +| 8 | 4 | 8,000 | 7,998.26 | 2,399,691 / 2,399,691 | 881ms | 1.257s | 1.816s | 70.1% | 2.003s | 2.014s | 2.038s | 41ms | 25ms | 574 | 782% total | pass | + +The 8-shard row doubles the load and keeps per-shard round size close to the +4-shard run. Throughput scales nearly linearly on the same host; the latency +tail grows, but all submissions and proofs still complete successfully. + +## Notes + +Separate isolated RocksDB-SMT probes, without BFT/Mongo/Redis/HTTP, showed that +the tree engine can still process large batches at high tree sizes. Around +255M-257M leaves (roughly a 50GB+ RocksDB directory), the Go RocksDB-SMT path +with static RocksDB 8.10 processed 10k-leaf batches in roughly 854-888ms, with +RocksDB engine write time around 87-90ms. + +This separates raw SMT capacity from end-to-end latency: the E2E rows measure +the full round pipeline, not only RocksDB-SMT insert/commit speed. diff --git a/ha-compose.yml b/ha-compose.yml index 0fd177e..b515832 100644 --- a/ha-compose.yml +++ b/ha-compose.yml @@ -225,6 +225,7 @@ services: CONCURRENCY_LIMIT: "1000" ENABLE_DOCS: "true" ENABLE_CORS: "true" + MALLOC_ARENA_MAX: "${MALLOC_ARENA_MAX:-2}" # Database Configuration MONGODB_URI: "mongodb://mongo1:27017,mongo2:27017,mongo3:27017/aggregator?replicaSet=rs0" diff --git a/internal/bft/client.go b/internal/bft/client.go index 1bfde58..3c0cb04 100644 --- a/internal/bft/client.go +++ b/internal/bft/client.go @@ -34,7 +34,9 @@ const ( var ( ErrStaleCertificationRound = errors.New("stale certification round") - ErrCertifiedStateMismatch = errors.New("certified state mismatch") + ErrCertifiedRootMismatch = errors.New("certified root does not match proposed block root") + ErrInvalidUCSequence = errors.New("invalid unicity certificate sequence") + ErrCertifiedStateMismatch = errors.New("local committed state does not match latest certified root") ) // BFTClientImpl handles communication with the BFT root chain via P2P network @@ -54,11 +56,14 @@ type ( signer cryptobft.Signer // Latest UC this node has seen. Can be ahead of the committed UC during recovery. - luc atomic.Pointer[types.UnicityCertificate] - roundManager RoundManager - proposedBlock *models.Block + luc atomic.Pointer[types.UnicityCertificate] + roundManager RoundManager + proposedBlock *models.Block + resumedDurableProposal bool // Track the next round number expected by root chain nextExpectedRound atomic.Uint64 + // Track the next epoch expected by root chain technical record + nextExpectedEpoch atomic.Uint64 // Track the root round number to detect repeat UCs lastRootRound atomic.Uint64 // Mutex to ensure sequential UC processing @@ -86,6 +91,19 @@ type ( FinalizeBlockWithRetry(ctx context.Context, block *models.Block) error StartNewRound(ctx context.Context, roundNumber *api.BigInt) error StartNextRoundFromPrecollector(ctx context.Context, roundNumber *api.BigInt) error + CommittedRoot(ctx context.Context) ([]byte, *api.BigInt, error) + } + + DurableProposalFinalizer interface { + FinalizeCertifiedProposal(ctx context.Context, blockNumber *api.BigInt, rootHash api.HexBytes, unicityCertificate api.HexBytes) (bool, error) + } + + DurableProposalLoader interface { + LoadDurableProposal(ctx context.Context, blockNumber *api.BigInt) (*models.Block, bool, error) + } + + DurableProposalAbandoner interface { + AbandonDurableProposal(ctx context.Context, blockNumber *api.BigInt, rootHash api.HexBytes) error } TrustBaseStore interface { @@ -329,6 +347,11 @@ func (c *BFTClientImpl) handleUnicityCertificate(ctx context.Context, uc *types. "rootRound", uc.GetRootRoundNumber()) prevLUC := c.luc.Load() + if prevLUC != nil { + if err := types.CheckNonEquivocatingCertificates(prevLUC, uc); err != nil { + return fmt.Errorf("%w: %w", ErrInvalidUCSequence, err) + } + } // as we can be connected to several root nodes, we can receive the same UC multiple times if uc.IsDuplicate(prevLUC) { c.logger.WithContext(ctx).Debug(fmt.Sprintf("duplicate UC (same root round %d)", uc.GetRootRoundNumber())) @@ -345,21 +368,20 @@ func (c *BFTClientImpl) handleUnicityCertificate(ctx context.Context, uc *types. "prevRootRound", prevLUC.GetRootRoundNumber(), "newRootRound", uc.GetRootRoundNumber()) - // Store the repeat UC and update root round tracking + if err := c.abandonProposedBlockLocked(ctx, "repeat UC"); err != nil { + return err + } + + // Store the repeat UC and update root round tracking after durable + // cleanup succeeds, so a failed cleanup can be retried by the next UC. c.luc.Store(uc) c.lastRootRound.Store(uc.GetRootRoundNumber()) - // Clear any proposed block as it wasn't accepted in time - if c.proposedBlock != nil { - c.logger.WithContext(ctx).Info("Clearing proposed block due to repeat UC", - "proposedBlockNumber", c.proposedBlock.Index.String()) - c.proposedBlock = nil - } - // Start new round immediately with the next expected round nextRoundNumber := big.NewInt(0) nextRoundNumber.SetUint64(tr.Round) c.nextExpectedRound.Store(tr.Round) + c.nextExpectedEpoch.Store(tr.Epoch) c.logger.WithContext(ctx).Info("Starting new round after repeat UC", "nextRoundNumber", nextRoundNumber.String()) @@ -386,6 +408,7 @@ func (c *BFTClientImpl) handleUnicityCertificate(ctx context.Context, uc *types. // Store the next expected round number c.nextExpectedRound.Store(tr.Round) + c.nextExpectedEpoch.Store(tr.Epoch) wasInitializing := c.status.Load() == initializing if wasInitializing { @@ -393,7 +416,30 @@ func (c *BFTClientImpl) handleUnicityCertificate(ctx context.Context, uc *types. "nextRoundNumber", nextRoundNumber.String()) // First UC received after an initial handshake with a root node -> initialization finished. c.status.Store(normal) - err := c.roundManager.StartNewRound(ctx, api.NewBigInt(nextRoundNumber)) + recovered, err := c.finalizeCertifiedDurableProposalLocked(ctx, uc) + if err != nil { + return err + } + if recovered { + c.logger.WithContext(ctx).Info("Durable proposal finalized from initialization UC", + "ucRound", uc.GetRoundNumber(), + "nextRoundNumber", nextRoundNumber.String()) + err := c.roundManager.StartNewRound(ctx, api.NewBigInt(nextRoundNumber)) + if err != nil { + c.logger.WithContext(ctx).Error("Failed to start first round after durable proposal recovery", + "nextRoundNumber", nextRoundNumber.String(), + "error", err.Error()) + } + return err + } + resumed, err := c.resumeDurableProposalLocked(ctx, api.NewBigInt(nextRoundNumber)) + if err != nil { + return err + } + if resumed { + return nil + } + err = c.roundManager.StartNewRound(ctx, api.NewBigInt(nextRoundNumber)) if err != nil { c.logger.WithContext(ctx).Error("Failed to start first round after initialization", "nextRoundNumber", nextRoundNumber.String(), @@ -422,10 +468,9 @@ func (c *BFTClientImpl) handleUnicityCertificate(ctx context.Context, uc *types. "ucRound", expectedRound, "ourProposedRound", proposedRound, "nextRoundRequired", tr.Round) - // This UC is for an older round, but we still need to process it - // to stay in sync with root chain. Clear our proposed block and - // start fresh with the root chain's expected round - c.proposedBlock = nil + if err := c.abandonProposedBlockLocked(ctx, "round mismatch"); err != nil { + return err + } // Start new round immediately with root chain's next round c.logger.WithContext(ctx).Info("Starting new round to sync with root chain", @@ -439,14 +484,59 @@ func (c *BFTClientImpl) handleUnicityCertificate(ctx context.Context, uc *types. return err } - // If UC is for an older round than proposed, it's a stale UC - c.logger.WithContext(ctx).Debug("Received UC for older round than proposed block, ignoring") - return nil + c.logger.WithContext(ctx).Warn("UC is newer than proposed block, abandoning stale proposal", + "ucRound", expectedRound, + "proposedBlockRound", proposedRound, + "nextRound", tr.Round) + if err := c.abandonProposedBlockLocked(ctx, "newer UC"); err != nil { + return err + } + blockNum = "nil" } } // Check if we have a proposed block to finalize if c.proposedBlock == nil { + if uc != nil && uc.InputRecord != nil { + ucCbor, err := types.Cbor.Marshal(uc) + if err != nil { + c.logger.WithContext(ctx).Error("Failed to encode unicity certificate", + "error", err.Error()) + return fmt.Errorf("failed to encode unicity certificate: %w", err) + } + if finalizer, ok := c.roundManager.(DurableProposalFinalizer); ok { + recovered, err := finalizer.FinalizeCertifiedProposal(ctx, + api.NewBigInt(new(big.Int).SetUint64(expectedRound)), + api.HexBytes(append([]byte(nil), uc.InputRecord.Hash...)), + api.NewHexBytes(ucCbor), + ) + if err != nil { + c.logger.WithContext(ctx).Error("Failed to finalize durable proposal", + "ucRound", expectedRound, + "error", err.Error()) + metrics.BFTErrorsTotal.Inc() + if c.eventBus != nil { + c.eventBus.Publish(events.TopicFatalError, events.FatalErrorEvent{ + Source: "bft", + Error: err.Error(), + }) + } + return fmt.Errorf("failed to finalize durable proposal: %w", err) + } + if recovered { + c.logger.WithContext(ctx).Info("Durable proposal finalized from UC", + "ucRound", expectedRound, + "nextRoundNumber", nextRoundNumber.String()) + err := c.roundManager.StartNewRound(ctx, api.NewBigInt(nextRoundNumber)) + if err != nil { + c.logger.WithContext(ctx).Error("Failed to start next round after durable proposal recovery", + "nextRoundNumber", nextRoundNumber.String(), + "error", err.Error()) + } + return err + } + } + } c.logger.WithContext(ctx).Warn("No proposed block to finalize, starting next round", "ucRound", expectedRound, "nextRoundNumber", nextRoundNumber.String()) @@ -465,27 +555,24 @@ func (c *BFTClientImpl) handleUnicityCertificate(ctx context.Context, uc *types. return err } - c.logger.WithContext(ctx).Info("Finalizing block with unicity certificate", - "blockNumber", blockNum, - "ucRound", expectedRound) - - if c.proposedBlock == nil || uc.InputRecord == nil || !bytes.Equal(c.proposedBlock.RootHash, uc.InputRecord.Hash) { - proposedRoot := "" - if c.proposedBlock != nil { - proposedRoot = c.proposedBlock.RootHash.String() - } - certifiedRoot := "" - if uc.InputRecord != nil { - certifiedRoot = api.HexBytes(uc.InputRecord.Hash).String() - } - err := fmt.Errorf("%w: block %s proposed root %s, UC root %s", - ErrCertifiedStateMismatch, blockNum, proposedRoot, certifiedRoot) + if uc == nil || uc.InputRecord == nil { + return fmt.Errorf("cannot finalize block %s: UC input record is missing", c.proposedBlock.Index.String()) + } + if !bytes.Equal(uc.InputRecord.Hash, c.proposedBlock.RootHash) { + err := fmt.Errorf("%w: block %s root %s, UC root %s", + ErrCertifiedRootMismatch, + c.proposedBlock.Index.String(), + c.proposedBlock.RootHash.String(), + api.HexBytes(uc.InputRecord.Hash).String()) c.logger.WithContext(ctx).Error("UC root does not match proposed block root", - "blockNumber", blockNum, - "proposedRoot", proposedRoot, - "certifiedRoot", certifiedRoot) + "blockNumber", c.proposedBlock.Index.String(), + "blockRoot", c.proposedBlock.RootHash.String(), + "ucRoot", api.HexBytes(uc.InputRecord.Hash).String()) + if abandonErr := c.abandonProposedBlockLocked(ctx, "certified root mismatch"); abandonErr != nil { + c.logger.WithContext(ctx).Error("Failed to abandon proposed block after certified root mismatch", + "error", abandonErr.Error()) + } metrics.BFTErrorsTotal.Inc() - c.proposedBlock = nil if c.eventBus != nil { c.eventBus.Publish(events.TopicFatalError, events.FatalErrorEvent{ Source: "bft", @@ -495,6 +582,10 @@ func (c *BFTClientImpl) handleUnicityCertificate(ctx context.Context, uc *types. return err } + c.logger.WithContext(ctx).Info("Finalizing block with unicity certificate", + "blockNumber", blockNum, + "ucRound", expectedRound) + if certStart := c.certRequestTime.Swap(0); certStart > 0 { metrics.BFTCertificationDuration.Observe(float64(time.Now().UnixNano()-certStart) / 1e9) } @@ -507,6 +598,46 @@ func (c *BFTClientImpl) handleUnicityCertificate(ctx context.Context, uc *types. } c.proposedBlock.UnicityCertificate = api.NewHexBytes(ucCbor) + if c.resumedDurableProposal { + finalizer, ok := c.roundManager.(DurableProposalFinalizer) + if !ok { + return errors.New("round manager does not support durable proposal finalization") + } + recovered, err := finalizer.FinalizeCertifiedProposal(ctx, + api.NewBigInt(new(big.Int).Set(c.proposedBlock.Index.Int)), + api.HexBytes(append([]byte(nil), uc.InputRecord.Hash...)), + api.NewHexBytes(ucCbor), + ) + if err != nil { + c.logger.WithContext(ctx).Error("Failed to finalize resumed durable proposal", + "blockNumber", c.proposedBlock.Index.String(), + "error", err.Error()) + metrics.BFTErrorsTotal.Inc() + if c.eventBus != nil { + c.eventBus.Publish(events.TopicFatalError, events.FatalErrorEvent{ + Source: "bft", + Error: err.Error(), + }) + } + return fmt.Errorf("failed to finalize resumed durable proposal: %w", err) + } + if !recovered { + return fmt.Errorf("resumed durable proposal %s was not finalized", c.proposedBlock.Index.String()) + } + c.proposedBlock = nil + c.resumedDurableProposal = false + + c.logger.WithContext(ctx).Info("Resumed durable proposal finalized, starting new round", + "nextRoundNumber", nextRoundNumber.String()) + err = c.roundManager.StartNewRound(ctx, api.NewBigInt(nextRoundNumber)) + if err != nil { + c.logger.WithContext(ctx).Error("Failed to start new round", + "nextRoundNumber", nextRoundNumber.String(), + "error", err.Error()) + } + return err + } + if err := c.roundManager.FinalizeBlockWithRetry(ctx, c.proposedBlock); err != nil { c.logger.WithContext(ctx).Error("Failed to finalize block after retries", "blockNumber", c.proposedBlock.Index.String(), @@ -523,6 +654,7 @@ func (c *BFTClientImpl) handleUnicityCertificate(ctx context.Context, uc *types. // Clear the proposed block after finalization c.proposedBlock = nil + c.resumedDurableProposal = false c.logger.WithContext(ctx).Info("Block finalized, starting new round", "nextRoundNumber", nextRoundNumber.String()) @@ -536,6 +668,95 @@ func (c *BFTClientImpl) handleUnicityCertificate(ctx context.Context, uc *types. return err } +func (c *BFTClientImpl) finalizeCertifiedDurableProposalLocked(ctx context.Context, uc *types.UnicityCertificate) (bool, error) { + if uc == nil || uc.InputRecord == nil { + return false, nil + } + if c.roundManager == nil { + return false, errors.New("round manager is not initialized") + } + localRoot, localBlockNumber, err := c.roundManager.CommittedRoot(ctx) + if err != nil { + return false, fmt.Errorf("failed to read local committed SMT state: %w", err) + } + if uc.GetRoundNumber() == 0 && len(uc.InputRecord.Hash) == 0 && localBlockNumber == nil { + return false, nil + } + if bytes.Equal(localRoot, uc.InputRecord.Hash) { + return false, nil + } + finalizer, ok := c.roundManager.(DurableProposalFinalizer) + if !ok { + return false, nil + } + ucCbor, err := types.Cbor.Marshal(uc) + if err != nil { + c.logger.WithContext(ctx).Error("Failed to encode unicity certificate", + "error", err.Error()) + return false, fmt.Errorf("failed to encode unicity certificate: %w", err) + } + recovered, err := finalizer.FinalizeCertifiedProposal(ctx, + api.NewBigInt(new(big.Int).SetUint64(uc.GetRoundNumber())), + api.HexBytes(append([]byte(nil), uc.InputRecord.Hash...)), + api.NewHexBytes(ucCbor), + ) + if err != nil { + return false, err + } + return recovered, nil +} + +func (c *BFTClientImpl) abandonProposedBlockLocked(ctx context.Context, reason string) error { + if c.proposedBlock == nil { + return nil + } + block := c.proposedBlock + c.logger.WithContext(ctx).Info("Abandoning proposed block", + "blockNumber", block.Index.String(), + "rootHash", block.RootHash.String(), + "reason", reason) + if abandoner, ok := c.roundManager.(DurableProposalAbandoner); ok { + if err := abandoner.AbandonDurableProposal(ctx, block.Index, block.RootHash); err != nil { + return fmt.Errorf("failed to abandon durable proposal %s: %w", block.Index.String(), err) + } + } + c.proposedBlock = nil + c.resumedDurableProposal = false + return nil +} + +func (c *BFTClientImpl) resumeDurableProposalLocked(ctx context.Context, roundNumber *api.BigInt) (bool, error) { + loader, ok := c.roundManager.(DurableProposalLoader) + if !ok { + return false, nil + } + block, found, err := loader.LoadDurableProposal(ctx, roundNumber) + if err != nil { + return false, fmt.Errorf("failed to load durable proposal for round %s: %w", roundNumber.String(), err) + } + if !found { + return false, nil + } + if block == nil { + return false, fmt.Errorf("durable proposal for round %s is nil", roundNumber.String()) + } + if block.Index.Cmp(roundNumber.Int) != 0 { + return false, fmt.Errorf("durable proposal round mismatch: expected %s, got %s", roundNumber.String(), block.Index.String()) + } + + c.logger.WithContext(ctx).Info("Resending durable proposal after BFT initialization", + "blockNumber", block.Index.String(), + "rootHash", block.RootHash.String()) + c.proposedBlock = block + c.resumedDurableProposal = true + c.certRequestTime.Store(time.Now().UnixNano()) + if err := c.sendCertificationRequest(ctx, block.RootHash.String(), block.Index.Uint64()); err != nil { + metrics.BFTErrorsTotal.Inc() + return true, fmt.Errorf("failed to resend durable proposal %s: %w", block.Index.String(), err) + } + return true, nil +} + func (c *BFTClientImpl) sendCertificationRequest(ctx context.Context, rootHash string, roundNumber uint64) error { rootHashBytes, err := hex.DecodeString(rootHash) if err != nil { @@ -547,9 +768,15 @@ func (c *BFTClientImpl) sendCertificationRequest(ctx context.Context, rootHash s return fmt.Errorf("failed to prepare certification request: %w", err) } - var blockHash []byte - if !bytes.Equal(rootHashBytes, luc.InputRecord.Hash) { - blockHash = rootHashBytes + inputRecord, err := c.buildCertificationInputRecord(luc, rootHashBytes, roundNumber) + if err != nil { + return err + } + if err := c.verifyLocalRootExtendsLatestUC(ctx, luc); err != nil { + return err + } + if c.network == nil || c.peer == nil || c.signer == nil { + return errors.New("BFT client network is not initialized") } // send new input record for certification @@ -557,18 +784,7 @@ func (c *BFTClientImpl) sendCertificationRequest(ctx context.Context, rootHash s PartitionID: c.PartitionID(), ShardID: c.ShardID(), NodeID: c.peer.ID().String(), - InputRecord: &types.InputRecord{ - Version: 1, - RoundNumber: roundNumber, - Epoch: luc.InputRecord.Epoch, - PreviousHash: luc.InputRecord.Hash, - Hash: rootHashBytes, - SummaryValue: []byte{}, // cant be nil if RoundNumber > 0 - Timestamp: luc.UnicitySeal.Timestamp, - BlockHash: blockHash, - SumOfEarnedFees: 0, - ETHash: nil, // can be nil, not validated - }, + InputRecord: inputRecord, } if err = req.Sign(c.signer); err != nil { @@ -588,6 +804,61 @@ func (c *BFTClientImpl) sendCertificationRequest(ctx context.Context, rootHash s return c.network.Send(ctx, req, rootIDs...) } +func (c *BFTClientImpl) buildCertificationInputRecord(luc *types.UnicityCertificate, rootHashBytes []byte, roundNumber uint64) (*types.InputRecord, error) { + if luc == nil || luc.InputRecord == nil || luc.UnicitySeal == nil { + return nil, errors.New("latest UC is incomplete") + } + + var blockHash []byte + if !bytes.Equal(rootHashBytes, luc.InputRecord.Hash) { + blockHash = rootHashBytes + } + + epoch := luc.InputRecord.Epoch + if expectedEpoch := c.nextExpectedEpoch.Load(); expectedEpoch > 0 { + epoch = expectedEpoch + } + + return &types.InputRecord{ + Version: 1, + RoundNumber: roundNumber, + Epoch: epoch, + PreviousHash: luc.InputRecord.Hash, + Hash: rootHashBytes, + SummaryValue: []byte{}, // cant be nil if RoundNumber > 0 + Timestamp: luc.UnicitySeal.Timestamp, + BlockHash: blockHash, + SumOfEarnedFees: 0, + ETHash: nil, // can be nil, not validated + }, nil +} + +func (c *BFTClientImpl) verifyLocalRootExtendsLatestUC(ctx context.Context, luc *types.UnicityCertificate) error { + if luc == nil || luc.InputRecord == nil { + return errors.New("latest UC input record is missing") + } + if c.roundManager == nil { + return errors.New("round manager is not initialized") + } + localRoot, localBlockNumber, err := c.roundManager.CommittedRoot(ctx) + if err != nil { + return fmt.Errorf("failed to read local committed SMT state: %w", err) + } + // BFT represents the genesis/no-state root as empty bytes; the local SMT + // represents its empty tree as a real hash. After the first committed block, + // both sides use the aggregator-computed root and must match exactly. + if luc.GetRoundNumber() == 0 && len(luc.InputRecord.Hash) == 0 && localBlockNumber == nil { + return nil + } + if !bytes.Equal(localRoot, luc.InputRecord.Hash) { + return fmt.Errorf("%w: local root %s, latest UC root %s", + ErrCertifiedStateMismatch, + api.HexBytes(localRoot).String(), + api.HexBytes(luc.InputRecord.Hash).String()) + } + return nil +} + func (c *BFTClientImpl) CertificationRequest(ctx context.Context, block *models.Block) error { c.logger.WithContext(ctx).Info("CertificationRequest called", "blockNumber", block.Index.String(), @@ -635,6 +906,7 @@ func (c *BFTClientImpl) CertificationRequest(ctx context.Context, block *models. } c.proposedBlock = block + c.resumedDurableProposal = false c.certRequestTime.Store(time.Now().UnixNano()) return nil }(); err != nil { diff --git a/internal/bft/client_stub.go b/internal/bft/client_stub.go index 2082e40..221be39 100644 --- a/internal/bft/client_stub.go +++ b/internal/bft/client_stub.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "math/big" + "sync" "time" "github.com/unicitynetwork/bft-go-base/types" @@ -19,6 +20,11 @@ type BFTClientStub struct { roundManager RoundManager nextRoundNumber *api.BigInt delay time.Duration + mu sync.Mutex + ctx context.Context + cancel context.CancelFunc + stopped bool + wg sync.WaitGroup } func NewBFTClientStub(logger *logger.Logger, roundManager RoundManager, nextRoundNumber *api.BigInt, delay time.Duration) *BFTClientStub { @@ -33,11 +39,27 @@ func NewBFTClientStub(logger *logger.Logger, roundManager RoundManager, nextRoun func (n *BFTClientStub) Start(ctx context.Context) error { n.logger.Info("Starting BFT Client Stub") - return n.roundManager.StartNewRound(ctx, n.nextRoundNumber) + stubCtx, cancel := context.WithCancel(ctx) + n.mu.Lock() + n.ctx = stubCtx + n.cancel = cancel + n.stopped = false + n.mu.Unlock() + return n.roundManager.StartNewRound(stubCtx, n.nextRoundNumber) } func (n *BFTClientStub) Stop() { n.logger.Info("Stopping BFT Client Stub") + n.mu.Lock() + n.stopped = true + cancel := n.cancel + n.cancel = nil + n.ctx = nil + n.mu.Unlock() + if cancel != nil { + cancel() + } + n.wg.Wait() } func (n *BFTClientStub) WaitForInitialized(ctx context.Context) error { @@ -82,8 +104,21 @@ func (n *BFTClientStub) CertificationRequest(ctx context.Context, block *models. nextRoundNumber := api.NewBigInt(nil) nextRoundNumber.Add(block.Index.Int, big.NewInt(1)) + n.mu.Lock() + if n.stopped { + n.mu.Unlock() + return nil + } + nextCtx := n.ctx + if nextCtx == nil { + nextCtx = ctx + } + n.wg.Add(1) + n.mu.Unlock() + go func() { - if err := n.roundManager.StartNextRoundFromPrecollector(ctx, nextRoundNumber); err != nil { + defer n.wg.Done() + if err := n.roundManager.StartNextRoundFromPrecollector(nextCtx, nextRoundNumber); err != nil { n.logger.Error("Failed to start next round", "error", err.Error()) } }() diff --git a/internal/bft/client_stub_test.go b/internal/bft/client_stub_test.go index 50d9942..09ac8c4 100644 --- a/internal/bft/client_stub_test.go +++ b/internal/bft/client_stub_test.go @@ -3,6 +3,7 @@ package bft import ( "bytes" "context" + "errors" "math/big" "testing" "time" @@ -10,7 +11,6 @@ import ( "github.com/stretchr/testify/require" "github.com/unicitynetwork/bft-core/network/protocol/certification" "github.com/unicitynetwork/bft-go-base/types" - "github.com/unicitynetwork/bft-go-base/types/hex" "github.com/unicitynetwork/aggregator-go/internal/events" "github.com/unicitynetwork/aggregator-go/internal/logger" @@ -19,11 +19,28 @@ import ( ) type stubRoundManager struct { - finalizedBlocks []*models.Block - startedRounds []*api.BigInt + finalizedBlocks []*models.Block + finalizeBlockCallCnt int + startedRounds []*api.BigInt + committedRoot []byte + committedBlock *api.BigInt + durableRecovered bool + durableRecoveryBlock *api.BigInt + durableRecoveryRoot api.HexBytes + durableRecoveryCert api.HexBytes + durableRecoveryCallCnt int + durableLoadedBlock *models.Block + durableLoadFound bool + durableLoadBlock *api.BigInt + durableLoadCallCnt int + durableAbandonBlock *api.BigInt + durableAbandonRoot api.HexBytes + durableAbandonCallCnt int + durableAbandonErr error } func (m *stubRoundManager) FinalizeBlock(ctx context.Context, block *models.Block) error { + m.finalizeBlockCallCnt++ m.finalizedBlocks = append(m.finalizedBlocks, block) return nil } @@ -41,6 +58,31 @@ func (m *stubRoundManager) StartNextRoundFromPrecollector(ctx context.Context, r return m.StartNewRound(ctx, roundNumber) } +func (m *stubRoundManager) CommittedRoot(context.Context) ([]byte, *api.BigInt, error) { + return m.committedRoot, m.committedBlock, nil +} + +func (m *stubRoundManager) FinalizeCertifiedProposal(_ context.Context, blockNumber *api.BigInt, rootHash api.HexBytes, unicityCertificate api.HexBytes) (bool, error) { + m.durableRecoveryCallCnt++ + m.durableRecoveryBlock = api.NewBigInt(new(big.Int).Set(blockNumber.Int)) + m.durableRecoveryRoot = append(api.HexBytes(nil), rootHash...) + m.durableRecoveryCert = append(api.HexBytes(nil), unicityCertificate...) + return m.durableRecovered, nil +} + +func (m *stubRoundManager) LoadDurableProposal(_ context.Context, blockNumber *api.BigInt) (*models.Block, bool, error) { + m.durableLoadCallCnt++ + m.durableLoadBlock = api.NewBigInt(new(big.Int).Set(blockNumber.Int)) + return m.durableLoadedBlock, m.durableLoadFound, nil +} + +func (m *stubRoundManager) AbandonDurableProposal(_ context.Context, blockNumber *api.BigInt, rootHash api.HexBytes) error { + m.durableAbandonCallCnt++ + m.durableAbandonBlock = api.NewBigInt(new(big.Int).Set(blockNumber.Int)) + m.durableAbandonRoot = append(api.HexBytes(nil), rootHash...) + return m.durableAbandonErr +} + func TestBFTClientCertificationRequestDoesNotRewriteBlockNumber(t *testing.T) { log, err := logger.New("warn", "json", "", false) require.NoError(t, err) @@ -68,57 +110,532 @@ func TestBFTClientCertificationRequestDoesNotRewriteBlockNumber(t *testing.T) { require.EqualValues(t, 52, block.Index.Uint64()) } -func TestBFTClientRejectsUCRootMismatch(t *testing.T) { +func TestBFTClientCertificationRequestRejectsLocalRootMismatch(t *testing.T) { log, err := logger.New("warn", "json", "", false) require.NoError(t, err) - rm := &stubRoundManager{} - bus := events.NewEventBus(log) - fatalEvents := bus.Subscribe(events.TopicFatalError) + localRoot := bytes.Repeat([]byte{0x10}, api.SiblingSize) + certifiedRoot := bytes.Repeat([]byte{0x20}, api.SiblingSize) + blockRoot := api.NewHexBytes(bytes.Repeat([]byte{0x30}, api.SiblingSize)) + rm := &stubRoundManager{committedRoot: localRoot} client := &BFTClientImpl{ logger: log, roundManager: rm, - eventBus: bus, } client.status.Store(normal) + client.nextExpectedRound.Store(7) + client.luc.Store(testUnicityCertificate(6, 12, certifiedRoot, nil)) - proposedRoot := api.NewHexBytes(bytes.Repeat([]byte{0x11}, api.SiblingSize)) - certifiedRoot := api.NewHexBytes(bytes.Repeat([]byte{0x22}, api.SiblingSize)) block := models.NewBlock( - api.NewBigIntFromUint64(12), + api.NewBigIntFromUint64(7), "unicity", 0, "1.0", "mainnet", - proposedRoot, + blockRoot, nil, nil, ) - client.proposedBlock = block - uc := &types.UnicityCertificate{ - InputRecord: &types.InputRecord{ - RoundNumber: 12, - Hash: hex.Bytes(certifiedRoot), + err = client.CertificationRequest(t.Context(), block) + + require.ErrorIs(t, err, ErrCertifiedStateMismatch) + require.Empty(t, rm.finalizedBlocks) +} + +func TestBFTClientVerifyLocalRootExtendsLatestUCAllowsMatchingRoot(t *testing.T) { + root := bytes.Repeat([]byte{0x21}, api.SiblingSize) + client := &BFTClientImpl{ + roundManager: &stubRoundManager{ + committedRoot: root, + committedBlock: api.NewBigIntFromUint64(6), }, - UnicitySeal: &types.UnicitySeal{ - RootChainRoundNumber: 12, + } + luc := testUnicityCertificate(6, 12, root, nil) + + require.NoError(t, client.verifyLocalRootExtendsLatestUC(t.Context(), luc)) +} + +func TestBFTClientVerifyLocalRootExtendsLatestUCAllowsGenesis(t *testing.T) { + client := &BFTClientImpl{ + roundManager: &stubRoundManager{ + committedRoot: bytes.Repeat([]byte{0x47}, api.SiblingSize), }, } - err = client.handleUnicityCertificate(context.Background(), uc, &certification.TechnicalRecord{Round: 13}) + luc := testUnicityCertificate(0, 12, nil, nil) + + require.NoError(t, client.verifyLocalRootExtendsLatestUC(t.Context(), luc)) +} + +func TestBFTClientCrashAfterProposalBeforeFinalizeWedgesWithoutDurableProposal(t *testing.T) { + log, err := logger.New("warn", "json", "", false) + require.NoError(t, err) + + committedRoot := bytes.Repeat([]byte{0x10}, api.SiblingSize) + certifiedRoot := bytes.Repeat([]byte{0x20}, api.SiblingSize) + nextRoot := bytes.Repeat([]byte{0x30}, api.SiblingSize) + rm := &stubRoundManager{ + committedRoot: committedRoot, + committedBlock: api.NewBigIntFromUint64(11), + } + client := &BFTClientImpl{ + logger: log, + roundManager: rm, + } + client.status.Store(normal) + client.luc.Store(testUnicityCertificate(11, 20, committedRoot, nil)) + client.lastRootRound.Store(20) + client.nextExpectedRound.Store(12) + + // Post-crash state: BFT certified round 12, but this node has no in-memory + // proposal and no durable proposal record to replay. + err = client.handleUnicityCertificate( + t.Context(), + testUnicityCertificate(12, 21, certifiedRoot, committedRoot), + &certification.TechnicalRecord{Round: 13, Epoch: 1}, + ) + require.NoError(t, err) + require.Len(t, rm.startedRounds, 1) + require.EqualValues(t, 13, rm.startedRounds[0].Uint64()) + require.Empty(t, rm.finalizedBlocks) + + block := models.NewBlock( + api.NewBigIntFromUint64(13), + "unicity", + 0, + "1.0", + "mainnet", + api.NewHexBytes(nextRoot), + nil, + nil, + ) + err = client.CertificationRequest(t.Context(), block) require.ErrorIs(t, err, ErrCertifiedStateMismatch) require.Empty(t, rm.finalizedBlocks) +} + +func TestBFTClientNoProposedBlockFinalizesDurableProposal(t *testing.T) { + log, err := logger.New("warn", "json", "", false) + require.NoError(t, err) + + committedRoot := bytes.Repeat([]byte{0x10}, api.SiblingSize) + certifiedRoot := bytes.Repeat([]byte{0x20}, api.SiblingSize) + rm := &stubRoundManager{ + committedRoot: committedRoot, + committedBlock: api.NewBigIntFromUint64(11), + durableRecovered: true, + } + client := &BFTClientImpl{ + logger: log, + roundManager: rm, + } + client.status.Store(normal) + client.luc.Store(testUnicityCertificate(11, 20, committedRoot, nil)) + client.lastRootRound.Store(20) + client.nextExpectedRound.Store(12) + + err = client.handleUnicityCertificate( + t.Context(), + testUnicityCertificate(12, 21, certifiedRoot, committedRoot), + &certification.TechnicalRecord{Round: 13, Epoch: 1}, + ) + + require.NoError(t, err) + require.Equal(t, 1, rm.durableRecoveryCallCnt) + require.EqualValues(t, 12, rm.durableRecoveryBlock.Uint64()) + require.Equal(t, api.HexBytes(certifiedRoot), rm.durableRecoveryRoot) + require.NotEmpty(t, rm.durableRecoveryCert) + require.Len(t, rm.startedRounds, 1) + require.EqualValues(t, 13, rm.startedRounds[0].Uint64()) + require.Empty(t, rm.finalizedBlocks) +} + +func TestBFTClientResumedDurableProposalFinalizesWithoutActiveRound(t *testing.T) { + log, err := logger.New("warn", "json", "", false) + require.NoError(t, err) + + committedRoot := bytes.Repeat([]byte{0x10}, api.SiblingSize) + proposalRoot := api.NewHexBytes(bytes.Repeat([]byte{0x20}, api.SiblingSize)) + proposal := models.NewBlock( + api.NewBigIntFromUint64(12), + "unicity", + 0, + "1.0", + "mainnet", + proposalRoot, + api.NewHexBytes(committedRoot), + nil, + ) + proposal.Status = models.FinalityStatusProposed + rm := &stubRoundManager{ + committedRoot: committedRoot, + committedBlock: api.NewBigIntFromUint64(11), + durableRecovered: true, + } + client := &BFTClientImpl{ + logger: log, + roundManager: rm, + proposedBlock: proposal, + resumedDurableProposal: true, + } + client.status.Store(normal) + client.luc.Store(testUnicityCertificate(11, 20, committedRoot, nil)) + client.lastRootRound.Store(20) + client.nextExpectedRound.Store(12) + + err = client.handleUnicityCertificate( + t.Context(), + testUnicityCertificate(12, 21, proposalRoot, committedRoot), + &certification.TechnicalRecord{Round: 13, Epoch: 1}, + ) + + require.NoError(t, err) + require.Equal(t, 1, rm.durableRecoveryCallCnt) + require.EqualValues(t, 12, rm.durableRecoveryBlock.Uint64()) + require.Equal(t, proposalRoot, rm.durableRecoveryRoot) + require.NotEmpty(t, rm.durableRecoveryCert) + require.Zero(t, rm.finalizeBlockCallCnt) + require.Empty(t, rm.finalizedBlocks) + require.Len(t, rm.startedRounds, 1) + require.EqualValues(t, 13, rm.startedRounds[0].Uint64()) require.Nil(t, client.proposedBlock) + require.False(t, client.resumedDurableProposal) +} + +func TestBFTClientInitializationResendsDurableProposal(t *testing.T) { + log, err := logger.New("warn", "json", "", false) + require.NoError(t, err) + + committedRoot := bytes.Repeat([]byte{0x10}, api.SiblingSize) + proposalRoot := api.NewHexBytes(bytes.Repeat([]byte{0x20}, api.SiblingSize)) + proposal := models.NewBlock( + api.NewBigIntFromUint64(12), + "unicity", + 0, + "1.0", + "mainnet", + proposalRoot, + api.NewHexBytes(committedRoot), + nil, + ) + proposal.Status = models.FinalityStatusProposed + rm := &stubRoundManager{ + committedRoot: committedRoot, + committedBlock: api.NewBigIntFromUint64(11), + durableLoadedBlock: proposal, + durableLoadFound: true, + durableRecoveryRoot: nil, + } + client := &BFTClientImpl{ + logger: log, + roundManager: rm, + } + client.status.Store(initializing) + + err = client.handleUnicityCertificate( + t.Context(), + testUnicityCertificate(11, 21, committedRoot, nil), + &certification.TechnicalRecord{Round: 12, Epoch: 1}, + ) + + require.ErrorContains(t, err, "BFT client network is not initialized") + require.Equal(t, 1, rm.durableLoadCallCnt) + require.EqualValues(t, 12, rm.durableLoadBlock.Uint64()) + require.Empty(t, rm.startedRounds) + require.Same(t, proposal, client.proposedBlock) + require.True(t, client.resumedDurableProposal) + require.Equal(t, normal, client.status.Load().(status)) +} + +func TestBFTClientInitializationFinalizesCertifiedDurableProposalBeforeStartingNextRound(t *testing.T) { + log, err := logger.New("warn", "json", "", false) + require.NoError(t, err) + + committedRoot := bytes.Repeat([]byte{0x10}, api.SiblingSize) + certifiedRoot := bytes.Repeat([]byte{0x20}, api.SiblingSize) + rm := &stubRoundManager{ + committedRoot: committedRoot, + committedBlock: api.NewBigIntFromUint64(120), + durableRecovered: true, + } + client := &BFTClientImpl{ + logger: log, + roundManager: rm, + } + client.status.Store(initializing) + + err = client.handleUnicityCertificate( + t.Context(), + testUnicityCertificate(121, 90, certifiedRoot, committedRoot), + &certification.TechnicalRecord{Round: 123, Epoch: 1}, + ) + + require.NoError(t, err) + require.Equal(t, 1, rm.durableRecoveryCallCnt) + require.EqualValues(t, 121, rm.durableRecoveryBlock.Uint64()) + require.Equal(t, api.HexBytes(certifiedRoot), rm.durableRecoveryRoot) + require.NotEmpty(t, rm.durableRecoveryCert) + require.Zero(t, rm.durableLoadCallCnt) + require.Len(t, rm.startedRounds, 1) + require.EqualValues(t, 123, rm.startedRounds[0].Uint64()) + require.Nil(t, client.proposedBlock) + require.Equal(t, normal, client.status.Load().(status)) +} + +func TestBFTClientInitializationStartsRoundWhenNoDurableProposal(t *testing.T) { + log, err := logger.New("warn", "json", "", false) + require.NoError(t, err) + + committedRoot := bytes.Repeat([]byte{0x10}, api.SiblingSize) + rm := &stubRoundManager{ + committedRoot: committedRoot, + committedBlock: api.NewBigIntFromUint64(11), + } + client := &BFTClientImpl{ + logger: log, + roundManager: rm, + } + client.status.Store(initializing) + + err = client.handleUnicityCertificate( + t.Context(), + testUnicityCertificate(11, 21, committedRoot, nil), + &certification.TechnicalRecord{Round: 12, Epoch: 1}, + ) + + require.NoError(t, err) + require.Equal(t, 1, rm.durableLoadCallCnt) + require.EqualValues(t, 12, rm.durableLoadBlock.Uint64()) + require.Len(t, rm.startedRounds, 1) + require.EqualValues(t, 12, rm.startedRounds[0].Uint64()) + require.Nil(t, client.proposedBlock) + require.Equal(t, normal, client.status.Load().(status)) +} + +func TestBFTClientRejectsUCRootMismatch(t *testing.T) { + log, err := logger.New("warn", "json", "", false) + require.NoError(t, err) + + rm := &stubRoundManager{} + blockRoot := api.NewHexBytes(bytes.Repeat([]byte{0x11}, api.SiblingSize)) + ucRoot := bytes.Repeat([]byte{0x22}, api.SiblingSize) + block := models.NewBlock( + api.NewBigIntFromUint64(7), + "unicity", + 0, + "1.0", + "mainnet", + blockRoot, + nil, + nil, + ) + client := &BFTClientImpl{ + logger: log, + roundManager: rm, + proposedBlock: block, + } + + err = client.handleUnicityCertificate( + t.Context(), + &types.UnicityCertificate{ + InputRecord: &types.InputRecord{ + Version: 1, + RoundNumber: 7, + Hash: ucRoot, + }, + UnicitySeal: &types.UnicitySeal{RootChainRoundNumber: 7}, + }, + &certification.TechnicalRecord{Round: 8}, + ) + + require.ErrorIs(t, err, ErrCertifiedRootMismatch) + require.Empty(t, rm.finalizedBlocks) + require.Nil(t, client.proposedBlock) + require.Empty(t, block.UnicityCertificate) + // The durable proposal must be abandoned too (not just cleared in memory), + // so a stale Proposed block cannot survive restart and re-trigger the fatal. + require.Equal(t, 1, rm.durableAbandonCallCnt) + require.True(t, bytes.Equal(blockRoot, rm.durableAbandonRoot)) +} + +func TestBFTClientUCRootMismatchPublishesFatalWhenAbandonFails(t *testing.T) { + log, err := logger.New("warn", "json", "", false) + require.NoError(t, err) + + abandonErr := errors.New("abandon failed") + rm := &stubRoundManager{durableAbandonErr: abandonErr} + eventBus := events.NewEventBus(log) + fatalEvents := eventBus.Subscribe(events.TopicFatalError) + blockRoot := api.NewHexBytes(bytes.Repeat([]byte{0x11}, api.SiblingSize)) + ucRoot := bytes.Repeat([]byte{0x22}, api.SiblingSize) + block := models.NewBlock( + api.NewBigIntFromUint64(7), + "unicity", + 0, + "1.0", + "mainnet", + blockRoot, + nil, + nil, + ) + client := &BFTClientImpl{ + logger: log, + roundManager: rm, + eventBus: eventBus, + proposedBlock: block, + } + + err = client.handleUnicityCertificate( + t.Context(), + &types.UnicityCertificate{ + InputRecord: &types.InputRecord{ + Version: 1, + RoundNumber: 7, + Hash: ucRoot, + }, + UnicitySeal: &types.UnicitySeal{RootChainRoundNumber: 7}, + }, + &certification.TechnicalRecord{Round: 8}, + ) + + require.ErrorIs(t, err, ErrCertifiedRootMismatch) + require.Equal(t, 1, rm.durableAbandonCallCnt) select { case event := <-fatalEvents: fatal, ok := event.(events.FatalErrorEvent) require.True(t, ok) require.Equal(t, "bft", fatal.Source) - require.Contains(t, fatal.Error, ErrCertifiedStateMismatch.Error()) - case <-time.After(time.Second): - t.Fatal("expected fatal event") + require.Contains(t, fatal.Error, ErrCertifiedRootMismatch.Error()) + default: + t.Fatal("expected fatal event for certified root mismatch") + } +} + +func TestBFTClientRejectsRegressingUC(t *testing.T) { + log, err := logger.New("warn", "json", "", false) + require.NoError(t, err) + + rm := &stubRoundManager{} + prevUC := testUnicityCertificate(10, 20, bytes.Repeat([]byte{0x10}, api.SiblingSize), nil) + oldUC := testUnicityCertificate(9, 19, bytes.Repeat([]byte{0x09}, api.SiblingSize), nil) + client := &BFTClientImpl{ + logger: log, + roundManager: rm, } + client.luc.Store(prevUC) + client.lastRootRound.Store(prevUC.GetRootRoundNumber()) + client.nextExpectedRound.Store(11) + + err = client.handleUnicityCertificate(t.Context(), oldUC, &certification.TechnicalRecord{Round: 10, Epoch: 1}) + + require.ErrorIs(t, err, ErrInvalidUCSequence) + require.Same(t, prevUC, client.luc.Load()) + require.EqualValues(t, 11, client.nextExpectedRound.Load()) + require.Empty(t, rm.startedRounds) + require.Empty(t, rm.finalizedBlocks) +} + +func TestBFTClientRepeatUCStartsFreshRound(t *testing.T) { + log, err := logger.New("warn", "json", "", false) + require.NoError(t, err) + + root := bytes.Repeat([]byte{0x44}, api.SiblingSize) + prevUC := testUnicityCertificate(52, 80, root, nil) + repeatUC := testUnicityCertificate(52, 81, root, prevUC.InputRecord.PreviousHash) + rm := &stubRoundManager{} + client := &BFTClientImpl{ + logger: log, + roundManager: rm, + } + client.luc.Store(prevUC) + client.lastRootRound.Store(prevUC.GetRootRoundNumber()) + client.nextExpectedRound.Store(53) + proposedRoot := api.NewHexBytes(bytes.Repeat([]byte{0x55}, api.SiblingSize)) + client.proposedBlock = models.NewBlock( + api.NewBigIntFromUint64(53), + "unicity", + 0, + "1.0", + "mainnet", + proposedRoot, + api.NewHexBytes(root), + nil, + ) + + err = client.handleUnicityCertificate(t.Context(), repeatUC, &certification.TechnicalRecord{Round: 55, Epoch: 1}) + + require.NoError(t, err) + require.Equal(t, 1, rm.durableAbandonCallCnt) + require.EqualValues(t, 53, rm.durableAbandonBlock.Uint64()) + require.Equal(t, proposedRoot, rm.durableAbandonRoot) + require.Nil(t, client.proposedBlock) + require.Len(t, rm.startedRounds, 1) + require.EqualValues(t, 55, rm.startedRounds[0].Uint64()) + require.Empty(t, rm.finalizedBlocks) + require.Same(t, repeatUC, client.luc.Load()) + require.EqualValues(t, 55, client.nextExpectedRound.Load()) +} + +func TestBFTClientNewerUCAbandonsStaleProposalAndRecoversDurableProposal(t *testing.T) { + log, err := logger.New("warn", "json", "", false) + require.NoError(t, err) + + staleRoot := api.NewHexBytes(bytes.Repeat([]byte{0x33}, api.SiblingSize)) + certifiedRoot := bytes.Repeat([]byte{0x44}, api.SiblingSize) + rm := &stubRoundManager{durableRecovered: true} + client := &BFTClientImpl{ + logger: log, + roundManager: rm, + proposedBlock: models.NewBlock( + api.NewBigIntFromUint64(12), + "unicity", + 0, + "1.0", + "mainnet", + staleRoot, + nil, + nil, + ), + } + client.status.Store(normal) + client.nextExpectedRound.Store(12) + + err = client.handleUnicityCertificate( + t.Context(), + testUnicityCertificate(13, 25, certifiedRoot, nil), + &certification.TechnicalRecord{Round: 14, Epoch: 1}, + ) + + require.NoError(t, err) + require.Equal(t, 1, rm.durableAbandonCallCnt) + require.EqualValues(t, 12, rm.durableAbandonBlock.Uint64()) + require.Equal(t, staleRoot, rm.durableAbandonRoot) + require.Equal(t, 1, rm.durableRecoveryCallCnt) + require.EqualValues(t, 13, rm.durableRecoveryBlock.Uint64()) + require.Equal(t, api.HexBytes(certifiedRoot), rm.durableRecoveryRoot) + require.Len(t, rm.startedRounds, 1) + require.EqualValues(t, 14, rm.startedRounds[0].Uint64()) + require.Nil(t, client.proposedBlock) +} + +func TestBFTClientCertificationInputRecordUsesTechnicalEpoch(t *testing.T) { + client := &BFTClientImpl{} + client.nextExpectedEpoch.Store(3) + + previousRoot := bytes.Repeat([]byte{0x11}, api.SiblingSize) + newRoot := bytes.Repeat([]byte{0x22}, api.SiblingSize) + luc := testUnicityCertificate(7, 12, previousRoot, nil) + luc.InputRecord.Epoch = 2 + + ir, err := client.buildCertificationInputRecord(luc, newRoot, 8) + + require.NoError(t, err) + require.EqualValues(t, 8, ir.RoundNumber) + require.EqualValues(t, 3, ir.Epoch) + require.Equal(t, previousRoot, []byte(ir.PreviousHash)) + require.Equal(t, newRoot, []byte(ir.Hash)) + require.Equal(t, newRoot, []byte(ir.BlockHash)) } func TestBFTClientStub_CertificationRequest_PopulatesSyntheticUC(t *testing.T) { @@ -148,3 +665,20 @@ func TestBFTClientStub_CertificationRequest_PopulatesSyntheticUC(t *testing.T) { require.EqualValues(t, 7, uc.GetRoundNumber()) require.EqualValues(t, 7, uc.GetRootRoundNumber()) } + +func testUnicityCertificate(round, rootRound uint64, root []byte, previous []byte) *types.UnicityCertificate { + if previous == nil { + previous = bytes.Repeat([]byte{0x00}, api.SiblingSize) + } + return &types.UnicityCertificate{ + InputRecord: &types.InputRecord{ + Version: 1, + RoundNumber: round, + Epoch: 1, + PreviousHash: previous, + Hash: root, + BlockHash: root, + }, + UnicitySeal: &types.UnicitySeal{RootChainRoundNumber: rootRound}, + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 80d944b..75061dc 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -117,6 +117,7 @@ type ProcessingConfig struct { PrecollectorGracePeriod time.Duration `mapstructure:"precollector_grace_period"` // Extra wait before cutting a precollected round snapshot MaxCommitmentsPerRound int `mapstructure:"max_commitments_per_round"` // Stop waiting once this many commitments collected CollectPhaseDuration time.Duration `mapstructure:"collect_phase_duration"` // Non-child fixed collection window before proposing a round + CollectMiniBatchSize int `mapstructure:"collect_mini_batch_size"` // SMT/proposal staging mini-batch size during collection CommitmentStreamBufferSize int `mapstructure:"commitment_stream_buffer_size"` // Buffer between queue streamer and round collection SkipDuplicateCheck bool `mapstructure:"skip_duplicate_check"` // Skip finalized record lookup on submit } @@ -384,6 +385,7 @@ func Load() (*Config, error) { PrecollectorGracePeriod: getEnvDurationOrDefault("PRECOLLECTOR_GRACE_PERIOD", "0s"), MaxCommitmentsPerRound: getEnvIntOrDefault("MAX_COMMITMENTS_PER_ROUND", 20000), CollectPhaseDuration: getEnvDurationOrDefault("COLLECT_PHASE_DURATION", "200ms"), + CollectMiniBatchSize: getEnvIntOrDefault("COLLECT_MINI_BATCH_SIZE", 500), CommitmentStreamBufferSize: getEnvIntOrDefault("COMMITMENT_STREAM_BUFFER_SIZE", 50000), SkipDuplicateCheck: getEnvBoolOrDefault("SKIP_DUPLICATE_CHECK", true), }, @@ -421,7 +423,7 @@ func Load() (*Config, error) { RocksDBSubcompactions: getEnvIntOrDefault("SMT_ROCKSDB_SUBCOMPACTIONS", 4), RocksDBBloomBits: getEnvFloatOrDefault("SMT_ROCKSDB_BLOOM_BITS", 10), RocksDBMemTableMB: getEnvIntOrDefault("SMT_ROCKSDB_MEMTABLE_MB", 64), - MaterializeWorkers: getEnvIntOrDefault("SMT_MATERIALIZE_WORKERS", 64), + MaterializeWorkers: getEnvIntOrDefault("SMT_MATERIALIZE_WORKERS", 16), StartupReplayLimitBlocks: getEnvIntOrDefault("SMT_STARTUP_REPLAY_LIMIT_BLOCKS", 100), PrecomputeProofs: getEnvBoolOrDefault("SMT_PRECOMPUTE_PROOFS", false), ProofMetadataCacheEntries: getEnvIntOrDefault("SMT_PROOF_METADATA_CACHE_ENTRIES", 250000), @@ -519,6 +521,9 @@ func (c *Config) Validate() error { if c.Processing.CollectPhaseDuration <= 0 { return fmt.Errorf("COLLECT_PHASE_DURATION must be positive") } + if c.Processing.CollectMiniBatchSize <= 0 { + return fmt.Errorf("COLLECT_MINI_BATCH_SIZE must be positive") + } if c.Processing.PrecollectorGracePeriod < 0 { return fmt.Errorf("PRECOLLECTOR_GRACE_PERIOD must be non-negative") } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index ee7a963..dad047e 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -30,6 +30,7 @@ func validTestConfig() *Config { Processing: ProcessingConfig{ CommitmentStreamBufferSize: 10000, CollectPhaseDuration: 200 * time.Millisecond, + CollectMiniBatchSize: 500, }, BFT: BFTConfig{ Enabled: false, @@ -152,6 +153,19 @@ func TestConfigValidate_CollectPhaseDuration(t *testing.T) { } } +func TestConfigValidate_CollectMiniBatchSize(t *testing.T) { + cfg := validTestConfig() + cfg.Processing.CollectMiniBatchSize = 0 + + err := cfg.Validate() + if err == nil { + t.Fatal("Validate() expected error, got nil") + } + if !strings.Contains(err.Error(), "COLLECT_MINI_BATCH_SIZE") { + t.Fatalf("Validate() error = %q, want mini batch env name", err.Error()) + } +} + func TestConfigValidateTraceLogLevel(t *testing.T) { cfg := validTestConfig() cfg.Logging.Level = "trace" @@ -273,6 +287,28 @@ func TestMongoWriteConcernEnvDefaults(t *testing.T) { } } +func TestCollectMiniBatchSizeEnv(t *testing.T) { + t.Setenv("BFT_ENABLED", "false") + t.Setenv("DISABLE_HIGH_AVAILABILITY", "true") + + cfg, err := Load() + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if cfg.Processing.CollectMiniBatchSize != 500 { + t.Fatalf("CollectMiniBatchSize = %d, want default 500", cfg.Processing.CollectMiniBatchSize) + } + + t.Setenv("COLLECT_MINI_BATCH_SIZE", "750") + cfg, err = Load() + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if cfg.Processing.CollectMiniBatchSize != 750 { + t.Fatalf("CollectMiniBatchSize = %d, want env override 750", cfg.Processing.CollectMiniBatchSize) + } +} + func TestMongoWriteConcernEnvOverrides(t *testing.T) { t.Setenv("BFT_ENABLED", "false") t.Setenv("DISABLE_HIGH_AVAILABILITY", "true") diff --git a/internal/ha/block_syncer.go b/internal/ha/block_syncer.go index e8753c9..a9516a8 100644 --- a/internal/ha/block_syncer.go +++ b/internal/ha/block_syncer.go @@ -170,30 +170,21 @@ func (bs *BlockSyncer) syncMemoryToBlock(ctx context.Context, endBlock *big.Int) // fetch last synced smt block number and last stored block number currBlock := bs.stateTracker.GetLastSyncedBlock() for currBlock.Cmp(endBlock) < 0 { - // fetch the next block record - b, err := bs.storage.BlockRecordsStorage().GetNextBlock(ctx, api.NewBigInt(currBlock)) + b, err := bs.nextFinalizedBlock(ctx, currBlock, endBlock) if err != nil { - return fmt.Errorf("failed to fetch next block: %w", err) + return err } if b == nil { - return fmt.Errorf("next block record not found block: %s", currBlock.String()) + return fmt.Errorf("next finalized block not found after block: %s", currBlock.String()) } - // skip empty blocks - if len(b.StateIDs) == 0 { - bs.logger.WithContext(ctx).Debug("skipping block sync (empty block)", "blockNumber", b.BlockNumber.String()) - currBlock = b.BlockNumber.Int - bs.stateTracker.SetLastSyncedBlock(currBlock) - continue - } - bs.logger.WithContext(ctx).Debug("updating SMT for round", "blockNumber", b.BlockNumber.String()) + bs.logger.WithContext(ctx).Debug("updating SMT for round", "blockNumber", b.Index.String()) - // apply changes from block record to SMT if err := bs.updateSMTForBlock(ctx, b); err != nil { return fmt.Errorf("failed to update SMT: %w", err) } - currBlock = b.BlockNumber.Int + currBlock = b.Index.Int bs.stateTracker.SetLastSyncedBlock(currBlock) bs.logger.Info("SMT updated for round", "roundNumber", currBlock) } @@ -211,28 +202,21 @@ func (bs *BlockSyncer) syncDiskToBlock(ctx context.Context, endBlock *big.Int) e } for currBlock.Cmp(endBlock) < 0 { - b, err := bs.storage.BlockRecordsStorage().GetNextBlock(ctx, api.NewBigInt(currBlock)) + b, err := bs.nextFinalizedBlock(ctx, currBlock, endBlock) if err != nil { - return fmt.Errorf("failed to fetch next block: %w", err) + return err } if b == nil { - return fmt.Errorf("next block record not found block: %s", currBlock.String()) - } - if b.BlockNumber.Int.Cmp(endBlock) > 0 { - return fmt.Errorf("next block record %s is after latest finalized block %s", b.BlockNumber.String(), endBlock.String()) + return fmt.Errorf("next finalized block not found after block: %s", currBlock.String()) } - if len(b.StateIDs) == 0 { - bs.logger.WithContext(ctx).Debug("advancing disk SMT metadata for empty block", "blockNumber", b.BlockNumber.String()) - } else { - bs.logger.WithContext(ctx).Debug("updating disk SMT for round", "blockNumber", b.BlockNumber.String()) - } + bs.logger.WithContext(ctx).Debug("updating disk SMT for round", "blockNumber", b.Index.String()) if err := bs.updateSMTForBlock(ctx, b); err != nil { return fmt.Errorf("failed to update disk SMT: %w", err) } - currBlock = new(big.Int).Set(b.BlockNumber.Int) + currBlock = new(big.Int).Set(b.Index.Int) if bs.stateTracker != nil { bs.stateTracker.SetLastSyncedBlock(currBlock) } @@ -264,24 +248,31 @@ func (bs *BlockSyncer) usesDiskSMTBackend() bool { return ok && diskBacked.IsDiskBackedSMT() } -func (bs *BlockSyncer) verifySMTForBlock(ctx context.Context, smtRootHash api.HexBytes, blockNumber *api.BigInt) error { - block, err := bs.storage.BlockStorage().GetByNumber(ctx, blockNumber) +func (bs *BlockSyncer) nextFinalizedBlock(ctx context.Context, currBlock, endBlock *big.Int) (*models.Block, error) { + block, err := bs.storage.BlockStorage().GetNextFinalizedAfter(ctx, api.NewBigInt(new(big.Int).Set(currBlock)), api.NewBigInt(new(big.Int).Set(endBlock))) if err != nil { - return fmt.Errorf("failed to fetch block: %w", err) + return nil, fmt.Errorf("failed to fetch finalized blocks after %s: %w", currBlock.String(), err) } + return block, nil +} + +func (bs *BlockSyncer) verifySMTForBlock(smtRootHash api.HexBytes, block *models.Block) error { if block == nil { - return fmt.Errorf("block not found for block number: %s", blockNumber.String()) + return fmt.Errorf("block is nil") } - expectedRootHash := block.RootHash - if !bytes.Equal(smtRootHash, expectedRootHash) { + if !bytes.Equal(smtRootHash, block.RootHash) { return fmt.Errorf("smt root hash %s does not match latest block root hash %s", - smtRootHash.String(), expectedRootHash.String()) + smtRootHash.String(), block.RootHash.String()) } return nil } -func (bs *BlockSyncer) updateSMTForBlock(ctx context.Context, blockRecord *models.BlockRecords) error { - leaves, err := bs.replayLeavesForStateIDs(ctx, blockRecord.StateIDs) +func (bs *BlockSyncer) updateSMTForBlock(ctx context.Context, block *models.Block) error { + records, err := bs.storage.AggregatorRecordStorage().GetByBlockNumber(ctx, block.Index) + if err != nil { + return fmt.Errorf("failed to load aggregator records for block %s: %w", block.Index.String(), err) + } + leaves, err := replayLeavesForAggregatorRecords(records) if err != nil { return err } @@ -297,23 +288,32 @@ func (bs *BlockSyncer) updateSMTForBlock(ctx context.Context, blockRecord *model snapshot.Discard(ctx) } }() - result, err := snapshot.AddLeavesClassified(ctx, leaves) - if err != nil { - return fmt.Errorf("failed to apply SMT updates for block %s: %w", blockRecord.BlockNumber.String(), err) - } - if err := result.ValidateAllAccepted(len(leaves)); err != nil { - return fmt.Errorf("failed to apply SMT updates for block %s: %w", blockRecord.BlockNumber.String(), err) + var smtRootHash api.HexBytes + if len(leaves) == 0 { + root, err := snapshot.RootHashRaw(ctx) + if err != nil { + return fmt.Errorf("failed to read SMT root for empty block %s: %w", block.Index.String(), err) + } + smtRootHash = api.HexBytes(root) + } else { + result, err := snapshot.AddLeavesClassified(ctx, leaves) + if err != nil { + return fmt.Errorf("failed to apply SMT updates for block %s: %w", block.Index.String(), err) + } + if err := result.ValidateAllAccepted(len(leaves)); err != nil { + return fmt.Errorf("failed to apply SMT updates for block %s: %w", block.Index.String(), err) + } + smtRootHash = api.HexBytes(result.CandidateRoot) } - smtRootHash := api.HexBytes(result.CandidateRoot) // verify smt root hash matches the raw 32-byte block root hash - if err := bs.verifySMTForBlock(ctx, smtRootHash, blockRecord.BlockNumber); err != nil { + if err := bs.verifySMTForBlock(smtRootHash, block); err != nil { if bs.usesDiskSMTBackend() { return fmt.Errorf("%w: %w", ErrDiskSMTDiverged, err) } return fmt.Errorf("failed to verify SMT: %w", err) } // commit smt snapshot - if err := snapshot.Commit(ctx, smtbackend.CommitMetadata{BlockNumber: blockRecord.BlockNumber, RootHash: smtRootHash}); err != nil { + if err := snapshot.Commit(ctx, smtbackend.CommitMetadata{BlockNumber: block.Index, RootHash: smtRootHash}); err != nil { return fmt.Errorf("failed to commit SMT snapshot: %w", err) } committed = true @@ -324,40 +324,22 @@ func (bs *BlockSyncer) updateSMTForBlock(ctx context.Context, blockRecord *model return nil } -func (bs *BlockSyncer) replayLeavesForStateIDs(ctx context.Context, stateIDs []api.StateID) ([]smtbackend.LeafInput, error) { - if len(stateIDs) == 0 { +func replayLeavesForAggregatorRecords(records []*models.AggregatorRecord) ([]smtbackend.LeafInput, error) { + if len(records) == 0 { return nil, nil } - - uniqueStateIds := make(map[string]struct{}, len(stateIDs)) - leafIDs := make([]api.HexBytes, 0, len(stateIDs)) - for _, stateID := range stateIDs { - key := stateID.String() - if _, exists := uniqueStateIds[key]; exists { - continue + leaves := make([]smtbackend.LeafInput, 0, len(records)) + for i, record := range records { + if record == nil { + return nil, fmt.Errorf("nil aggregator record at index %d", i) } - uniqueStateIds[key] = struct{}{} - - keyBytes, err := stateID.GetTreeKey() + keyBytes, err := record.StateID.GetTreeKey() if err != nil { return nil, fmt.Errorf("failed to get SMT key: %w", err) } - leafIDs = append(leafIDs, api.NewHexBytes(keyBytes)) - } - - smtNodes, err := bs.storage.SmtStorage().GetByKeys(ctx, leafIDs) - if err != nil { - return nil, fmt.Errorf("failed to load smt nodes by keys: %w", err) - } - if len(smtNodes) != len(leafIDs) { - return nil, fmt.Errorf("expected %d SMT leaves, found %d", len(leafIDs), len(smtNodes)) - } - - leaves := make([]smtbackend.LeafInput, 0, len(smtNodes)) - for _, smtNode := range smtNodes { leaves = append(leaves, smtbackend.LeafInput{ - Key: smtNode.Key, - Value: smtNode.Value, + Key: append([]byte(nil), keyBytes...), + Value: append([]byte(nil), record.CertificationData.TransactionHash...), }) } return leaves, nil diff --git a/internal/ha/block_syncer_test.go b/internal/ha/block_syncer_test.go index 0749118..83b71ee 100644 --- a/internal/ha/block_syncer_test.go +++ b/internal/ha/block_syncer_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "math/big" + "sort" "sync/atomic" "testing" "time" @@ -216,6 +217,7 @@ func createBlock(t *testing.T, storage *mongodb.Storage, blockNum int64) api.Hex // persist aggregator records leaves := make([]*smt.Leaf, len(testCommitments)) records := make([]*models.AggregatorRecord, len(testCommitments)) + proposalID := fmt.Sprintf("proposal-%s", blockNumber.String()) for i, c := range testCommitments { path, err := c.StateID.GetPath() require.NoError(t, err) @@ -225,6 +227,7 @@ func createBlock(t *testing.T, storage *mongodb.Storage, blockNum int64) api.Hex leaves[i] = &smt.Leaf{Path: path, Value: val} records[i] = models.NewAggregatorRecord(c, blockNumber, api.NewBigInt(big.NewInt(int64(i)))) + records[i].ProposalID = proposalID err = storage.AggregatorRecordStorage().Store(ctx, records[i]) require.NoError(t, err) @@ -250,18 +253,10 @@ func createBlock(t *testing.T, storage *mongodb.Storage, blockNum int64) api.Hex // persist block block := models.NewBlock(blockNumber, "unicity", 0, "1.0", "mainnet", rootHash, nil, nil) block.Finalized = true // Mark as finalized so GetLatestNumber finds it + block.ProposalID = proposalID err = storage.BlockStorage().Store(ctx, block) require.NoError(t, err) - // persist block records (mapping state IDs) - stateIDs := make([]api.StateID, len(testCommitments)) - for i, c := range testCommitments { - stateIDs[i] = c.StateID - } - blockRecords := models.NewBlockRecords(blockNumber, stateIDs) - err = storage.BlockRecordsStorage().Store(ctx, blockRecords) - require.NoError(t, err) - return rootHash } @@ -310,8 +305,9 @@ func (f *blockSyncerFixture) addBlock(t *testing.T, blockNum int64, commitmentCo ctx := t.Context() blockNumber := api.NewBigInt(big.NewInt(blockNum)) leaves := make([]*smt.Leaf, 0, commitmentCount) - stateIDs := make([]api.StateID, 0, commitmentCount) nodes := make([]*models.SmtNode, 0, commitmentCount) + records := make([]*models.AggregatorRecord, 0, commitmentCount) + proposalID := fmt.Sprintf("proposal-%d", blockNum) for i := 0; i < commitmentCount; i++ { c := testutil.CreateTestCertificationRequest(t, fmt.Sprintf("block_%d_request_%d", blockNum, i)) @@ -323,8 +319,10 @@ func (f *blockSyncerFixture) addBlock(t *testing.T, blockNum int64, commitmentCo require.NoError(t, err) leaves = append(leaves, &smt.Leaf{Path: path, Value: value}) - stateIDs = append(stateIDs, c.StateID) nodes = append(nodes, models.NewSmtNode(api.HexBytes(key), api.HexBytes(value))) + record := models.NewAggregatorRecord(c, blockNumber, api.NewBigInt(big.NewInt(int64(i)))) + record.ProposalID = proposalID + records = append(records, record) } if len(leaves) > 0 { @@ -334,8 +332,9 @@ func (f *blockSyncerFixture) addBlock(t *testing.T, blockNum int64, commitmentCo block := models.NewBlock(blockNumber, "unicity", 0, "1.0", "test", rootHash, nil, nil) block.Finalized = true + block.ProposalID = proposalID require.NoError(t, f.storage.BlockStorage().Store(ctx, block)) - require.NoError(t, f.storage.BlockRecordsStorage().Store(ctx, models.NewBlockRecords(blockNumber, stateIDs))) + require.NoError(t, f.storage.AggregatorRecordStorage().StoreBatch(ctx, records)) require.NoError(t, f.storage.SmtStorage().StoreBatch(ctx, nodes)) return rootHash } @@ -346,11 +345,10 @@ func commitFixtureBlockToBackend(t *testing.T, ctx context.Context, backend smtb block, err := storage.BlockStorage().GetByNumber(ctx, blockNumber) require.NoError(t, err) require.NotNil(t, block) - blockRecord, err := storage.BlockRecordsStorage().GetByBlockNumber(ctx, blockNumber) + records, err := storage.AggregatorRecordStorage().GetByBlockNumber(ctx, blockNumber) require.NoError(t, err) - require.NotNil(t, blockRecord) - leaves, err := fixtureLeavesForStateIDs(ctx, storage, blockRecord.StateIDs) + leaves, err := replayLeavesForAggregatorRecords(records) require.NoError(t, err) snapshot, err := backend.CreateSnapshot(ctx) require.NoError(t, err) @@ -361,26 +359,6 @@ func commitFixtureBlockToBackend(t *testing.T, ctx context.Context, backend smtb require.NoError(t, snapshot.Commit(ctx, smtbackend.CommitMetadata{BlockNumber: blockNumber, RootHash: block.RootHash})) } -func fixtureLeavesForStateIDs(ctx context.Context, storage *blockSyncerTestStorage, stateIDs []api.StateID) ([]smtbackend.LeafInput, error) { - keys := make([]api.HexBytes, 0, len(stateIDs)) - for _, stateID := range stateIDs { - key, err := stateID.GetTreeKey() - if err != nil { - return nil, err - } - keys = append(keys, api.HexBytes(key)) - } - nodes, err := storage.SmtStorage().GetByKeys(ctx, keys) - if err != nil { - return nil, err - } - leaves := make([]smtbackend.LeafInput, len(nodes)) - for i, node := range nodes { - leaves[i] = smtbackend.LeafInput{Key: node.Key, Value: node.Value} - } - return leaves, nil -} - func bytesOf(n int, value byte) []byte { out := make([]byte, n) for i := range out { @@ -390,21 +368,21 @@ func bytesOf(n int, value byte) []byte { } type blockSyncerTestStorage struct { - blocks *blockSyncerTestBlockStorage - blockRecords *blockSyncerTestBlockRecordsStorage - smtNodes *blockSyncerTestSMTStorage + blocks *blockSyncerTestBlockStorage + records *blockSyncerTestAggregatorRecordStorage + smtNodes *blockSyncerTestSMTStorage } func newBlockSyncerTestStorage() *blockSyncerTestStorage { return &blockSyncerTestStorage{ - blocks: &blockSyncerTestBlockStorage{byNumber: make(map[string]*models.Block)}, - blockRecords: &blockSyncerTestBlockRecordsStorage{byNumber: make(map[string]*models.BlockRecords)}, - smtNodes: &blockSyncerTestSMTStorage{byKey: make(map[string]*models.SmtNode)}, + blocks: &blockSyncerTestBlockStorage{byNumber: make(map[string]*models.Block)}, + records: &blockSyncerTestAggregatorRecordStorage{byBlock: make(map[string][]*models.AggregatorRecord)}, + smtNodes: &blockSyncerTestSMTStorage{byKey: make(map[string]*models.SmtNode)}, } } func (s *blockSyncerTestStorage) AggregatorRecordStorage() interfaces.AggregatorRecordStorage { - return nil + return s.records } func (s *blockSyncerTestStorage) BlockStorage() interfaces.BlockStorage { @@ -415,10 +393,6 @@ func (s *blockSyncerTestStorage) SmtStorage() interfaces.SmtStorage { return s.smtNodes } -func (s *blockSyncerTestStorage) BlockRecordsStorage() interfaces.BlockRecordsStorage { - return s.blockRecords -} - func (s *blockSyncerTestStorage) LeadershipStorage() interfaces.LeadershipStorage { return nil } @@ -480,8 +454,37 @@ func (s *blockSyncerTestBlockStorage) Count(context.Context) (int64, error) { return int64(len(s.byNumber)), nil } -func (s *blockSyncerTestBlockStorage) GetRange(context.Context, *api.BigInt, *api.BigInt) ([]*models.Block, error) { - return nil, nil +func (s *blockSyncerTestBlockStorage) GetRange(_ context.Context, fromBlock, toBlock *api.BigInt) ([]*models.Block, error) { + var out []*models.Block + for _, block := range s.byNumber { + if !block.Finalized { + continue + } + if block.Index.Cmp(fromBlock.Int) < 0 || block.Index.Cmp(toBlock.Int) > 0 { + continue + } + out = append(out, block) + } + sort.Slice(out, func(i, j int) bool { + return out[i].Index.Cmp(out[j].Index.Int) < 0 + }) + return out, nil +} + +func (s *blockSyncerTestBlockStorage) GetNextFinalizedAfter(_ context.Context, afterBlock, toBlock *api.BigInt) (*models.Block, error) { + var next *models.Block + for _, block := range s.byNumber { + if !block.Finalized { + continue + } + if block.Index.Cmp(afterBlock.Int) <= 0 || block.Index.Cmp(toBlock.Int) > 0 { + continue + } + if next == nil || block.Index.Cmp(next.Index.Int) < 0 { + next = block + } + } + return next, nil } func (s *blockSyncerTestBlockStorage) SetFinalized(_ context.Context, blockNumber *api.BigInt, finalized bool) error { @@ -501,44 +504,77 @@ func (s *blockSyncerTestBlockStorage) GetUnfinalized(context.Context) ([]*models return out, nil } -type blockSyncerTestBlockRecordsStorage struct { - byNumber map[string]*models.BlockRecords +type blockSyncerTestAggregatorRecordStorage struct { + byBlock map[string][]*models.AggregatorRecord } -func (s *blockSyncerTestBlockRecordsStorage) Store(_ context.Context, records *models.BlockRecords) error { - s.byNumber[records.BlockNumber.String()] = records +func (s *blockSyncerTestAggregatorRecordStorage) Store(_ context.Context, record *models.AggregatorRecord) error { + s.byBlock[record.BlockNumber.String()] = append(s.byBlock[record.BlockNumber.String()], record) return nil } -func (s *blockSyncerTestBlockRecordsStorage) GetByBlockNumber(_ context.Context, blockNumber *api.BigInt) (*models.BlockRecords, error) { - return s.byNumber[blockNumber.String()], nil +func (s *blockSyncerTestAggregatorRecordStorage) StoreBatch(ctx context.Context, records []*models.AggregatorRecord) error { + for _, record := range records { + if err := s.Store(ctx, record); err != nil { + return err + } + } + return nil } -func (s *blockSyncerTestBlockRecordsStorage) Count(context.Context) (int64, error) { - return int64(len(s.byNumber)), nil +func (s *blockSyncerTestAggregatorRecordStorage) GetByStateID(_ context.Context, stateID api.StateID) (*models.AggregatorRecord, error) { + for _, records := range s.byBlock { + for _, record := range records { + if record != nil && string(record.StateID) == string(stateID) { + return record, nil + } + } + } + return nil, nil } -func (s *blockSyncerTestBlockRecordsStorage) GetNextBlock(_ context.Context, blockNumber *api.BigInt) (*models.BlockRecords, error) { - var next *models.BlockRecords - for _, record := range s.byNumber { - if blockNumber != nil && record.BlockNumber.Cmp(blockNumber.Int) <= 0 { - continue +func (s *blockSyncerTestAggregatorRecordStorage) GetByBlockNumber(_ context.Context, blockNumber *api.BigInt) ([]*models.AggregatorRecord, error) { + records := append([]*models.AggregatorRecord(nil), s.byBlock[blockNumber.String()]...) + sort.SliceStable(records, func(i, j int) bool { + left := records[i].LeafIndex + right := records[j].LeafIndex + if left == nil || left.Int == nil { + return right != nil && right.Int != nil } - if next == nil || record.BlockNumber.Cmp(next.BlockNumber.Int) < 0 { - next = record + if right == nil || right.Int == nil { + return false } + return left.Int.Cmp(right.Int) < 0 + }) + return records, nil +} + +func (s *blockSyncerTestAggregatorRecordStorage) Count(context.Context) (int64, error) { + var count int64 + for _, records := range s.byBlock { + count += int64(len(records)) } - return next, nil + return count, nil } -func (s *blockSyncerTestBlockRecordsStorage) GetLatestBlockNumber(context.Context) (*api.BigInt, error) { - var latest *api.BigInt - for _, record := range s.byNumber { - if latest == nil || record.BlockNumber.Cmp(latest.Int) > 0 { - latest = record.BlockNumber +func (s *blockSyncerTestAggregatorRecordStorage) GetExistingStateIDs(_ context.Context, stateIDs []string) (map[string]bool, error) { + out := make(map[string]bool, len(stateIDs)) + wanted := make(map[string]struct{}, len(stateIDs)) + for _, stateID := range stateIDs { + wanted[stateID] = struct{}{} + } + for _, records := range s.byBlock { + for _, record := range records { + if record == nil { + continue + } + key := record.StateID.String() + if _, ok := wanted[key]; ok { + out[key] = true + } } } - return latest, nil + return out, nil } type blockSyncerTestSMTStorage struct { @@ -616,10 +652,9 @@ func (s *blockSyncerTestSMTStorage) GetExistingKeys(_ context.Context, keys []st } var ( - _ interfaces.Storage = (*blockSyncerTestStorage)(nil) - _ interfaces.BlockStorage = (*blockSyncerTestBlockStorage)(nil) - _ interfaces.BlockRecordsStorage = (*blockSyncerTestBlockRecordsStorage)(nil) - _ interfaces.SmtStorage = (*blockSyncerTestSMTStorage)(nil) - _ smtbackend.DiskBacked = (*fakeDiskBackend)(nil) - _ smtbackend.ProofViewPublisher = (*fakeDiskBackend)(nil) + _ interfaces.Storage = (*blockSyncerTestStorage)(nil) + _ interfaces.BlockStorage = (*blockSyncerTestBlockStorage)(nil) + _ interfaces.SmtStorage = (*blockSyncerTestSMTStorage)(nil) + _ smtbackend.DiskBacked = (*fakeDiskBackend)(nil) + _ smtbackend.ProofViewPublisher = (*fakeDiskBackend)(nil) ) diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 4364e07..90f162c 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -244,6 +244,17 @@ var ( []string{"path"}, ) + SMTInclusionCertBuildDuration = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "aggregator_smt_inclusion_cert_build_duration_seconds", + Help: "SMT inclusion certificate construction latency inside get_inclusion_proof.v2; excludes proof-readiness and client polling.", + Buckets: []float64{ + .0005, .001, .0025, .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, + }, + }, + []string{"source", "result"}, + ) + ParentProofErrorsTotal = promauto.NewCounter( prometheus.CounterOpts{ Name: "aggregator_parent_proof_errors_total", diff --git a/internal/models/aggregator_record.go b/internal/models/aggregator_record.go index 4586207..b5930be 100644 --- a/internal/models/aggregator_record.go +++ b/internal/models/aggregator_record.go @@ -17,8 +17,8 @@ type AggregatorRecord struct { AggregateRequestCount uint64 `json:"aggregateRequestCount"` BlockNumber *api.BigInt `json:"blockNumber"` LeafIndex *api.BigInt `json:"leafIndex"` + ProposalID string `json:"proposalId,omitempty"` CreatedAt *api.Timestamp `json:"createdAt"` - FinalizedAt *api.Timestamp `json:"finalizedAt"` } // AggregatorRecordBSON represents the BSON version of AggregatorRecord for MongoDB storage @@ -29,8 +29,8 @@ type AggregatorRecordBSON struct { AggregateRequestCount uint64 `bson:"aggregateRequestCount"` BlockNumber primitive.Decimal128 `bson:"blockNumber"` LeafIndex primitive.Decimal128 `bson:"leafIndex"` + ProposalID string `bson:"proposalId,omitempty"` CreatedAt time.Time `bson:"createdAt"` - FinalizedAt time.Time `bson:"finalizedAt"` } // NewAggregatorRecord creates a new aggregator record from a certification request @@ -43,7 +43,6 @@ func NewAggregatorRecord(certRequest *CertificationRequest, blockNumber, leafInd BlockNumber: blockNumber, LeafIndex: leafIndex, CreatedAt: certRequest.CreatedAt, - FinalizedAt: api.Now(), } } @@ -57,6 +56,10 @@ func (ar *AggregatorRecord) ToBSON() (*AggregatorRecordBSON, error) { if err != nil { return nil, fmt.Errorf("error converting leaf index to decimal-128: %w", err) } + var createdAt time.Time + if ar.CreatedAt != nil { + createdAt = ar.CreatedAt.Time + } return &AggregatorRecordBSON{ Version: ar.Version, StateID: ar.StateID.String(), @@ -64,8 +67,8 @@ func (ar *AggregatorRecord) ToBSON() (*AggregatorRecordBSON, error) { AggregateRequestCount: ar.AggregateRequestCount, BlockNumber: blockNumber, LeafIndex: leafIndex, - CreatedAt: ar.CreatedAt.Time, - FinalizedAt: ar.FinalizedAt.Time, + ProposalID: ar.ProposalID, + CreatedAt: createdAt, }, nil } @@ -91,14 +94,15 @@ func (arb *AggregatorRecordBSON) FromBSON() (*AggregatorRecord, error) { return nil, fmt.Errorf("failed to decode stateID: %w", err) } - return &AggregatorRecord{ + record := &AggregatorRecord{ Version: arb.Version, StateID: stateID, CertificationData: *certDataBSON, AggregateRequestCount: arb.AggregateRequestCount, BlockNumber: blockNumber, LeafIndex: leafIndex, + ProposalID: arb.ProposalID, CreatedAt: api.NewTimestamp(arb.CreatedAt), - FinalizedAt: api.NewTimestamp(arb.FinalizedAt), - }, nil + } + return record, nil } diff --git a/internal/models/aggregator_record_test.go b/internal/models/aggregator_record_test.go index 0db4750..496b1cd 100644 --- a/internal/models/aggregator_record_test.go +++ b/internal/models/aggregator_record_test.go @@ -28,7 +28,6 @@ func TestAggregatorRecordSerialization(t *testing.T) { BlockNumber: originalBlockNumber, LeafIndex: originalLeafIndex, CreatedAt: api.Now(), - FinalizedAt: api.Now(), } // Convert to BSON diff --git a/internal/models/block.go b/internal/models/block.go index 72495d5..f7942ac 100644 --- a/internal/models/block.go +++ b/internal/models/block.go @@ -24,6 +24,8 @@ type Block struct { ParentFragment *api.ParentInclusionFragment `json:"parentFragment,omitempty"` // child mode only ParentBlockNumber uint64 `json:"parentBlockNumber,omitempty"` // child mode only Finalized bool `json:"finalized"` // true when all data is persisted + Status string `json:"status"` + ProposalID string `json:"proposalId,omitempty"` } // BlockBSON represents the BSON version of Block for MongoDB storage @@ -41,6 +43,8 @@ type BlockBSON struct { ParentFragment *ParentFragmentBSON `bson:"parentFragment,omitempty"` // child mode only ParentBlockNumber uint64 `bson:"parentBlockNumber,omitempty"` Finalized bool `bson:"finalized"` + Status string `bson:"status,omitempty"` + ProposalID string `bson:"proposalId,omitempty"` } // ParentFragmentBSON is the BSON representation of ParentInclusionFragment. @@ -76,6 +80,8 @@ func (b *Block) ToBSON() (*BlockBSON, error) { ParentFragment: parentFragment, ParentBlockNumber: b.ParentBlockNumber, Finalized: b.Finalized, + Status: b.Status, + ProposalID: b.ProposalID, }, nil } @@ -128,6 +134,8 @@ func (bb *BlockBSON) FromBSON() (*Block, error) { ParentFragment: parentFragment, ParentBlockNumber: bb.ParentBlockNumber, Finalized: bb.Finalized, + Status: bb.Status, + ProposalID: bb.ProposalID, }, nil } diff --git a/internal/models/block_record.go b/internal/models/block_record.go deleted file mode 100644 index 0f41bc7..0000000 --- a/internal/models/block_record.go +++ /dev/null @@ -1,75 +0,0 @@ -package models - -import ( - "fmt" - "time" - - "go.mongodb.org/mongo-driver/bson/primitive" - - "github.com/unicitynetwork/aggregator-go/pkg/api" -) - -// BlockRecords represents the mapping of block numbers to state IDs -type BlockRecords struct { - BlockNumber *api.BigInt `json:"blockNumber"` - StateIDs []api.StateID `json:"stateIds"` - CreatedAt *api.Timestamp `json:"createdAt"` -} - -// BlockRecordsBSON is the MongoDB representation of BlockRecords -type BlockRecordsBSON struct { - BlockNumber primitive.Decimal128 `bson:"blockNumber"` - StateIDs []string `bson:"stateIds"` - CreatedAt time.Time `bson:"createdAt"` -} - -// NewBlockRecords creates a new block records entry -func NewBlockRecords(blockNumber *api.BigInt, stateIDs []api.StateID) *BlockRecords { - return &BlockRecords{ - BlockNumber: blockNumber, - StateIDs: stateIDs, - CreatedAt: api.Now(), - } -} - -// ToBSON converts BlockRecords to BlockRecordsBSON -func (br *BlockRecords) ToBSON() (*BlockRecordsBSON, error) { - blockNumberDecimal, err := primitive.ParseDecimal128(br.BlockNumber.String()) - if err != nil { - return nil, fmt.Errorf("error converting block number to decimal: %w", err) - } - - stateIDs := make([]string, len(br.StateIDs)) - for i, r := range br.StateIDs { - stateIDs[i] = r.String() - } - - return &BlockRecordsBSON{ - BlockNumber: blockNumberDecimal, - StateIDs: stateIDs, - CreatedAt: br.CreatedAt.Time, - }, nil -} - -// FromBSON converts BlockRecordsBSON to BlockRecords -func (brb *BlockRecordsBSON) FromBSON() (*BlockRecords, error) { - blockNumber, _, err := brb.BlockNumber.BigInt() - if err != nil { - return nil, fmt.Errorf("failed to parse blockNumber: %w", err) - } - - stateIDs := make([]api.StateID, len(brb.StateIDs)) - for i, r := range brb.StateIDs { - id, err := api.NewImprintV2(r) - if err != nil { - return nil, fmt.Errorf("failed to decode stateID %s: %w", r, err) - } - stateIDs[i] = id - } - - return &BlockRecords{ - BlockNumber: api.NewBigInt(blockNumber), - StateIDs: stateIDs, - CreatedAt: api.NewTimestamp(brb.CreatedAt), - }, nil -} diff --git a/internal/models/finality_status.go b/internal/models/finality_status.go new file mode 100644 index 0000000..75d30ac --- /dev/null +++ b/internal/models/finality_status.go @@ -0,0 +1,8 @@ +package models + +const ( + FinalityStatusFinalized = "finalized" + FinalityStatusFinalizing = "finalizing" + FinalityStatusProposed = "proposed" + FinalityStatusAbandoned = "abandoned" +) diff --git a/internal/round/batch_processor.go b/internal/round/batch_processor.go index 6c36d18..9c1762d 100644 --- a/internal/round/batch_processor.go +++ b/internal/round/batch_processor.go @@ -8,7 +8,6 @@ import ( "math/big" "runtime" "sort" - "sync" "time" "github.com/unicitynetwork/bft-go-base/types" @@ -77,17 +76,19 @@ func (rm *RoundManager) proposeBlock(ctx context.Context, round *Round, blockNum "blockNumber", blockNumber.String(), "rootHash", rootHash.String()) + if round == nil { + return bft.ErrStaleCertificationRound + } + rm.roundMutex.Lock() if rm.currentRound != round { rm.roundMutex.Unlock() return bft.ErrStaleCertificationRound } - if round != nil { - rm.logger.WithContext(ctx).Debug("Changing round state to finalizing", - "roundNumber", round.Number.String(), - "previousState", round.State.String()) - round.State = RoundStateFinalizing - } + rm.logger.WithContext(ctx).Debug("Changing round state to finalizing", + "roundNumber", round.Number.String(), + "previousState", round.State.String()) + round.State = RoundStateFinalizing rm.roundMutex.Unlock() rm.logger.WithContext(ctx).Info("Creating block proposal", @@ -133,6 +134,10 @@ func (rm *RoundManager) proposeBlock(ctx context.Context, round *Round, blockNum parentHash, nil, ) + block.ProposalID = round.ProposalID + if err := rm.ensureDurableProposal(ctx, round, block); err != nil { + return err + } if round != nil && !round.StartTime.IsZero() { metrics.RoundPreparationDuration.Observe(time.Since(round.StartTime).Seconds()) } @@ -142,12 +147,20 @@ func (rm *RoundManager) proposeBlock(ctx context.Context, round *Round, blockNum superseded := rm.currentRound != round rm.roundMutex.RUnlock() if superseded { + if err := rm.AbandonDurableProposal(ctx, block.Index, block.RootHash); err != nil { + return fmt.Errorf("failed to abandon superseded durable proposal: %w", err) + } return bft.ErrStaleCertificationRound } rm.logger.WithContext(ctx).Info("Sending certification request to BFT client", "blockNumber", blockNumber.String(), "bftClientType", fmt.Sprintf("%T", rm.bftClient)) if err := rm.bftClient.CertificationRequest(ctx, block); err != nil { + if errors.Is(err, bft.ErrStaleCertificationRound) { + if abandonErr := rm.AbandonDurableProposal(ctx, block.Index, block.RootHash); abandonErr != nil { + return fmt.Errorf("failed to abandon stale durable proposal: %w", abandonErr) + } + } rm.logger.WithContext(ctx).Error("Failed to send certification request", "blockNumber", blockNumber.String(), "error", err.Error()) @@ -225,6 +238,10 @@ func (rm *RoundManager) proposeBlock(ctx context.Context, round *Round, blockNum proof.ParentFragment, proof.BlockNumber, ) + block.ProposalID = round.ProposalID + if err := rm.ensureDurableProposal(ctx, round, block); err != nil { + return err + } if err := rm.FinalizeBlockWithRetry(ctx, block); err != nil { return fmt.Errorf("failed to finalize block after retries: %w", err) } @@ -240,6 +257,12 @@ func (rm *RoundManager) proposeBlock(ctx context.Context, round *Round, blockNum if cp != nil { preResult, advErr := rm.advancePrecollectorForHandoff(cp) if advErr == nil { + nextRound := api.NewBigInt(nextRoundNumber) + if err := validatePrecollectorBlockNumber(preResult, nextRound); err != nil { + preResult.snapshot.Discard(ctx) + rm.logger.WithContext(ctx).Error("Precollector block number mismatch.", "error", err.Error()) + return err + } if err := preResult.snapshot.SetCommitTarget(ctx, rm.smtBackend); err != nil { preResult.snapshot.Discard(ctx) rm.logger.WithContext(ctx).Error("Failed to set precollector commit target.", "error", err.Error()) @@ -247,7 +270,7 @@ func (rm *RoundManager) proposeBlock(ctx context.Context, round *Round, blockNum } // StartNewRoundWithSnapshot atomically checks precollectorDisabled // under roundMutex — no race with concurrent Deactivate. - if err := rm.StartNewRoundWithSnapshot(ctx, api.NewBigInt(nextRoundNumber), preResult.snapshot, preResult.commitments, preResult.leaves); err != nil { + if err := rm.StartNewRoundWithSnapshot(ctx, nextRound, preResult.snapshot, preResult.commitments, preResult.leaves, preResult.recordsStaged, preResult.proposalID); err != nil { preResult.snapshot.Discard(ctx) if !errors.Is(err, ErrDeactivated) { rm.logger.WithContext(ctx).Error("Failed to start new round with snapshot.", "error", err.Error()) @@ -272,6 +295,61 @@ func (rm *RoundManager) proposeBlock(ctx context.Context, round *Round, blockNum } } +func (rm *RoundManager) ensureDurableProposal(ctx context.Context, round *Round, block *models.Block) error { + records, err := rm.proposalRecordsForRound(round, block.Index) + if err != nil { + return err + } + if err := rm.storeProposedBlockAndRecords(ctx, block, records); err != nil { + if !errors.Is(err, interfaces.ErrDuplicateKey) { + return fmt.Errorf("failed to store durable proposal: %w", err) + } + if _, err := rm.validateStoredDurableProposalBlock(ctx, block); err != nil { + return err + } + rm.logger.WithContext(ctx).Info("Durable proposal already exists", + "blockNumber", block.Index.String()) + } + return nil +} + +func (rm *RoundManager) proposalRecordsForRound(round *Round, blockNumber *api.BigInt) ([]*models.AggregatorRecord, error) { + if round == nil { + return nil, bft.ErrStaleCertificationRound + } + rm.roundMutex.RLock() + if rm.currentRound != round { + rm.roundMutex.RUnlock() + return nil, bft.ErrStaleCertificationRound + } + pendingCommitments := append([]*models.CertificationRequest(nil), round.PendingCommitments...) + recordsStaged := round.ProposalRecordsStaged + rm.roundMutex.RUnlock() + + if recordsStaged { + return nil, nil + } + records := rm.convertCommitmentsToRecords(pendingCommitments, blockNumber) + for _, record := range records { + record.ProposalID = round.ProposalID + } + return records, nil +} + +func (rm *RoundManager) stageProposedCommitments(ctx context.Context, blockNumber *api.BigInt, proposalID string, leafIndexOffset int, commitments []*models.CertificationRequest) error { + if blockNumber == nil || len(commitments) == 0 { + return nil + } + records := rm.convertCommitmentsToRecordsFrom(commitments, blockNumber, leafIndexOffset) + for _, record := range records { + record.ProposalID = proposalID + } + if err := rm.storage.AggregatorRecordStorage().StoreBatch(ctx, records); err != nil { + return fmt.Errorf("failed to stage proposed aggregator records: %w", err) + } + return nil +} + func (rm *RoundManager) pollForParentProof(ctx context.Context, rootHash api.HexBytes) (*api.RootShardInclusionProof, *types.UnicityCertificate, error) { pollingCtx, cancel := context.WithTimeout(ctx, rm.config.Sharding.Child.ParentPollTimeout) defer cancel() @@ -371,10 +449,20 @@ const ( // FinalizeBlockWithRetry retries finalization and uses recovery if block was partially stored. func (rm *RoundManager) FinalizeBlockWithRetry(ctx context.Context, block *models.Block) error { for attempt := 1; attempt <= maxFinalizeRetries; attempt++ { + if err := ctx.Err(); err != nil { + return err + } + err := rm.FinalizeBlock(ctx, block) if err == nil { return nil } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return err + } + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } rm.logger.Error("FinalizeBlock failed", "attempt", attempt, @@ -388,7 +476,7 @@ func (rm *RoundManager) FinalizeBlockWithRetry(ctx context.Context, block *model } else if len(unfinalizedBlocks) > 0 { rm.logger.Info("Found unfinalized block, attempting recovery", "blockNumber", unfinalizedBlocks[0].Index.String()) - recoveryResult, recoverErr := RecoverUnfinalizedBlock(ctx, rm.logger, rm.storage, rm.commitmentQueue) + recoveryResult, recoverErr := recoverUnfinalizedBlock(ctx, rm.logger, rm.storage, rm.commitmentQueue, !rm.usesDiskSMTBackend()) if recoverErr != nil { return fmt.Errorf("recovery failed: %w", recoverErr) } @@ -403,7 +491,18 @@ func (rm *RoundManager) FinalizeBlockWithRetry(ctx context.Context, block *model if attempt < maxFinalizeRetries { rm.logger.Info("Retrying FinalizeBlock", "attempt", attempt) - time.Sleep(finalizeRetryDelay) + timer := time.NewTimer(finalizeRetryDelay) + select { + case <-timer.C: + case <-ctx.Done(): + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + return ctx.Err() + } } } return fmt.Errorf("FinalizeBlock failed after %d attempts", maxFinalizeRetries) @@ -512,68 +611,73 @@ func (rm *RoundManager) FinalizeBlock(ctx context.Context, block *models.Block) } finalizationScanDuration := time.Since(finalizationScanStart) - finalizationConvertStart := time.Now() - smtNodes, err := rm.convertLeavesToNodes(pendingLeaves) - if err != nil { - return fmt.Errorf("failed to convert leaves to storage nodes: %w", err) + var smtNodesToStore []*models.SmtNode + var finalizationConvertDuration time.Duration + if !rm.usesDiskSMTBackend() { + finalizationConvertStart := time.Now() + var err error + smtNodesToStore, err = rm.convertLeavesToNodes(pendingLeaves) + if err != nil { + return fmt.Errorf("failed to convert leaves to storage nodes: %w", err) + } + finalizationConvertDuration = time.Since(finalizationConvertStart) } - records := rm.convertCommitmentsToRecords(pendingCommitments, block.Index) - finalizationConvertDuration := time.Since(finalizationConvertStart) - block.Finalized = false storeBlockStart := time.Now() - if err := rm.storeBlockAndRecords(ctx, block, stateIDs); err != nil { - if !errors.Is(err, interfaces.ErrDuplicateKey) { - return fmt.Errorf("failed to store block and records: %w", err) - } - existingBlock, err := rm.validateDuplicateBlock(ctx, block, stateIDs) + var existingBlock *models.Block + var records []*models.AggregatorRecord + var err error + existingBlock, err = rm.validateStoredDurableProposalBlock(ctx, block) + if err != nil { + return err + } + records = rm.convertCommitmentsToRecords(pendingCommitments, block.Index) + for _, record := range records { + record.ProposalID = existingBlock.ProposalID + } + block.ProposalID = existingBlock.ProposalID + if existingBlock.Finalized { + localRoot, err := rm.smtBackend.RootHashRaw(ctx) if err != nil { - return err + return fmt.Errorf("failed to validate local SMT root for finalized duplicate block %s: %w", block.Index.String(), err) } - if existingBlock.Finalized { - localRoot, err := rm.smtBackend.RootHashRaw(ctx) - if err != nil { - return fmt.Errorf("failed to validate local SMT root for finalized duplicate block %s: %w", block.Index.String(), err) - } - if bytes.Equal(localRoot, block.RootHash) { - if publisher, ok := rm.smtBackend.(smtbackend.ProofViewPublisher); ok { - if err := publisher.RefreshPublishedProofView(ctx, block.RootHash); err != nil { - return fmt.Errorf("failed to refresh proof view for finalized duplicate block %s: %w", block.Index.String(), err) - } + if bytes.Equal(localRoot, block.RootHash) { + if publisher, ok := rm.smtBackend.(smtbackend.ProofViewPublisher); ok { + if err := publisher.RefreshPublishedProofView(ctx, block.RootHash); err != nil { + return fmt.Errorf("failed to refresh proof view for finalized duplicate block %s: %w", block.Index.String(), err) } - block.Finalized = true - rm.markProofsReady(block, stateIDs, records) - if snapshot != nil { - snapshot.Discard(ctx) - } - rm.clearFinalizedRound(round, block) - rm.logger.WithContext(ctx).Info("Block already finalized with matching local SMT root, skipping duplicate finalization", - "blockNumber", block.Index.String()) - return nil } + block.Finalized = true + rm.markProofsReady(block, stateIDs, records) + if snapshot != nil { + snapshot.Discard(ctx) + } + rm.clearFinalizedRound(round, block) + rm.logger.WithContext(ctx).Info("Block already finalized with matching local SMT root, skipping duplicate finalization", + "blockNumber", block.Index.String()) + return nil } - rm.logger.WithContext(ctx).Info("Block already exists, continuing with remaining steps", - "blockNumber", block.Index.String()) } storeBlockDuration := time.Since(storeBlockStart) var storeDataTiming storeDataTiming var smtCommitDuration time.Duration + var smtCommitTiming smtbackend.CommitTiming var finalizationLockWaitDuration time.Duration var preparedProofView smtbackend.PreparedProofView diskFinalize := rm.usesDiskSMTBackend() && snapshot != nil - if diskFinalize { - var err error - storeDataTiming, smtCommitDuration, finalizationLockWaitDuration, preparedProofView, err = rm.storeDataAndCommitDiskSnapshot(ctx, block, snapshot, smtNodes, records) - if err != nil { + if !diskFinalize { + if err := setBlockFinalizingWithCertificate(ctx, rm.storage, block); err != nil { return err } - } else { + block.Finalized = false + block.Status = models.FinalityStatusFinalizing + var err error - storeDataTiming, err = rm.storeDataParallel(ctx, smtNodes, records) + storeDataTiming, err = rm.storeDataParallel(ctx, smtNodesToStore) if err != nil { - return fmt.Errorf("failed to store SMT nodes and aggregator records: %w", err) + return fmt.Errorf("failed to store SMT nodes: %w", err) } lockWaitStart := time.Now() @@ -589,7 +693,6 @@ func (rm *RoundManager) FinalizeBlock(ctx context.Context, block *models.Block) } smtCommitDuration = time.Since(smtCommitStart) } - metrics.SMTCommitDuration.Observe(smtCommitDuration.Seconds()) proofViewPublished := false defer func() { if preparedProofView != nil && !proofViewPublished { @@ -598,14 +701,25 @@ func (rm *RoundManager) FinalizeBlock(ctx context.Context, block *models.Block) }() setFinalizedStart := time.Now() - if err := rm.storage.BlockStorage().SetFinalized(ctx, block.Index, true); err != nil { + if err := setBlockFinalizedWithCertificate(ctx, rm.storage, block); err != nil { if !diskFinalize { rm.finalizationMu.Unlock() } - return fmt.Errorf("failed to set block as finalized: %w", err) + return err } setFinalizedDuration := time.Since(setFinalizedStart) block.Finalized = true + block.Status = models.FinalityStatusFinalized + + if diskFinalize { + var err error + smtCommitDuration, smtCommitTiming, preparedProofView, err = rm.commitDiskSnapshotForFinalizedBlock(ctx, block, snapshot) + if err != nil { + return err + } + } + metrics.SMTCommitDuration.Observe(smtCommitDuration.Seconds()) + precomputeProofDuration := time.Duration(0) precomputeProofTiming := precomputeProofTiming{} if rm.config.SMT.PrecomputeProofs { @@ -684,12 +798,7 @@ func (rm *RoundManager) FinalizeBlock(ctx context.Context, block *models.Block) totalRoundTime = processingTime + actualFinalizationTime } - shortDur := func(d time.Duration) string { - if d <= 0 { - return "0ms" - } - return fmt.Sprintf("%dms", d.Milliseconds()) - } + shortDur := shortPerfDuration logFields := []interface{}{ "block", block.Index.String(), @@ -706,6 +815,14 @@ func (rm *RoundManager) FinalizeBlock(ctx context.Context, block *models.Block) "finalizeStoreRecords", shortDur(storeDataTiming.records), "finalizeLockWait", shortDur(finalizationLockWaitDuration), "finalizeSmtCommit", shortDur(smtCommitDuration), + "finalizeSmtCommitCollect", shortDur(smtCommitTiming.CollectDuration), + "finalizeSmtCommitTombstone", shortDur(smtCommitTiming.TombstoneDuration), + "finalizeSmtCommitBatchBuild", shortDur(smtCommitTiming.BatchBuildDuration), + "finalizeSmtCommitRootHash", shortDur(smtCommitTiming.RootHashDuration), + "finalizeSmtCommitEngineWrite", shortDur(smtCommitTiming.EngineWriteDuration), + "finalizeSmtCommitCacheUpdate", shortDur(smtCommitTiming.CacheUpdateDuration), + "finalizeSmtCommitNodeWrites", smtCommitTiming.NodeWrites, + "finalizeSmtCommitNodeDeletes", smtCommitTiming.NodeDeletes, "finalizeSetFinalized", shortDur(setFinalizedDuration), "finalizePrecomputeProofs", shortDur(precomputeProofDuration), "finalizePrecomputeKeys", shortDur(precomputeProofTiming.keys), @@ -714,6 +831,11 @@ func (rm *RoundManager) FinalizeBlock(ctx context.Context, block *models.Block) "finalizePrecomputeStore", shortDur(precomputeProofTiming.store), "finalizeAck", shortDur(ackDuration), } + if !proposalTime.IsZero() { + logFields = append(logFields, + "proposalToProofReady", shortDur(proofReadyAt.Sub(proposalTime)), + ) + } if len(proofTimes) > 0 { sorted := make([]time.Duration, len(proofTimes)) @@ -751,7 +873,7 @@ func (rm *RoundManager) FinalizeBlock(ctx context.Context, block *models.Block) ) } - rm.logger.WithContext(ctx).Info("PERF: Round completed", logFields...) + rm.logger.WithContext(ctx).Info("Round completed", logFields...) rm.logger.WithContext(ctx).Info("Block finalized and stored successfully", "blockNumber", block.Index.String(), @@ -792,6 +914,13 @@ func (rm *RoundManager) validateBlockForMode(block *models.Block) error { return nil } +func shortPerfDuration(d time.Duration) string { + if d <= 0 { + return "0ms" + } + return fmt.Sprintf("%dms", d.Milliseconds()) +} + // convertLeavesToNodes converts backend leaf inputs to storage models. func (rm *RoundManager) convertLeavesToNodes(leaves []smtbackend.LeafInput) ([]*models.SmtNode, error) { if len(leaves) == 0 { @@ -812,13 +941,17 @@ func (rm *RoundManager) convertLeavesToNodes(leaves []smtbackend.LeafInput) ([]* // convertCommitmentsToRecords converts commitments to aggregator records func (rm *RoundManager) convertCommitmentsToRecords(commitments []*models.CertificationRequest, blockIndex *api.BigInt) []*models.AggregatorRecord { + return rm.convertCommitmentsToRecordsFrom(commitments, blockIndex, 0) +} + +func (rm *RoundManager) convertCommitmentsToRecordsFrom(commitments []*models.CertificationRequest, blockIndex *api.BigInt, leafIndexOffset int) []*models.AggregatorRecord { if len(commitments) == 0 { return nil } records := make([]*models.AggregatorRecord, len(commitments)) for i, commitment := range commitments { - leafIndex := api.NewBigInt(big.NewInt(int64(i))) + leafIndex := api.NewBigInt(big.NewInt(int64(leafIndexOffset + i))) records[i] = models.NewAggregatorRecord(commitment, blockIndex, leafIndex) } return records @@ -924,16 +1057,30 @@ func precomputedProofBlockNumber(mode config.ShardingMode, block *models.Block) return block.ParentBlockNumber, nil } -// executeBlockTransaction executes the block finalization transaction. -// storeBlockAndRecords stores the block and block records in a mini-transaction. -// The block is stored with finalized=false. -func (rm *RoundManager) storeBlockAndRecords(ctx context.Context, block *models.Block, stateIDs []api.StateID) error { +func (rm *RoundManager) storeProposedBlockAndRecords( + ctx context.Context, + block *models.Block, + records []*models.AggregatorRecord, +) error { + block.Finalized = false + block.Status = models.FinalityStatusProposed + return rm.storage.WithTransaction(ctx, func(txCtx context.Context) error { if err := rm.storage.BlockStorage().Store(txCtx, block); err != nil { - return fmt.Errorf("failed to store block: %w", err) + return fmt.Errorf("failed to store proposed block: %w", err) } - if err := rm.storage.BlockRecordsStorage().Store(txCtx, models.NewBlockRecords(block.Index, stateIDs)); err != nil { - return fmt.Errorf("failed to store block records: %w", err) + if len(records) > 0 { + if serial, ok := rm.storage.AggregatorRecordStorage().(aggregatorRecordSerialBatchStorage); ok { + if err := serial.StoreBatchSerial(txCtx, records); err != nil { + return fmt.Errorf("failed to store proposed aggregator records: %w", err) + } + } else { + for _, record := range records { + if err := rm.storage.AggregatorRecordStorage().Store(txCtx, record); err != nil { + return fmt.Errorf("failed to store proposed aggregator record: %w", err) + } + } + } } return nil }) @@ -951,61 +1098,35 @@ func (rm *RoundManager) clearFinalizedRound(round *Round, block *models.Block) { } } -func (rm *RoundManager) validateDuplicateBlock(ctx context.Context, block *models.Block, stateIDs []api.StateID) (*models.Block, error) { - existingBlock, err := rm.getExistingBlockAnyFinalization(ctx, block.Index) +func (rm *RoundManager) validateStoredDurableProposalBlock(ctx context.Context, block *models.Block) (*models.Block, error) { + existingBlock, err := getBlockAnyFinalization(ctx, rm.storage, block.Index) if err != nil { - return nil, fmt.Errorf("failed to validate duplicate block %s: %w", block.Index.String(), err) + return nil, fmt.Errorf("failed to load durable proposal block %s: %w", block.Index.String(), err) } if existingBlock == nil { - return nil, fmt.Errorf("duplicate block %s exists but could not be loaded", block.Index.String()) + return nil, fmt.Errorf("durable proposal block %s not found before finalization", block.Index.String()) } if !bytes.Equal(existingBlock.RootHash, block.RootHash) { - return nil, fmt.Errorf("duplicate block %s root mismatch: existing %s new %s", + return nil, fmt.Errorf("durable proposal block %s root mismatch: existing %s new %s", block.Index.String(), existingBlock.RootHash.String(), block.RootHash.String()) } - - existingRecords, err := rm.storage.BlockRecordsStorage().GetByBlockNumber(ctx, block.Index) - if err != nil { - return nil, fmt.Errorf("failed to load duplicate block records %s: %w", block.Index.String(), err) - } - if existingRecords == nil { - return nil, fmt.Errorf("duplicate block %s has no block records", block.Index.String()) - } - if !sameStateIDs(existingRecords.StateIDs, stateIDs) { - return nil, fmt.Errorf("duplicate block %s state IDs mismatch", block.Index.String()) - } return existingBlock, nil } -func (rm *RoundManager) getExistingBlockAnyFinalization(ctx context.Context, blockNumber *api.BigInt) (*models.Block, error) { - block, err := rm.storage.BlockStorage().GetByNumber(ctx, blockNumber) - if err != nil { - return nil, err - } - if block != nil { - return block, nil - } - - unfinalized, err := rm.storage.BlockStorage().GetUnfinalized(ctx) - if err != nil { - return nil, err - } - for _, candidate := range unfinalized { - if candidate != nil && candidate.Index.Cmp(blockNumber.Int) == 0 { - return candidate, nil - } - } - return nil, nil -} - func sameStateIDs(a, b []api.StateID) bool { if len(a) != len(b) { return false } - for i := range a { - if !bytes.Equal(a[i], b[i]) { + counts := make(map[string]int, len(a)) + for _, stateID := range a { + counts[stateID.String()]++ + } + for _, stateID := range b { + key := stateID.String() + if counts[key] == 0 { return false } + counts[key]-- } return true } @@ -1084,33 +1205,21 @@ func signedBytesToMB(bytes int64) int64 { return bytes / (1024 * 1024) } -type storeDataResult struct { - timing storeDataTiming - err error -} - -// storeDataAndCommitDiskSnapshot overlaps Mongo finalization data writes with -// the RocksDB snapshot commit. The block is still marked finalized only after -// both operations succeed, so either side can be recovered from the existing -// unfinalized-block startup path if the process crashes or one operation fails. +// commitDiskSnapshotForFinalizedBlock advances the local RocksDB committed +// state only after Mongo has persisted the UC/finalized block. If the process +// crashes between the Mongo finalize and this commit, startup can replay the +// finalized block from Mongo. The reverse ordering can leave RocksDB ahead of +// Mongo with only an uncertified proposed block, which is not recoverable. // // Disk proof reads do not use finalizationMu: they read from the last published // proof view. On success this returns a proof view captured at the RocksDB // commit point; the caller publishes it only after Mongo finalization and proof // metadata are ready. -func (rm *RoundManager) storeDataAndCommitDiskSnapshot( +func (rm *RoundManager) commitDiskSnapshotForFinalizedBlock( ctx context.Context, block *models.Block, snapshot smtbackend.Snapshot, - smtNodes []*models.SmtNode, - records []*models.AggregatorRecord, -) (storeDataTiming, time.Duration, time.Duration, smtbackend.PreparedProofView, error) { - storeCh := make(chan storeDataResult, 1) - go func() { - timing, err := rm.storeDataParallel(ctx, smtNodes, records) - storeCh <- storeDataResult{timing: timing, err: err} - }() - +) (time.Duration, smtbackend.CommitTiming, smtbackend.PreparedProofView, error) { smtCommitStart := time.Now() var preparedProofView smtbackend.PreparedProofView var commitErr error @@ -1120,72 +1229,47 @@ func (rm *RoundManager) storeDataAndCommitDiskSnapshot( commitErr = snapshot.Commit(ctx, smtbackend.CommitMetadata{BlockNumber: block.Index, RootHash: block.RootHash}) } smtCommitDuration := time.Since(smtCommitStart) + var smtCommitTiming smtbackend.CommitTiming + if timingProvider, ok := snapshot.(smtbackend.CommitTimingProvider); ok { + smtCommitTiming = timingProvider.LastCommitTiming() + } - storeResult := <-storeCh - if commitErr != nil || storeResult.err != nil { + if commitErr != nil { if preparedProofView != nil { preparedProofView.Discard(ctx) } - if commitErr != nil && storeResult.err != nil { - return storeResult.timing, smtCommitDuration, 0, nil, - fmt.Errorf("failed to store finalization data and commit SMT snapshot: store=%v commit=%w", storeResult.err, commitErr) - } - if commitErr != nil { - return storeResult.timing, smtCommitDuration, 0, nil, - fmt.Errorf("failed to commit SMT snapshot: %w", commitErr) - } - return storeResult.timing, smtCommitDuration, 0, nil, - fmt.Errorf("failed to store SMT nodes and aggregator records: %w", storeResult.err) + return smtCommitDuration, smtCommitTiming, nil, fmt.Errorf("failed to commit SMT snapshot: %w", commitErr) } - return storeResult.timing, smtCommitDuration, 0, preparedProofView, nil + return smtCommitDuration, smtCommitTiming, preparedProofView, nil } -// storeDataParallel stores SMT nodes and aggregator records in parallel. -// StoreBatch handles duplicates internally (ignores duplicate key errors). +// storeDataParallel stores SMT nodes. Aggregator records are written once as a +// durable proposal before certification, never during finalization. func (rm *RoundManager) storeDataParallel( ctx context.Context, smtNodes []*models.SmtNode, - records []*models.AggregatorRecord, ) (storeDataTiming, error) { start := time.Now() - var smtErr, recordsErr error - var smtTime, recordsTime time.Duration - - // Run SMT and AggregatorRecords storage in parallel - var wg sync.WaitGroup + var smtErr error + var smtTime time.Duration if len(smtNodes) > 0 { - wg.Go(func() { - t := time.Now() - smtErr = rm.storage.SmtStorage().StoreBatch(ctx, smtNodes) - smtTime = time.Since(t) - }) + t := time.Now() + smtErr = rm.storage.SmtStorage().StoreBatch(ctx, smtNodes) + smtTime = time.Since(t) } - if len(records) > 0 { - wg.Go(func() { - t := time.Now() - recordsErr = rm.storage.AggregatorRecordStorage().StoreBatch(ctx, records) - recordsTime = time.Since(t) - }) - } - - wg.Wait() - timing := storeDataTiming{ total: time.Since(start), smt: smtTime, - records: recordsTime, + records: 0, } if smtErr != nil { return timing, fmt.Errorf("failed to store SMT nodes: %w", smtErr) } - if recordsErr != nil { - return timing, fmt.Errorf("failed to store aggregator records: %w", recordsErr) - } return timing, nil } diff --git a/internal/round/disk_bft_integration_rocksdb_test.go b/internal/round/disk_bft_integration_rocksdb_test.go index 4a062fd..9939a37 100644 --- a/internal/round/disk_bft_integration_rocksdb_test.go +++ b/internal/round/disk_bft_integration_rocksdb_test.go @@ -60,10 +60,10 @@ func TestDiskSMTBFTShardPendingAlignmentWithDuplicateFiltering(t *testing.T) { require.Len(t, dropped, 1) require.True(t, block1.Finalized) - block1Records, err := storage.BlockRecordsStorage().GetByBlockNumber(ctx, block1.Index) + block1Records, err := storage.AggregatorRecordStorage().GetByBlockNumber(ctx, block1.Index) require.NoError(t, err) - require.NotNil(t, block1Records) - require.Equal(t, []api.StateID{c1.StateID, c2.StateID}, block1Records.StateIDs) + require.Len(t, block1Records, 2) + require.Equal(t, []api.StateID{c1.StateID, c2.StateID}, stateIDsFromAggregatorRecords(block1Records)) c3 := testutil.CreateTestCertificationRequest(t, "disk_align_3") committedConflict := *c1 @@ -73,10 +73,10 @@ func TestDiskSMTBFTShardPendingAlignmentWithDuplicateFiltering(t *testing.T) { require.Len(t, dropped, 1) require.True(t, block2.Finalized) - block2Records, err := storage.BlockRecordsStorage().GetByBlockNumber(ctx, block2.Index) + block2Records, err := storage.AggregatorRecordStorage().GetByBlockNumber(ctx, block2.Index) require.NoError(t, err) - require.NotNil(t, block2Records) - require.Equal(t, []api.StateID{c3.StateID}, block2Records.StateIDs) + require.Len(t, block2Records, 1) + require.Equal(t, []api.StateID{c3.StateID}, stateIDsFromAggregatorRecords(block2Records)) } func TestDiskSMTBFTShardSequentialRoundsMatchMemoryBackend(t *testing.T) { @@ -127,10 +127,9 @@ func TestDiskSMTBFTShardSequentialRoundsMatchMemoryBackend(t *testing.T) { require.Equal(t, api.HexBytes(expectedRoot), block.RootHash, "round %d root mismatch", blockNumber) require.Equal(t, stateIDsFromAcks(expectedDropped), stateIDsFromAcks(diskDropped), "round %d dropped set mismatch", blockNumber) - blockRecords, err := storage.BlockRecordsStorage().GetByBlockNumber(ctx, block.Index) + blockRecords, err := storage.AggregatorRecordStorage().GetByBlockNumber(ctx, block.Index) require.NoError(t, err) - require.NotNil(t, blockRecords) - require.Equal(t, expectedStateIDs, blockRecords.StateIDs, "round %d block records mismatch", blockNumber) + require.Equal(t, expectedStateIDs, stateIDsFromAggregatorRecords(blockRecords), "round %d block records mismatch", blockNumber) diskState, err := rm.smtBackend.CommittedState(ctx) require.NoError(t, err) @@ -313,10 +312,19 @@ func finalizeManualDiskRound( api.HexBytes{}, uc, ) + storeDurableProposalForCurrentRound(t, ctx, rm, block) require.NoError(t, rm.FinalizeBlock(ctx, block)) return block, dropped } +func stateIDsFromAggregatorRecords(records []*models.AggregatorRecord) []api.StateID { + stateIDs := make([]api.StateID, len(records)) + for i, record := range records { + stateIDs[i] = record.StateID + } + return stateIDs +} + func testDiskStoreCounters(t *testing.T, ctx context.Context, rm *RoundManager) diskstorage.Counters { t.Helper() stats := rm.smtBackend.Stats(ctx) diff --git a/internal/round/disk_ha_failover_integration_rocksdb_test.go b/internal/round/disk_ha_failover_integration_rocksdb_test.go index 8d7d773..59eb9d0 100644 --- a/internal/round/disk_ha_failover_integration_rocksdb_test.go +++ b/internal/round/disk_ha_failover_integration_rocksdb_test.go @@ -111,8 +111,12 @@ func TestDiskSMTHAFollowerRejectsDivergentFinalizedRoot(t *testing.T) { uc, ) block.Finalized = true + block.Status = models.FinalityStatusFinalized + block.ProposalID = "proposal-" + block.Index.String() require.NoError(t, storage.BlockStorage().Store(ctx, block)) - require.NoError(t, storage.BlockRecordsStorage().Store(ctx, models.NewBlockRecords(block.Index, []api.StateID{commitment.StateID}))) + record := models.NewAggregatorRecord(commitment, block.Index, api.NewBigIntFromUint64(0)) + record.ProposalID = block.ProposalID + require.NoError(t, storage.AggregatorRecordStorage().Store(ctx, record)) require.NoError(t, storage.SmtStorage().Store(ctx, models.NewSmtNode(leaf.Key, leaf.Value))) followerCfg := cfg diff --git a/internal/round/disk_smt_startup.go b/internal/round/disk_smt_startup.go index 4823290..3003a5f 100644 --- a/internal/round/disk_smt_startup.go +++ b/internal/round/disk_smt_startup.go @@ -268,22 +268,16 @@ func (rm *RoundManager) replayDiskSMTGap(ctx context.Context, fromBlock, latestB "latestBlock", latestBlock.String(), "blocksBehind", diff.String()) - current := new(big.Int).Set(fromBlock.Int) - for current.Cmp(latestBlock.Int) < 0 { - blockRecords, err := rm.storage.BlockRecordsStorage().GetNextBlock(ctx, api.NewBigInt(new(big.Int).Set(current))) - if err != nil { - return nil, recordDiskSMTStartupFailure(fmt.Errorf("failed to fetch next disk SMT replay block after %s: %w", current.String(), err)) - } - if blockRecords == nil { - return nil, recordDiskSMTStartupFailure(fmt.Errorf("next finalized block record not found after %s before latest block %s", current.String(), latestBlock.String())) - } - if blockRecords.BlockNumber.Int.Cmp(latestBlock.Int) > 0 { - return nil, recordDiskSMTStartupFailure(fmt.Errorf("next block record %s is after latest finalized block %s", blockRecords.BlockNumber.String(), latestBlock.String())) - } - if err := rm.replayDiskSMTBlock(ctx, blockRecords.BlockNumber); err != nil { + from := api.NewBigInt(new(big.Int).Add(fromBlock.Int, big.NewInt(1))) + blocks, err := rm.storage.BlockStorage().GetRange(ctx, from, latestBlock) + if err != nil { + return nil, diskSMTStartupFailure("failed to fetch finalized blocks for disk SMT replay from %s to %s: %w", + from.String(), latestBlock.String(), err) + } + for _, block := range blocks { + if err := rm.replayDiskSMTBlock(ctx, block); err != nil { return nil, recordDiskSMTStartupFailure(err) } - current = new(big.Int).Set(blockRecords.BlockNumber.Int) } state, err := rm.smtBackend.CommittedState(ctx) @@ -293,13 +287,13 @@ func (rm *RoundManager) replayDiskSMTGap(ctx context.Context, fromBlock, latestB if state.BlockNumber == nil || state.BlockNumber.Cmp(latestBlock.Int) != 0 { return nil, diskSMTStartupFailure("disk SMT replay ended at block %v, expected %s", state.BlockNumber, latestBlock.String()) } - latest, err := rm.storage.BlockStorage().GetLatest(ctx) + replayedTo, err := rm.storage.BlockStorage().GetByNumber(ctx, latestBlock) if err != nil { - return nil, diskSMTStartupFailure("failed to reload latest block after disk SMT replay: %w", err) + return nil, diskSMTStartupFailure("failed to reload replay target block %s after disk SMT replay: %w", latestBlock.String(), err) } - if latest == nil || !bytes.Equal(state.RootHash, latest.RootHash) { - return nil, diskSMTStartupFailure("disk SMT replay root mismatch: disk=%s latest=%v", - api.HexBytes(state.RootHash).String(), latest) + if replayedTo == nil || !bytes.Equal(state.RootHash, replayedTo.RootHash) { + return nil, diskSMTStartupFailure("disk SMT replay root mismatch: disk=%s replayTarget=%v", + api.HexBytes(state.RootHash).String(), replayedTo) } if rm.stateTracker != nil { rm.stateTracker.SetLastSyncedBlock(latestBlock.Int) @@ -311,18 +305,14 @@ func (rm *RoundManager) replayDiskSMTGap(ctx context.Context, fromBlock, latestB return latestBlock, nil } -func (rm *RoundManager) replayDiskSMTBlock(ctx context.Context, blockNumber *api.BigInt) error { - block, err := rm.storage.BlockStorage().GetByNumber(ctx, blockNumber) - if err != nil { - return fmt.Errorf("failed to load block %s for disk SMT replay: %w", blockNumber.String(), err) - } +func (rm *RoundManager) replayDiskSMTBlock(ctx context.Context, block *models.Block) error { if block == nil || !block.Finalized { - return fmt.Errorf("finalized block %s not found for disk SMT replay", blockNumber.String()) + return fmt.Errorf("finalized block not found for disk SMT replay") } state, err := rm.smtBackend.CommittedState(ctx) if err != nil { - return fmt.Errorf("failed to read disk SMT committed state before replay block %s: %w", blockNumber.String(), err) + return fmt.Errorf("failed to read disk SMT committed state before replay block %s: %w", block.Index.String(), err) } if state.BlockNumber != nil && state.BlockNumber.Cmp(block.Index.Int) >= 0 { if state.BlockNumber.Cmp(block.Index.Int) == 0 && bytes.Equal(state.RootHash, block.RootHash) { @@ -332,82 +322,40 @@ func (rm *RoundManager) replayDiskSMTBlock(ctx context.Context, blockNumber *api state.BlockNumber.String(), api.HexBytes(state.RootHash).String(), block.Index.String(), block.RootHash.String()) } - blockRecords, err := rm.storage.BlockRecordsStorage().GetByBlockNumber(ctx, blockNumber) + records, err := loadAggregatorRecordsForBlockAnyFinalization(ctx, rm.storage, block) if err != nil { - return fmt.Errorf("failed to load block records for disk SMT replay block %s: %w", blockNumber.String(), err) - } - if blockRecords == nil { - return fmt.Errorf("block records for disk SMT replay block %s not found", blockNumber.String()) + return fmt.Errorf("failed to load aggregator records for disk SMT replay block %s: %w", block.Index.String(), err) } - - leaves, err := rm.replayLeavesForStateIDs(ctx, blockRecords.StateIDs) + _, leaves, err := stateIDsAndLeavesFromAggregatorRecords(records) if err != nil { - return fmt.Errorf("failed to load replay leaves for block %s: %w", blockNumber.String(), err) + return fmt.Errorf("failed to load replay leaves for block %s: %w", block.Index.String(), err) } snapshot, err := rm.smtBackend.CreateSnapshot(ctx) if err != nil { - return fmt.Errorf("failed to create disk SMT replay snapshot for block %s: %w", blockNumber.String(), err) + return fmt.Errorf("failed to create disk SMT replay snapshot for block %s: %w", block.Index.String(), err) } result, err := snapshot.AddLeavesClassified(ctx, leaves) if err != nil { snapshot.Discard(ctx) - return fmt.Errorf("failed to replay disk SMT leaves for block %s: %w", blockNumber.String(), err) + return fmt.Errorf("failed to replay disk SMT leaves for block %s: %w", block.Index.String(), err) } if err := result.ValidateAllAccepted(len(leaves)); err != nil { snapshot.Discard(ctx) - return fmt.Errorf("failed to replay disk SMT leaves for block %s: %w", blockNumber.String(), err) + return fmt.Errorf("failed to replay disk SMT leaves for block %s: %w", block.Index.String(), err) } if !bytes.Equal(result.CandidateRoot, block.RootHash) { snapshot.Discard(ctx) return fmt.Errorf("disk SMT replay root mismatch at block %s: candidate=%s block=%s", - blockNumber.String(), api.HexBytes(result.CandidateRoot).String(), block.RootHash.String()) + block.Index.String(), api.HexBytes(result.CandidateRoot).String(), block.RootHash.String()) } if err := snapshot.Commit(ctx, smtbackend.CommitMetadata{BlockNumber: block.Index, RootHash: block.RootHash}); err != nil { snapshot.Discard(ctx) - return fmt.Errorf("failed to commit disk SMT replay block %s: %w", blockNumber.String(), err) + return fmt.Errorf("failed to commit disk SMT replay block %s: %w", block.Index.String(), err) } return nil } -func (rm *RoundManager) replayLeavesForStateIDs(ctx context.Context, stateIDs []api.StateID) ([]smtbackend.LeafInput, error) { - if len(stateIDs) == 0 { - return nil, nil - } - - seen := make(map[string]struct{}, len(stateIDs)) - keys := make([]api.HexBytes, 0, len(stateIDs)) - for _, stateID := range stateIDs { - key := string(stateID) - if _, exists := seen[key]; exists { - continue - } - seen[key] = struct{}{} - keyBytes, err := stateID.GetTreeKey() - if err != nil { - return nil, fmt.Errorf("stateID %s tree key: %w", stateID.String(), err) - } - keys = append(keys, api.HexBytes(keyBytes)) - } - - nodes, err := rm.storage.SmtStorage().GetByKeys(ctx, keys) - if err != nil { - return nil, err - } - if len(nodes) != len(keys) { - return nil, fmt.Errorf("expected %d SMT nodes, found %d", len(keys), len(nodes)) - } - - leaves := make([]smtbackend.LeafInput, len(nodes)) - for i, node := range nodes { - leaves[i] = smtbackend.LeafInput{ - Key: node.Key, - Value: node.Value, - } - } - return leaves, nil -} - func emptyStateRoot() []byte { root := disk.EmptyRootHash() out := make([]byte, len(root)) diff --git a/internal/round/disk_smt_startup_test.go b/internal/round/disk_smt_startup_test.go index b555006..548a631 100644 --- a/internal/round/disk_smt_startup_test.go +++ b/internal/round/disk_smt_startup_test.go @@ -128,11 +128,11 @@ func TestDiskSMTStartupBoundedReplay(t *testing.T) { root2 := result.CandidateRoot snapshot.Discard(ctx) - stateID2 := api.StateID(leaf2.Key) + block2 := diskStartupBlock(2, root2) storage := &diskStartupStorage{ - blocks: newDiskStartupBlockStorage(diskStartupBlock(1, root1), diskStartupBlock(2, root2)), - records: newDiskStartupBlockRecordsStorage(models.NewBlockRecords(api.NewBigIntFromUint64(2), []api.StateID{stateID2})), - smt: newDiskStartupSMTStorage(models.NewSmtNode(leaf2.Key, leaf2.Value)), + blocks: newDiskStartupBlockStorage(diskStartupBlock(1, root1), block2), + smt: newDiskStartupSMTStorage(models.NewSmtNode(leaf2.Key, leaf2.Value)), + aggregator: newDiskStartupAggregatorRecordStorage(diskStartupAggregatorRecord(block2, leaf2, 0)), } rm := newDiskStartupRoundManager(t, backend, storage) rm.config.SMT.StartupReplayLimitBlocks = 1 @@ -147,46 +147,85 @@ func TestDiskSMTStartupBoundedReplay(t *testing.T) { require.Equal(t, root2, state.RootHash) } -func TestDiskSMTStartupBoundedReplayToleratesMissingEmptyRounds(t *testing.T) { +func TestDiskSMTStartupBoundedReplayIgnoresLatestAdvanceDuringReplay(t *testing.T) { ctx := context.Background() backend := newTestDiskBackendForStartup(t) leaf1 := diskStartupLeaf(1, 11) root1 := commitDiskStartupLeaves(t, ctx, backend, 1, []smtbackend.LeafInput{leaf1}) + leaf2 := diskStartupLeaf(2, 22) + snapshot, err := backend.CreateSnapshot(ctx) + require.NoError(t, err) + result, err := snapshot.AddLeavesClassified(ctx, []smtbackend.LeafInput{leaf2}) + require.NoError(t, err) + root2 := result.CandidateRoot + snapshot.Discard(ctx) + + leaf3 := diskStartupLeaf(3, 33) + snapshot, err = backend.CreateSnapshot(ctx) + require.NoError(t, err) + result, err = snapshot.AddLeavesClassified(ctx, []smtbackend.LeafInput{leaf2, leaf3}) + require.NoError(t, err) + root3 := result.CandidateRoot + snapshot.Discard(ctx) + + block2 := diskStartupBlock(2, root2) + block3 := diskStartupBlock(3, root3) + blockStorage := newDiskStartupBlockStorage(diskStartupBlock(1, root1), block2) + blockStorage.blocks[block3.Index.String()] = block3 + blockStorage.afterGetRange = func() { + blockStorage.latest = block3 + } storage := &diskStartupStorage{ - blocks: newDiskStartupBlockStorage(diskStartupBlock(1, root1), diskStartupBlock(3, root1)), - records: newDiskStartupBlockRecordsStorage(models.NewBlockRecords(api.NewBigIntFromUint64(3), nil)), + blocks: blockStorage, + smt: newDiskStartupSMTStorage(models.NewSmtNode(leaf2.Key, leaf2.Value)), + aggregator: newDiskStartupAggregatorRecordStorage(diskStartupAggregatorRecord(block2, leaf2, 0)), } rm := newDiskStartupRoundManager(t, backend, storage) - rm.config.SMT.StartupReplayLimitBlocks = 2 + rm.config.SMT.StartupReplayLimitBlocks = 1 block, err := rm.verifyDiskSMTStartup(ctx) require.NoError(t, err) - require.Equal(t, uint64(3), block.Uint64()) + require.Equal(t, uint64(2), block.Uint64()) state, err := backend.CommittedState(ctx) require.NoError(t, err) - require.Equal(t, uint64(3), state.BlockNumber.Uint64()) - require.Equal(t, root1, state.RootHash) + require.Equal(t, uint64(2), state.BlockNumber.Uint64()) + require.Equal(t, root2, state.RootHash) + require.Equal(t, block3, blockStorage.latest, "test setup must advance latest while replay is in progress") } -func TestDiskSMTStartupBoundedReplayGapStillFailsOnRootMismatch(t *testing.T) { +func TestDiskSMTStartupBoundedReplaySkipsMissingRepeatUCRounds(t *testing.T) { ctx := context.Background() backend := newTestDiskBackendForStartup(t) leaf1 := diskStartupLeaf(1, 11) root1 := commitDiskStartupLeaves(t, ctx, backend, 1, []smtbackend.LeafInput{leaf1}) - wrongRoot := append([]byte(nil), root1...) - wrongRoot[0] ^= 0xff + leaf3 := diskStartupLeaf(3, 33) + snapshot, err := backend.CreateSnapshot(ctx) + require.NoError(t, err) + result, err := snapshot.AddLeavesClassified(ctx, []smtbackend.LeafInput{leaf3}) + require.NoError(t, err) + root3 := result.CandidateRoot + snapshot.Discard(ctx) + + block3 := diskStartupBlock(3, root3) storage := &diskStartupStorage{ - blocks: newDiskStartupBlockStorage(diskStartupBlock(1, root1), diskStartupBlock(3, wrongRoot)), - records: newDiskStartupBlockRecordsStorage(models.NewBlockRecords(api.NewBigIntFromUint64(3), nil)), + blocks: newDiskStartupBlockStorage(diskStartupBlock(1, root1), block3), + smt: newDiskStartupSMTStorage(models.NewSmtNode(leaf3.Key, leaf3.Value)), + aggregator: newDiskStartupAggregatorRecordStorage(diskStartupAggregatorRecord(block3, leaf3, 0)), } rm := newDiskStartupRoundManager(t, backend, storage) rm.config.SMT.StartupReplayLimitBlocks = 2 - _, err := rm.verifyDiskSMTStartup(ctx) - require.ErrorContains(t, err, "disk SMT replay root mismatch at block 3") + block, err := rm.verifyDiskSMTStartup(ctx) + require.NoError(t, err) + require.Equal(t, uint64(3), block.Uint64()) + + state, err := backend.CommittedState(ctx) + require.NoError(t, err) + require.Equal(t, uint64(3), state.BlockNumber.Uint64()) + require.Equal(t, root3, state.RootHash) } func TestDiskSMTStartupReplayLimitExceeded(t *testing.T) { @@ -260,17 +299,15 @@ func TestDiskSMTStartReplaysFinalizedBeforeRecoveringUnfinalizedBlock(t *testing block3 := diskStartupBlock(3, root3) block3.Finalized = false - stateID2 := api.StateID(leaf2.Key) - stateID3 := api.StateID(leaf3.Key) + block2 := diskStartupBlock(2, root2) storage := &diskStartupStorage{ blocks: newDiskStartupBlockStorage( diskStartupBlock(1, root1), - diskStartupBlock(2, root2), + block2, block3, ), - records: newDiskStartupBlockRecordsStorage(models.NewBlockRecords(api.NewBigIntFromUint64(2), []api.StateID{stateID2}), models.NewBlockRecords(api.NewBigIntFromUint64(3), []api.StateID{stateID3})), smt: newDiskStartupSMTStorage(models.NewSmtNode(leaf2.Key, leaf2.Value), models.NewSmtNode(leaf3.Key, leaf3.Value)), - aggregator: newDiskStartupAggregatorRecordStorage(stateID3), + aggregator: newDiskStartupAggregatorRecordStorage(diskStartupAggregatorRecord(block2, leaf2, 0), diskStartupAggregatorRecord(block3, leaf3, 0)), } rm := newDiskStartupRoundManager(t, backend, storage) rm.commitmentQueue = &diskStartupCommitmentQueue{} @@ -296,12 +333,10 @@ func TestDiskSMTStartFinalizesAlreadyCommittedUnfinalizedBlock(t *testing.T) { block2 := diskStartupBlock(2, root2) block2.Finalized = false - stateID2 := api.StateID(leaf2.Key) storage := &diskStartupStorage{ blocks: newDiskStartupBlockStorage(diskStartupBlock(1, root1), block2), - records: newDiskStartupBlockRecordsStorage(models.NewBlockRecords(api.NewBigIntFromUint64(2), []api.StateID{stateID2})), smt: newDiskStartupSMTStorage(models.NewSmtNode(leaf2.Key, leaf2.Value)), - aggregator: newDiskStartupAggregatorRecordStorage(stateID2), + aggregator: newDiskStartupAggregatorRecordStorage(diskStartupAggregatorRecord(block2, leaf2, 0)), } rm := newDiskStartupRoundManager(t, backend, storage) rm.commitmentQueue = &diskStartupCommitmentQueue{} @@ -324,12 +359,10 @@ func TestDiskSMTStartFinalizesAlreadyCommittedFirstBlock(t *testing.T) { block := diskStartupBlock(1, root) block.Finalized = false - stateID := api.StateID(leaf.Key) storage := &diskStartupStorage{ blocks: newDiskStartupBlockStorage(block), - records: newDiskStartupBlockRecordsStorage(models.NewBlockRecords(api.NewBigIntFromUint64(1), []api.StateID{stateID})), smt: newDiskStartupSMTStorage(models.NewSmtNode(leaf.Key, leaf.Value)), - aggregator: newDiskStartupAggregatorRecordStorage(stateID), + aggregator: newDiskStartupAggregatorRecordStorage(diskStartupAggregatorRecord(block, leaf, 0)), } rm := newDiskStartupRoundManager(t, backend, storage) rm.commitmentQueue = &diskStartupCommitmentQueue{} @@ -359,12 +392,10 @@ func TestDiskSMTStartAppliesUnfinalizedFirstBlockFromEmptyDisk(t *testing.T) { block := diskStartupBlock(1, root) block.Finalized = false - stateID := api.StateID(leaf.Key) storage := &diskStartupStorage{ blocks: newDiskStartupBlockStorage(block), - records: newDiskStartupBlockRecordsStorage(models.NewBlockRecords(api.NewBigIntFromUint64(1), []api.StateID{stateID})), smt: newDiskStartupSMTStorage(models.NewSmtNode(leaf.Key, leaf.Value)), - aggregator: newDiskStartupAggregatorRecordStorage(stateID), + aggregator: newDiskStartupAggregatorRecordStorage(diskStartupAggregatorRecord(block, leaf, 0)), } rm := newDiskStartupRoundManager(t, backend, storage) rm.commitmentQueue = &diskStartupCommitmentQueue{} @@ -387,9 +418,10 @@ func TestLoadRecoveredNodesIntoBackendRejectsRecoveredRootMismatch(t *testing.T) wrongRoot := append([]byte(nil), leaf.Value...) wrongRoot[0] ^= 0xff blockNumber := api.NewBigIntFromUint64(1) + block := diskStartupBlock(1, wrongRoot) storage := &diskStartupStorage{ - blocks: newDiskStartupBlockStorage(diskStartupBlock(1, wrongRoot)), - smt: newDiskStartupSMTStorage(models.NewSmtNode(leaf.Key, leaf.Value)), + blocks: newDiskStartupBlockStorage(block), + aggregator: newDiskStartupAggregatorRecordStorage(diskStartupAggregatorRecord(block, leaf, 0)), } rm := newDiskStartupRoundManager(t, backend, storage) @@ -475,12 +507,25 @@ func diskStartupBlock(number uint64, root []byte) *models.Block { nil, ) block.Finalized = true + block.Status = models.FinalityStatusFinalized + block.ProposalID = "proposal-" + block.Index.String() return block } +func diskStartupAggregatorRecord(block *models.Block, leaf smtbackend.LeafInput, leafIndex uint64) *models.AggregatorRecord { + return &models.AggregatorRecord{ + StateID: api.StateID(append([]byte(nil), leaf.Key...)), + CertificationData: models.CertificationData{ + TransactionHash: append([]byte(nil), leaf.Value...), + }, + BlockNumber: block.Index, + LeafIndex: api.NewBigIntFromUint64(leafIndex), + ProposalID: block.ProposalID, + } +} + type diskStartupStorage struct { blocks *diskStartupBlockStorage - records *diskStartupBlockRecordsStorage smt *diskStartupSMTStorage aggregator *diskStartupAggregatorRecordStorage } @@ -503,12 +548,6 @@ func (s *diskStartupStorage) SmtStorage() interfaces.SmtStorage { } return s.smt } -func (s *diskStartupStorage) BlockRecordsStorage() interfaces.BlockRecordsStorage { - if s.records == nil { - return newDiskStartupBlockRecordsStorage() - } - return s.records -} func (s *diskStartupStorage) LeadershipStorage() interfaces.LeadershipStorage { return nil } func (s *diskStartupStorage) TrustBaseStorage() interfaces.TrustBaseStorage { return nil } func (s *diskStartupStorage) Initialize(context.Context) error { return nil } @@ -519,8 +558,9 @@ func (s *diskStartupStorage) WithTransaction(ctx context.Context, fn func(contex } type diskStartupBlockStorage struct { - blocks map[string]*models.Block - latest *models.Block + blocks map[string]*models.Block + latest *models.Block + afterGetRange func() } func newDiskStartupBlockStorage(blocks ...*models.Block) *diskStartupBlockStorage { @@ -563,8 +603,33 @@ func (s *diskStartupBlockStorage) GetLatestByRootHash(context.Context, api.HexBy func (s *diskStartupBlockStorage) Count(context.Context) (int64, error) { return int64(len(s.blocks)), nil } -func (s *diskStartupBlockStorage) GetRange(context.Context, *api.BigInt, *api.BigInt) ([]*models.Block, error) { - return nil, nil +func (s *diskStartupBlockStorage) GetRange(_ context.Context, fromBlock, toBlock *api.BigInt) ([]*models.Block, error) { + var blocks []*models.Block + for _, block := range s.blocks { + if !block.Finalized || block.Index.Cmp(fromBlock.Int) < 0 || block.Index.Cmp(toBlock.Int) > 0 { + continue + } + blocks = append(blocks, block) + } + sort.Slice(blocks, func(i, j int) bool { + return blocks[i].Index.Cmp(blocks[j].Index.Int) < 0 + }) + if s.afterGetRange != nil { + s.afterGetRange() + } + return blocks, nil +} +func (s *diskStartupBlockStorage) GetNextFinalizedAfter(_ context.Context, afterBlock, toBlock *api.BigInt) (*models.Block, error) { + var next *models.Block + for _, block := range s.blocks { + if !block.Finalized || block.Index.Cmp(afterBlock.Int) <= 0 || block.Index.Cmp(toBlock.Int) > 0 { + continue + } + if next == nil || block.Index.Cmp(next.Index.Int) < 0 { + next = block + } + } + return next, nil } func (s *diskStartupBlockStorage) SetFinalized(_ context.Context, blockNumber *api.BigInt, finalized bool) error { block := s.blocks[blockNumber.String()] @@ -597,13 +662,10 @@ type diskStartupAggregatorRecordStorage struct { records map[string]*models.AggregatorRecord } -func newDiskStartupAggregatorRecordStorage(stateIDs ...api.StateID) *diskStartupAggregatorRecordStorage { +func newDiskStartupAggregatorRecordStorage(records ...*models.AggregatorRecord) *diskStartupAggregatorRecordStorage { out := &diskStartupAggregatorRecordStorage{records: make(map[string]*models.AggregatorRecord)} - for i, stateID := range stateIDs { - out.records[stateID.String()] = &models.AggregatorRecord{ - StateID: stateID, - LeafIndex: api.NewBigIntFromUint64(uint64(i)), - } + for _, record := range records { + out.records[record.StateID.String()] = record } return out } @@ -624,11 +686,14 @@ func (s *diskStartupAggregatorRecordStorage) GetByStateID(_ context.Context, sta return s.records[stateID.String()], nil } -func (s *diskStartupAggregatorRecordStorage) GetByBlockNumber(context.Context, *api.BigInt) ([]*models.AggregatorRecord, error) { +func (s *diskStartupAggregatorRecordStorage) GetByBlockNumber(_ context.Context, blockNumber *api.BigInt) ([]*models.AggregatorRecord, error) { records := make([]*models.AggregatorRecord, 0, len(s.records)) for _, record := range s.records { - records = append(records, record) + if record.BlockNumber != nil && record.BlockNumber.Cmp(blockNumber.Int) == 0 { + records = append(records, record) + } } + sortAggregatorRecordsByLeafIndex(records) return records, nil } @@ -646,43 +711,6 @@ func (s *diskStartupAggregatorRecordStorage) GetExistingStateIDs(_ context.Conte return existing, nil } -type diskStartupBlockRecordsStorage struct { - records map[string]*models.BlockRecords -} - -func newDiskStartupBlockRecordsStorage(records ...*models.BlockRecords) *diskStartupBlockRecordsStorage { - out := &diskStartupBlockRecordsStorage{records: make(map[string]*models.BlockRecords)} - for _, record := range records { - out.records[record.BlockNumber.String()] = record - } - return out -} - -func (s *diskStartupBlockRecordsStorage) Store(context.Context, *models.BlockRecords) error { - return nil -} -func (s *diskStartupBlockRecordsStorage) GetByBlockNumber(_ context.Context, blockNumber *api.BigInt) (*models.BlockRecords, error) { - return s.records[blockNumber.String()], nil -} -func (s *diskStartupBlockRecordsStorage) Count(context.Context) (int64, error) { - return int64(len(s.records)), nil -} -func (s *diskStartupBlockRecordsStorage) GetNextBlock(_ context.Context, blockNumber *api.BigInt) (*models.BlockRecords, error) { - var next *models.BlockRecords - for _, record := range s.records { - if blockNumber != nil && record.BlockNumber.Cmp(blockNumber.Int) <= 0 { - continue - } - if next == nil || record.BlockNumber.Cmp(next.BlockNumber.Int) < 0 { - next = record - } - } - return next, nil -} -func (s *diskStartupBlockRecordsStorage) GetLatestBlockNumber(context.Context) (*api.BigInt, error) { - return nil, nil -} - type diskStartupSMTStorage struct { nodes map[string]*models.SmtNode } @@ -796,7 +824,6 @@ func (q *diskStartupCommitmentQueue) Close(context.Context) error { var ( _ interfaces.Storage = (*diskStartupStorage)(nil) _ interfaces.BlockStorage = (*diskStartupBlockStorage)(nil) - _ interfaces.BlockRecordsStorage = (*diskStartupBlockRecordsStorage)(nil) _ interfaces.SmtStorage = (*diskStartupSMTStorage)(nil) _ interfaces.AggregatorRecordStorage = (*diskStartupAggregatorRecordStorage)(nil) _ interfaces.CommitmentQueue = (*diskStartupCommitmentQueue)(nil) diff --git a/internal/round/finalize_duplicate_test.go b/internal/round/finalize_duplicate_test.go index dbd06d5..921f054 100644 --- a/internal/round/finalize_duplicate_test.go +++ b/internal/round/finalize_duplicate_test.go @@ -38,6 +38,32 @@ func (q *failingMarkProcessedQueue) MarkProcessed(context.Context, []interfaces. return q.err } +type finalizationFailStorage struct { + interfaces.Storage + blockStorage *finalizationFailBlockStorage +} + +func (s *finalizationFailStorage) BlockStorage() interfaces.BlockStorage { + return s.blockStorage +} + +type finalizationFailBlockStorage struct { + interfaces.BlockStorage + finalizeErr error +} + +func (s *finalizationFailBlockStorage) SetFinalizedWithCertificate(context.Context, *models.Block) error { + return s.finalizeErr +} + +func (s *finalizationFailBlockStorage) SetFinalizingWithCertificate(ctx context.Context, block *models.Block) error { + return s.BlockStorage.(blockCertificateFinalizingMarker).SetFinalizingWithCertificate(ctx, block) +} + +func (s *finalizationFailBlockStorage) GetByNumberAnyFinalization(ctx context.Context, blockNumber *api.BigInt) (*models.Block, error) { + return s.BlockStorage.(blockAnyFinalizationStorage).GetByNumberAnyFinalization(ctx, blockNumber) +} + type discardCountingSnapshot struct { smtbackend.Snapshot discards int @@ -48,6 +74,44 @@ func (s *discardCountingSnapshot) Discard(ctx context.Context) { s.Snapshot.Discard(ctx) } +func storeDurableProposalForCurrentRound(t *testing.T, ctx context.Context, rm *RoundManager, block *models.Block) []*models.AggregatorRecord { + t.Helper() + require.NotNil(t, rm.currentRound) + if rm.currentRound.ProposalID == "" { + rm.currentRound.ProposalID = "proposal-" + block.Index.String() + } + block.ProposalID = rm.currentRound.ProposalID + records, err := rm.proposalRecordsForRound(rm.currentRound, block.Index) + require.NoError(t, err) + require.NoError(t, rm.storeProposedBlockAndRecords(ctx, block, records)) + return records +} + +func recordsForBlock(commitments []*models.CertificationRequest, block *models.Block) []*models.AggregatorRecord { + records := make([]*models.AggregatorRecord, len(commitments)) + for i, commitment := range commitments { + record := models.NewAggregatorRecord(commitment, block.Index, api.NewBigInt(big.NewInt(int64(i)))) + record.ProposalID = block.ProposalID + records[i] = record + } + return records +} + +func TestSameStateIDsRequiresSameMembershipNotOrder(t *testing.T) { + a := []api.StateID{ + api.RequireNewImprintV2("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + api.RequireNewImprintV2("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"), + api.RequireNewImprintV2("cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"), + } + b := []api.StateID{a[2], a[0], a[1]} + missing := []api.StateID{a[0], a[1]} + duplicate := []api.StateID{a[0], a[1], a[1]} + + require.True(t, sameStateIDs(a, b)) + require.False(t, sameStateIDs(a, missing)) + require.False(t, sameStateIDs(a, duplicate)) +} + func TestFinalizeDuplicateSuite(t *testing.T) { suite.Run(t, new(FinalizeDuplicateTestSuite)) } @@ -74,8 +138,8 @@ func (s *FinalizeDuplicateTestSuite) SetupSuite() { } } -// TestDuplicateRecovery tests that FinalizeBlock succeeds even when -// some SMT nodes and aggregator records already exist (simulating crash recovery). +// TestDuplicateRecovery tests that FinalizeBlock succeeds even when some SMT +// nodes already exist (simulating a retry after a partial local node write). func (s *FinalizeDuplicateTestSuite) Test1_DuplicateRecovery() { t := s.T() ctx, cancel := context.WithCancel(context.Background()) @@ -111,22 +175,18 @@ func (s *FinalizeDuplicateTestSuite) Test1_DuplicateRecovery() { smtCountBefore, _ := s.storage.SmtStorage().Count(ctx) recordCountBefore, _ := s.storage.AggregatorRecordStorage().Count(ctx) - // Pre-populate storage with 2 out of 5 records (simulating partial write before crash) + // Pre-populate storage with 2 out of 5 SMT nodes (simulating partial write before crash) partialLeaves := rm.currentRound.PendingLeaves[:2] preExistingNodes, err := rm.convertLeavesToNodes(partialLeaves) require.NoError(t, err) err = s.storage.SmtStorage().StoreBatch(ctx, preExistingNodes) require.NoError(t, err, "Pre-populating SMT nodes should succeed") - preExistingRecords := rm.convertCommitmentsToRecords(commitments[:2], api.NewBigInt(big.NewInt(1))) - err = s.storage.AggregatorRecordStorage().StoreBatch(ctx, preExistingRecords) - require.NoError(t, err, "Pre-populating aggregator records should succeed") - // Verify pre-existing data added smtCount, _ := s.storage.SmtStorage().Count(ctx) require.Equal(t, smtCountBefore+2, smtCount, "Should have added 2 pre-existing SMT nodes") recordCount, _ := s.storage.AggregatorRecordStorage().Count(ctx) - require.Equal(t, recordCountBefore+2, recordCount, "Should have added 2 pre-existing aggregator records") + require.Equal(t, recordCountBefore, recordCount, "Should not have pre-existing aggregator records") // Get root hash from snapshot rootHash := testSnapshotRootHex(t, ctx, rm.currentRound.Snapshot) @@ -145,6 +205,7 @@ func (s *FinalizeDuplicateTestSuite) Test1_DuplicateRecovery() { api.HexBytes{}, api.HexBytes{}, ) + storeDurableProposalForCurrentRound(t, ctx, rm, block) // FinalizeBlock should succeed despite duplicates err = rm.FinalizeBlock(ctx, block) @@ -203,6 +264,7 @@ func (s *FinalizeDuplicateTestSuite) Test2_NoDuplicates() { api.HexBytes{}, api.HexBytes{}, ) + storeDurableProposalForCurrentRound(t, ctx, rm, block) // Should succeed on first try (no duplicates) err = rm.FinalizeBlock(ctx, block) @@ -244,16 +306,13 @@ func (s *FinalizeDuplicateTestSuite) Test3_AllDuplicates() { smtCountBefore, _ := s.storage.SmtStorage().Count(ctx) recordCountBefore, _ := s.storage.AggregatorRecordStorage().Count(ctx) - // Pre-populate ALL SMT nodes and aggregator records + // Pre-populate ALL SMT nodes. Aggregator records are stored once as the + // durable proposal before certification. allNodes, err := rm.convertLeavesToNodes(rm.currentRound.PendingLeaves) require.NoError(t, err) err = s.storage.SmtStorage().StoreBatch(ctx, allNodes) require.NoError(t, err) - allRecords := rm.convertCommitmentsToRecords(commitments, api.NewBigInt(big.NewInt(3))) - err = s.storage.AggregatorRecordStorage().StoreBatch(ctx, allRecords) - require.NoError(t, err) - rootHash := testSnapshotRootHex(t, ctx, rm.currentRound.Snapshot) rootHashBytes, err := api.NewHexBytesFromString(rootHash) require.NoError(t, err) @@ -268,6 +327,7 @@ func (s *FinalizeDuplicateTestSuite) Test3_AllDuplicates() { api.HexBytes{}, api.HexBytes{}, ) + storeDurableProposalForCurrentRound(t, ctx, rm, block) // Should succeed even when all records are duplicates err = rm.FinalizeBlock(ctx, block) @@ -326,25 +386,14 @@ func (s *FinalizeDuplicateTestSuite) Test4_DuplicateBlock() { api.HexBytes{}, ) - // Pre-store the block (simulating previous attempt that stored block but failed on MarkProcessed) - block.Finalized = false - err = s.storage.BlockStorage().Store(ctx, block) - require.NoError(t, err, "Pre-storing block should succeed") - - // Also pre-store block records - stateIDs := make([]api.StateID, len(commitments)) - for i, c := range commitments { - stateIDs[i] = c.StateID - } - err = s.storage.BlockRecordsStorage().Store(ctx, models.NewBlockRecords(block.Index, stateIDs)) - require.NoError(t, err, "Pre-storing block records should succeed") + // Pre-store the durable proposal (simulating the real pre-certification flow). + storeDurableProposalForCurrentRound(t, ctx, rm, block) // Get counts before FinalizeBlock smtCountBefore, _ := s.storage.SmtStorage().Count(ctx) recordCountBefore, _ := s.storage.AggregatorRecordStorage().Count(ctx) - // FinalizeBlock should succeed despite duplicate block - // It should skip block storage and continue with remaining steps + // FinalizeBlock should use the already-stored proposal and not rewrite records. err = rm.FinalizeBlock(ctx, block) require.NoError(t, err, "FinalizeBlock should succeed with duplicate block") @@ -353,7 +402,7 @@ func (s *FinalizeDuplicateTestSuite) Test4_DuplicateBlock() { require.Equal(t, smtCountBefore+3, smtCountAfter, "Should have stored 3 SMT nodes despite duplicate block") recordCountAfter, _ := s.storage.AggregatorRecordStorage().Count(ctx) - require.Equal(t, recordCountBefore+3, recordCountAfter, "Should have stored 3 aggregator records despite duplicate block") + require.Equal(t, recordCountBefore, recordCountAfter, "FinalizeBlock should not store aggregator records") // Verify block was finalized storedBlock, err := s.storage.BlockStorage().GetByNumber(ctx, block.Index) @@ -409,6 +458,7 @@ func (s *FinalizeDuplicateTestSuite) Test4b_MarkProcessedFailureAfterFinalizatio api.HexBytes{}, api.HexBytes{}, ) + storeDurableProposalForCurrentRound(t, ctx, rm, block) err = rm.FinalizeBlock(ctx, block) require.NoError(t, err, "post-finalization ACK failure should not fail FinalizeBlock") @@ -424,6 +474,66 @@ func (s *FinalizeDuplicateTestSuite) Test4b_MarkProcessedFailureAfterFinalizatio t.Log("✓ FinalizeBlock succeeded despite post-finalization MarkProcessed failure") } +func (s *FinalizeDuplicateTestSuite) Test4c_FinalizeFailureLeavesCertifiedBlockRecoverable() { + t := s.T() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + testLogger, err := logger.New("info", "text", "stdout", false) + require.NoError(t, err) + + finalizeErr := errors.New("finalize update failed") + storage := &finalizationFailStorage{ + Storage: s.storage, + blockStorage: &finalizationFailBlockStorage{ + BlockStorage: s.storage.BlockStorage(), + finalizeErr: finalizeErr, + }, + } + threadSafeSMT := smt.NewThreadSafeSMT(smt.NewSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits)) + rm, err := NewRoundManager(ctx, s.cfg, testLogger, s.storage.CommitmentQueue(), storage, nil, state.NewSyncStateTracker(), nil, events.NewEventBus(testLogger), threadSafeSMT, nil) + require.NoError(t, err) + + commitments := testutil.CreateTestCertificationRequests(t, 3, "t4c_req") + rm.currentRound = &Round{ + Number: api.NewBigInt(big.NewInt(47)), + State: RoundStateProcessing, + Commitments: commitments, + Snapshot: testRMSnapshot(t, ctx, rm), + } + + rm.roundMutex.Lock() + _, err = rm.processMiniBatch(ctx, commitments) + rm.roundMutex.Unlock() + require.NoError(t, err) + + rootHash := testSnapshotRootHex(t, ctx, rm.currentRound.Snapshot) + rootHashBytes, err := api.NewHexBytesFromString(rootHash) + require.NoError(t, err) + certBytes := api.NewHexBytes([]byte{0x04, 0x0c}) + block := models.NewBlock( + api.NewBigInt(big.NewInt(47)), + "unicity", + 0, + "1.0", + "mainnet", + rootHashBytes, + api.HexBytes{}, + certBytes, + ) + storeDurableProposalForCurrentRound(t, ctx, rm, block) + + err = rm.FinalizeBlock(ctx, block) + require.ErrorIs(t, err, finalizeErr) + + storedBlock, err := getBlockAnyFinalization(ctx, s.storage, block.Index) + require.NoError(t, err) + require.NotNil(t, storedBlock) + require.False(t, storedBlock.Finalized) + require.Equal(t, models.FinalityStatusFinalizing, storedBlock.Status) + require.Equal(t, certBytes, storedBlock.UnicityCertificate) +} + // Test5_DuplicateBlockAlreadyFinalized tests that FinalizeBlock succeeds when // the block already exists AND is already finalized (full retry scenario). func (s *FinalizeDuplicateTestSuite) Test5_DuplicateBlockAlreadyFinalized() { @@ -469,24 +579,18 @@ func (s *FinalizeDuplicateTestSuite) Test5_DuplicateBlockAlreadyFinalized() { // Pre-store the block as FINALIZED (simulating previous successful attempt except MarkProcessed) block.Finalized = true + block.Status = models.FinalityStatusFinalized + block.ProposalID = "proposal-" + block.Index.String() err = s.storage.BlockStorage().Store(ctx, block) require.NoError(t, err, "Pre-storing finalized block should succeed") - // Pre-store block records - stateIDs := make([]api.StateID, len(commitments)) - for i, c := range commitments { - stateIDs[i] = c.StateID - } - err = s.storage.BlockRecordsStorage().Store(ctx, models.NewBlockRecords(block.Index, stateIDs)) - require.NoError(t, err, "Pre-storing block records should succeed") - // Pre-store all SMT nodes and records (simulating full previous attempt) allNodes, err := rm.convertLeavesToNodes(rm.currentRound.PendingLeaves) require.NoError(t, err) err = s.storage.SmtStorage().StoreBatch(ctx, allNodes) require.NoError(t, err) - allRecords := rm.convertCommitmentsToRecords(commitments, block.Index) + allRecords := recordsForBlock(commitments, block) err = s.storage.AggregatorRecordStorage().StoreBatch(ctx, allRecords) require.NoError(t, err) @@ -519,10 +623,10 @@ func (s *FinalizeDuplicateTestSuite) Test5_DuplicateBlockAlreadyFinalized() { t.Log("✓ FinalizeBlock succeeded with already-finalized block") } -// Test6_BlockRecordsMatchPendingCommitmentsOnConflict verifies that FinalizeBlock -// stores BlockRecords/SMT nodes/aggregator records from the filtered pending set, -// not from all round commitments when one commitment conflicts. -func (s *FinalizeDuplicateTestSuite) Test6_BlockRecordsMatchPendingCommitmentsOnConflict() { +// Test6_ProposalRecordsMatchPendingCommitmentsOnConflict verifies that the +// durable proposal is built from the filtered pending set, not from all round +// commitments when one commitment conflicts. +func (s *FinalizeDuplicateTestSuite) Test6_ProposalRecordsMatchPendingCommitmentsOnConflict() { t := s.T() ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -584,18 +688,15 @@ func (s *FinalizeDuplicateTestSuite) Test6_BlockRecordsMatchPendingCommitmentsOn api.HexBytes{}, api.HexBytes{}, ) + storeDurableProposalForCurrentRound(t, ctx, rm, block) err = rm.FinalizeBlock(ctx, block) require.NoError(t, err) - blockRecords, err := s.storage.BlockRecordsStorage().GetByBlockNumber(ctx, block.Index) - require.NoError(t, err) - require.NotNil(t, blockRecords) - require.Equal(t, []api.StateID{commitment1.StateID, commitment2.StateID}, blockRecords.StateIDs) - recordsByBlock, err := s.storage.AggregatorRecordStorage().GetByBlockNumber(ctx, block.Index) require.NoError(t, err) require.Len(t, recordsByBlock, 2) + require.Equal(t, []api.StateID{commitment1.StateID, commitment2.StateID}, []api.StateID{recordsByBlock[0].StateID, recordsByBlock[1].StateID}) smtCountAfter, err := s.storage.SmtStorage().Count(ctx) require.NoError(t, err) @@ -603,7 +704,7 @@ func (s *FinalizeDuplicateTestSuite) Test6_BlockRecordsMatchPendingCommitmentsOn recordCountAfter, err := s.storage.AggregatorRecordStorage().Count(ctx) require.NoError(t, err) - require.Equal(t, recordCountBefore+2, recordCountAfter, "should persist only filtered aggregator records") + require.Equal(t, recordCountBefore+2, recordCountAfter, "should stage only filtered aggregator records") } func (s *FinalizeDuplicateTestSuite) Test7_FinalizeBlockRejectsRoundNumberMismatch() { @@ -669,12 +770,14 @@ func (s *FinalizeDuplicateTestSuite) Test8_DuplicateBlockMustMatchRootAndStateID existing := models.NewBlock(api.NewBigInt(big.NewInt(8)), "unicity", 0, "1.0", "mainnet", api.HexBytes(repeatByte(32, 7)), api.HexBytes{}, api.HexBytes{}) existing.Finalized = true + existing.Status = models.FinalityStatusFinalized + existing.ProposalID = "proposal-8-existing" require.NoError(t, s.storage.BlockStorage().Store(ctx, existing)) - require.NoError(t, s.storage.BlockRecordsStorage().Store(ctx, models.NewBlockRecords(existing.Index, []api.StateID{commitments[0].StateID}))) + require.NoError(t, s.storage.AggregatorRecordStorage().StoreBatch(ctx, recordsForBlock(commitments[:1], existing))) block := models.NewBlock(api.NewBigInt(big.NewInt(8)), "unicity", 0, "1.0", "mainnet", rootHashBytes, api.HexBytes{}, api.HexBytes{}) err = rm.FinalizeBlock(ctx, block) - require.ErrorContains(t, err, "duplicate block") + require.ErrorContains(t, err, "root mismatch") } func (s *FinalizeDuplicateTestSuite) Test9_EmptyRoundCannotFinalizeChangedRoot() { @@ -703,6 +806,17 @@ func (s *FinalizeDuplicateTestSuite) Test9_EmptyRoundCannotFinalizeChangedRoot() require.ErrorContains(t, err, "snapshot root") } +func (s *FinalizeDuplicateTestSuite) Test10_FinalizeBlockWithRetryPropagatesCancellation() { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + block := models.NewBlock(api.NewBigInt(big.NewInt(10)), "unicity", 0, "1.0", "mainnet", api.HexBytes{}, api.HexBytes{}, api.HexBytes{}) + rm := &RoundManager{} + + err := rm.FinalizeBlockWithRetry(ctx, block) + require.ErrorIs(s.T(), err, context.Canceled) +} + func repeatByte(n int, value byte) []byte { out := make([]byte, n) for i := range out { diff --git a/internal/round/parent_round_manager.go b/internal/round/parent_round_manager.go index 612df9b..d43c7f3 100644 --- a/internal/round/parent_round_manager.go +++ b/internal/round/parent_round_manager.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/binary" + "errors" "fmt" "math/big" "sync" @@ -59,6 +60,9 @@ type ParentRoundManager struct { roundMutex sync.RWMutex stopChan chan struct{} wg sync.WaitGroup + roundWG sync.WaitGroup + activeCtx context.Context + activeCancel context.CancelFunc // Metrics totalRounds int64 @@ -185,12 +189,33 @@ func (prm *ParentRoundManager) StartNewRound(ctx context.Context, roundNumber *a // StartNextRoundFromPrecollector exists to satisfy the BFT RoundManager // interface. Parent mode keeps its existing collect behavior. func (prm *ParentRoundManager) StartNextRoundFromPrecollector(ctx context.Context, roundNumber *api.BigInt) error { - return prm.StartNewRound(ctx, roundNumber) + prm.roundMutex.Lock() + activeCtx := prm.activeCtx + if activeCtx == nil { + prm.roundMutex.Unlock() + return ErrDeactivated + } + + prm.roundWG.Add(1) + prm.roundMutex.Unlock() + go func() { + defer prm.roundWG.Done() + if err := prm.StartNewRound(activeCtx, roundNumber); err != nil && !errors.Is(err, ErrDeactivated) && !errors.Is(err, context.Canceled) { + prm.logger.WithContext(ctx).Error("Failed to start next parent round", + "roundNumber", roundNumber.String(), + "error", err.Error()) + } + }() + return nil } // startNewRound is the internal implementation func (prm *ParentRoundManager) startNewRound(ctx context.Context, roundNumber *api.BigInt) error { prm.roundMutex.Lock() + if prm.activeCtx == nil { + prm.roundMutex.Unlock() + return ErrDeactivated + } // Check if this round or a later one is already in progress if prm.currentRound != nil && prm.currentRound.Number.Cmp(roundNumber.Int) >= 0 { @@ -248,6 +273,9 @@ func (prm *ParentRoundManager) processCurrentRound(ctx context.Context) error { if prm.currentRound == nil { prm.roundMutex.Unlock() + if ctx.Err() != nil { + return ctx.Err() + } return fmt.Errorf("no current round to process") } @@ -403,6 +431,17 @@ func (prm *ParentRoundManager) GetSMTBackend() smtbackend.Backend { return prm.smtBackend } +func (prm *ParentRoundManager) CommittedRoot(ctx context.Context) ([]byte, *api.BigInt, error) { + if prm.smtBackend == nil { + return nil, nil, fmt.Errorf("SMT backend is not initialized") + } + state, err := prm.smtBackend.CommittedState(ctx) + if err != nil { + return nil, nil, err + } + return state.RootHash, state.BlockNumber, nil +} + // IsReady reports whether the parent round manager is ready to accept shard roots. func (prm *ParentRoundManager) IsReady() bool { return prm.ready.Load() @@ -434,6 +473,34 @@ func (prm *ParentRoundManager) Activate(ctx context.Context) error { prm.logger.WithContext(ctx).Info("Activating parent round manager") prm.ready.Store(false) + prm.roundMutex.RLock() + alreadyActive := prm.activeCancel != nil + prm.roundMutex.RUnlock() + if alreadyActive { + prm.logger.WithContext(ctx).Warn("Parent round manager already active, deactivating previous activation") + if err := prm.Deactivate(ctx); err != nil { + return fmt.Errorf("failed to deactivate previous activation: %w", err) + } + } + + activeCtx, activeCancel := context.WithCancel(ctx) + activated := false + defer func() { + if activated { + return + } + activeCancel() + if err := prm.Deactivate(ctx); err != nil { + prm.logger.WithContext(ctx).Error("Failed to deactivate after activation failure", + "error", err.Error()) + } + }() + + prm.roundMutex.Lock() + prm.activeCtx = activeCtx + prm.activeCancel = activeCancel + prm.roundMutex.Unlock() + // Reconstruct parent SMT from current shard states in storage // This ensures the follower-turned-leader has the latest state prm.logger.WithContext(ctx).Info("Reconstructing parent SMT from storage on leadership transition") @@ -441,16 +508,6 @@ func (prm *ParentRoundManager) Activate(ctx context.Context) error { return fmt.Errorf("failed to reconstruct parent SMT on activation: %w", err) } - // Start BFT client - if err := prm.bftClient.Start(ctx); err != nil { - return fmt.Errorf("failed to start BFT client: %w", err) - } - - // Wait for BFT client to receive first UC from root chain before starting rounds. - if err := prm.bftClient.WaitForInitialized(ctx); err != nil { - return fmt.Errorf("failed to wait for BFT client initialization: %w", err) - } - // Get latest block number to determine starting round latestBlockNumber, err := prm.storage.BlockStorage().GetLatestNumber(ctx) if err != nil { @@ -468,8 +525,20 @@ func (prm *ParentRoundManager) Activate(ctx context.Context) error { nextRoundNumber.Add(nextRoundNumber.Int, big.NewInt(1)) } + // Start BFT client after the initial round number is known. The real BFT + // client only initializes network state; the local stub would otherwise + // start its own round and race the explicit parent round start below. + if prm.config.BFT.Enabled { + if err := prm.bftClient.Start(activeCtx); err != nil { + return fmt.Errorf("failed to start BFT client: %w", err) + } + if err := prm.bftClient.WaitForInitialized(activeCtx); err != nil { + return fmt.Errorf("failed to wait for BFT client initialization: %w", err) + } + } + // Start first round - if err := prm.startNewRound(ctx, nextRoundNumber); err != nil { + if err := prm.startNewRound(activeCtx, nextRoundNumber); err != nil { return fmt.Errorf("failed to start initial round: %w", err) } @@ -477,6 +546,7 @@ func (prm *ParentRoundManager) Activate(ctx context.Context) error { "initialRound", nextRoundNumber.String()) prm.ready.Store(true) + activated = true return nil } @@ -485,13 +555,20 @@ func (prm *ParentRoundManager) Deactivate(ctx context.Context) error { prm.logger.WithContext(ctx).Info("Deactivating parent round manager") prm.ready.Store(false) - // Stop BFT client - prm.bftClient.Stop() - prm.roundMutex.Lock() + activeCancel := prm.activeCancel + prm.activeCancel = nil + prm.activeCtx = nil prm.currentRound = nil prm.roundMutex.Unlock() + if activeCancel != nil { + activeCancel() + } + // Stop BFT client + prm.bftClient.Stop() + prm.roundWG.Wait() + prm.logger.WithContext(ctx).Info("Parent round manager deactivated successfully") return nil } @@ -617,7 +694,7 @@ func (prm *ParentRoundManager) FinalizeBlock(ctx context.Context, block *models. "bftWait", shortDur(bftWait), "finalization", shortDur(finalizationTime), } - prm.logger.WithContext(ctx).Info("PERF: Parent round completed", logFields...) + prm.logger.WithContext(ctx).Info("Parent round completed", logFields...) prm.logger.WithContext(ctx).Info("Parent block finalized successfully", "blockNumber", block.Index.String()) diff --git a/internal/round/parent_round_manager_test.go b/internal/round/parent_round_manager_test.go index a7e9f51..e4ba6f2 100644 --- a/internal/round/parent_round_manager_test.go +++ b/internal/round/parent_round_manager_test.go @@ -459,6 +459,52 @@ func (suite *ParentRoundManagerTestSuite) TestBlockRootMatchesSMTRoot() { suite.Assert().Equal(expectedRoot, latestBlock.RootHash, "stored block root should match SMT root") } +func (suite *ParentRoundManagerTestSuite) TestReactivateDeactivatesPreviousActivation() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + cfg := *suite.cfg + cfg.Sharding.ParentCollectPhaseDuration = 0 + prm, err := NewParentRoundManager(ctx, &cfg, suite.logger, suite.storage, nil, suite.eventBus, smt.NewThreadSafeSMT(smt.NewParentSparseMerkleTree(api.SHA256, cfg.Sharding.ShardIDLength)), suite.storage.TrustBaseStorage()) + suite.Require().NoError(err) + defer prm.Stop(ctx) + suite.Require().NoError(prm.Start(ctx)) + + oldCtx, oldCancel := context.WithCancel(context.Background()) + oldRound := &ParentRound{ + Number: api.NewBigIntFromUint64(99), + State: RoundStateCollecting, + ShardUpdates: make(map[int]*models.ShardRootUpdate), + } + oldDone := make(chan struct{}) + + prm.roundMutex.Lock() + prm.activeCtx = oldCtx + prm.activeCancel = oldCancel + prm.currentRound = oldRound + prm.roundMutex.Unlock() + + prm.roundWG.Add(1) + go func() { + defer prm.roundWG.Done() + defer close(oldDone) + <-oldCtx.Done() + }() + + suite.Require().NoError(prm.Activate(ctx)) + + select { + case <-oldDone: + default: + suite.Fail("reactivation returned before the previous active round stopped") + } + + currentRound, ok := prm.GetCurrentRound().(*ParentRound) + suite.Require().True(ok) + suite.Require().NotNil(currentRound) + suite.Require().NotEqual(oldRound.Number.String(), currentRound.Number.String(), "reactivation must clear stale parent round state") +} + // TestParentRoundManagerSuite runs the test suite func TestParentRoundManagerSuite(t *testing.T) { suite.Run(t, new(ParentRoundManagerTestSuite)) diff --git a/internal/round/precollection_test.go b/internal/round/precollection_test.go index 1132797..5c853e6 100644 --- a/internal/round/precollection_test.go +++ b/internal/round/precollection_test.go @@ -277,7 +277,7 @@ func newTestPrecollector(t *testing.T, stream chan *models.CertificationRequest, if maxPerRound <= 0 { maxPerRound = 10000 } - cp := newChildPrecollector(stream, nil, log, maxPerRound, nil) + cp := newChildPrecollector(stream, nil, log, maxPerRound, nil, nil, "", nil) return cp, smtInstance } @@ -432,31 +432,6 @@ func TestReconcileRecoveredFinalization_MismatchedBlockClearsProofPendingOnly(t require.False(t, pendingAfter) } -func TestDrainBufferedCommitments_StopsAtRoundBoundary(t *testing.T) { - stream := make(chan *models.CertificationRequest, 8) - pending := make([]*models.CertificationRequest, 0, miniBatchSize) - collected := make([]*models.CertificationRequest, 0, 5) - count := 0 - maxPerRound := 5 - - flush := func() error { - collected = append(collected, pending...) - count += len(pending) - pending = pending[:0] - return nil - } - - for i := 0; i < maxPerRound+2; i++ { - stream <- testutil.CreateTestCertificationRequest(t, "drain_boundary") - } - - require.NoError(t, drainBufferedCommitments(stream, maxPerRound, &count, &pending, flush)) - require.NoError(t, flush()) - - require.Len(t, collected, maxPerRound) - require.Len(t, stream, 2) -} - // --- Tests for childPrecollector --- func TestChildPrecollector_CollectsContinuouslyAcrossRound(t *testing.T) { @@ -509,8 +484,8 @@ func TestChildPrecollector_AdvanceRound_FlushesPendingBatch(t *testing.T) { cp.Start(ctx, baseSnapshot) defer cp.Stop() - // Send fewer than miniBatchSize commitments (pending but not flushed) - count := miniBatchSize - 1 + // Send fewer than defaultMiniBatchSize commitments (pending but not flushed) + count := defaultMiniBatchSize - 1 for i := 0; i < count; i++ { stream <- testutil.CreateTestCertificationRequest(t, "pending_flush") } @@ -522,6 +497,67 @@ func TestChildPrecollector_AdvanceRound_FlushesPendingBatch(t *testing.T) { assert.Len(t, result.leaves, count) } +func TestChildPrecollector_AdvanceRoundStagesOneShotHandoff(t *testing.T) { + stream := make(chan *models.CertificationRequest, 200) + cp, smtInstance := newTestPrecollector(t, stream, 10000) + cp.blockNumber = api.NewBigIntFromUint64(12) + cp.proposalID = "proposal-12" + + var stagedMu sync.Mutex + var stagedBlocks []string + var stagedProposalIDs []string + var stagedOffsets []int + var stagedCounts []int + cp.stageProposed = func(_ context.Context, blockNumber *api.BigInt, proposalID string, leafIndexOffset int, commitments []*models.CertificationRequest) error { + stagedMu.Lock() + defer stagedMu.Unlock() + stagedBlocks = append(stagedBlocks, blockNumber.String()) + stagedProposalIDs = append(stagedProposalIDs, proposalID) + stagedOffsets = append(stagedOffsets, leafIndexOffset) + stagedCounts = append(stagedCounts, len(commitments)) + return nil + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + baseSnapshot := testBackendSnapshot(t, ctx, testMemoryBackend(smtInstance)) + cp.Start(ctx, baseSnapshot) + defer cp.Stop() + + for i := 0; i < 5; i++ { + stream <- testutil.CreateTestCertificationRequest(t, "one_shot_stage") + } + time.Sleep(50 * time.Millisecond) + + result, err := cp.AdvanceRound() + require.NoError(t, err) + require.True(t, result.recordsStaged) + require.EqualValues(t, 12, result.blockNumber.Uint64()) + require.Len(t, result.commitments, 5) + require.Len(t, result.leaves, 5) + + stagedMu.Lock() + defer stagedMu.Unlock() + require.Equal(t, []string{"12"}, stagedBlocks) + require.Equal(t, []string{"proposal-12"}, stagedProposalIDs) + require.Equal(t, []int{0}, stagedOffsets) + require.Equal(t, []int{5}, stagedCounts) +} + +func TestValidatePrecollectorBlockNumberRejectsStagedMismatch(t *testing.T) { + preResult := &preCollectionResult{ + recordsStaged: true, + blockNumber: api.NewBigIntFromUint64(12), + } + + require.NoError(t, validatePrecollectorBlockNumber(preResult, api.NewBigIntFromUint64(12))) + require.ErrorContains(t, validatePrecollectorBlockNumber(preResult, api.NewBigIntFromUint64(13)), "cannot start round 13") + + preResult.recordsStaged = false + require.NoError(t, validatePrecollectorBlockNumber(preResult, api.NewBigIntFromUint64(13))) +} + func TestChildPrecollector_AdvanceRound_NoDropAcrossBoundary(t *testing.T) { stream := make(chan *models.CertificationRequest, 200) cp, smtInstance := newTestPrecollector(t, stream, 10000) @@ -708,7 +744,7 @@ func TestChildPrecollector_AdvanceRoundFailsOnSnapshotAddError(t *testing.T) { addErr := errors.New("snapshot hash mismatch") childSnapshot := &precollectorErrorSnapshot{addErr: addErr} baseSnapshot := &precollectorErrorSnapshot{forked: childSnapshot} - cp := newChildPrecollector(stream, nil, testLogger, 10000, nil) + cp := newChildPrecollector(stream, nil, testLogger, 10000, nil, nil, "", nil) t.Cleanup(cp.Stop) ctx, cancel := context.WithCancel(context.Background()) @@ -718,6 +754,7 @@ func TestChildPrecollector_AdvanceRoundFailsOnSnapshotAddError(t *testing.T) { require.NoError(t, err) cp.Start(ctx, forked) stream <- testutil.CreateTestCertificationRequest(t, "snapshot_add_error") + time.Sleep(50 * time.Millisecond) result, err := cp.AdvanceRound() require.Error(t, err) @@ -726,8 +763,10 @@ func TestChildPrecollector_AdvanceRoundFailsOnSnapshotAddError(t *testing.T) { } type precollectorErrorSnapshot struct { - forked smtbackend.Snapshot - addErr error + forked smtbackend.Snapshot + addErr error + setCommitTargetErr error + discardCount int } func (s *precollectorErrorSnapshot) AddLeavesClassified(context.Context, []smtbackend.LeafInput) (smtbackend.BatchApplyResult, error) { @@ -749,14 +788,16 @@ func (s *precollectorErrorSnapshot) Fork(context.Context) (smtbackend.Snapshot, } func (s *precollectorErrorSnapshot) SetCommitTarget(context.Context, smtbackend.Backend) error { - return nil + return s.setCommitTargetErr } func (s *precollectorErrorSnapshot) Commit(context.Context, smtbackend.CommitMetadata) error { return nil } -func (s *precollectorErrorSnapshot) Discard(context.Context) {} +func (s *precollectorErrorSnapshot) Discard(context.Context) { + s.discardCount++ +} // --- Snapshot reparenting test (still valid with precollector) --- @@ -782,7 +823,7 @@ func TestPreCollectionReparenting(t *testing.T) { // Start precollector from Round N's snapshot stream := make(chan *models.CertificationRequest, 100) - cp := newChildPrecollector(stream, nil, testLogger, 10000, nil) + cp := newChildPrecollector(stream, nil, testLogger, 10000, nil, nil, "", nil) forked, err := roundNSnapshot.Fork(ctx) require.NoError(t, err) cp.Start(ctx, forked) @@ -900,6 +941,74 @@ func TestChildPrecollector_DeactivateDuringInFlightRound(t *testing.T) { rm.wg.Wait() } +func TestChildRound_ReactivateCancelsInFlightRound(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + cfg := config.Config{ + Database: config.DatabaseConfig{ + Database: "test_child_reactivate_inflight", + }, + Processing: config.ProcessingConfig{ + RoundDuration: 100 * time.Millisecond, + MaxCommitmentsPerRound: 1000, + }, + Sharding: config.ShardingConfig{ + Mode: config.ShardingModeChild, + ShardIDLength: 1, + Child: config.ChildConfig{ + ShardID: 0b11, + ParentPollTimeout: 5 * time.Second, + ParentPollInterval: 10 * time.Millisecond, + }, + }, + } + store := testutil.SetupTestStorage(t, cfg) + + testLogger := newTestLogger(t) + rootClient := newBlockingProofRootAggregatorClient() + smtInstance := smt.NewThreadSafeSMT(smt.NewSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits)) + + rm, err := NewRoundManager( + ctx, + &cfg, + testLogger, + store.CommitmentQueue(), + store, + rootClient, + state.NewSyncStateTracker(), + nil, + events.NewEventBus(testLogger), + smtInstance, + nil, + ) + require.NoError(t, err) + require.NoError(t, rm.Start(ctx)) + require.NoError(t, rm.Activate(ctx)) + + select { + case <-rootClient.proofPollStarted: + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for round 1 to enter proof polling") + } + + rm.roundMutex.RLock() + oldRound := rm.currentRound + rm.roundMutex.RUnlock() + require.NotNil(t, oldRound) + + require.NoError(t, rm.Activate(ctx)) + + rm.roundMutex.RLock() + newRound := rm.currentRound + rm.roundMutex.RUnlock() + require.NotNil(t, newRound) + require.NotSame(t, oldRound, newRound, "reactivation must not keep the stale in-flight round") + + require.NoError(t, rm.Deactivate(ctx)) + rm.wg.Wait() +} + func TestChildRound_ParentProofTimeoutIsRetriable(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -1019,7 +1128,7 @@ func TestStartNewRoundWithSnapshot(t *testing.T) { preLeaves := []smtbackend.LeafInput{testLeafInputFromLegacyLeaf(t, leaf)} startTime := time.Now() - err = rm.StartNewRoundWithSnapshot(ctx, api.NewBigInt(big.NewInt(1)), preSnapshot, preCommitments, preLeaves) + err = rm.StartNewRoundWithSnapshot(ctx, api.NewBigInt(big.NewInt(1)), preSnapshot, preCommitments, preLeaves, false, "") require.NoError(t, err) rm.roundMutex.RLock() @@ -1086,6 +1195,9 @@ func TestStandalonePrecollectorGraceIncludesLateCommitment(t *testing.T) { rm.logger, rm.config.Processing.MaxCommitmentsPerRound, rm.markProofsPending, + nil, + "", + nil, ) rm.roundMutex.Lock() rm.precollectorDisabled = false @@ -1118,6 +1230,133 @@ func TestStandalonePrecollectorGraceIncludesLateCommitment(t *testing.T) { assert.Equal(t, lateCommitment.StateID, commitments[0].StateID) } +func TestUsesActivePrecollectorDoesNotDependOnGrace(t *testing.T) { + cfg := config.Config{ + Processing: config.ProcessingConfig{PrecollectorGracePeriod: 0}, + Storage: config.StorageConfig{UseRedisForCommitments: true}, + Sharding: config.ShardingConfig{Mode: config.ShardingModeBFTShard}, + } + rm := &RoundManager{config: &cfg} + require.True(t, rm.usesActivePrecollector()) + + cfg.Processing.PrecollectorGracePeriod = 150 * time.Millisecond + require.True(t, rm.usesActivePrecollector()) + + cfg.Storage.UseRedisForCommitments = false + require.False(t, rm.usesActivePrecollector()) +} + +// TestStartNextRoundFromPrecollectorDiscardsFailedPrecollector guards the merge +// fix at round_manager.go: when a precollector handoff fails and we fall back to +// StartNewRound, the dead precollector must be discarded. Otherwise StartNewRound +// (which only discards when abandoning a pending round) leaves rm.precollector set, +// and startActivePrecollectorIfNeeded then refuses to start a fresh one. +func TestStartNextRoundFromPrecollectorDiscardsFailedPrecollector(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + cfg := config.Config{ + Database: config.DatabaseConfig{Database: "test_precollector_handoff_failure_discard"}, + Processing: config.ProcessingConfig{ + RoundDuration: 100 * time.Millisecond, + PrecollectorGracePeriod: 150 * time.Millisecond, + MaxCommitmentsPerRound: 1000, + }, + Storage: config.StorageConfig{UseRedisForCommitments: true}, + Sharding: config.ShardingConfig{Mode: config.ShardingModeStandalone}, + BFT: config.BFTConfig{ + Enabled: false, + StubDelay: 5 * time.Second, + }, + } + storage := testutil.SetupTestStorage(t, cfg) + testLogger := newTestLogger(t) + smtInstance := smt.NewThreadSafeSMT(smt.NewSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits)) + + rm, err := NewRoundManager( + ctx, + &cfg, + testLogger, + storage.CommitmentQueue(), + storage, + nil, + state.NewSyncStateTracker(), + nil, + events.NewEventBus(testLogger), + smtInstance, + nil, + ) + require.NoError(t, err) + t.Cleanup(func() { + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 2*time.Second) + defer shutdownCancel() + _ = rm.Stop(shutdownCtx) + }) + + stageErr := errors.New("stage boom") + cp := newChildPrecollector( + rm.commitmentStream, + rm.commitmentQueue, + rm.logger, + rm.config.Processing.MaxCommitmentsPerRound, + rm.markProofsPending, + api.NewBigInt(big.NewInt(2)), + "failed-handoff-proposal", + func(context.Context, *api.BigInt, string, int, []*models.CertificationRequest) error { + return stageErr + }, + ) + rm.roundMutex.Lock() + rm.precollectorDisabled = false + rm.precollectorDone = make(chan struct{}) + rm.precollector = cp + rm.roundMutex.Unlock() + cp.Start(ctx, testBackendSnapshot(t, ctx, testMemoryBackend(smtInstance))) + + // One commitment so the handoff has records to stage (and fail on); the grace + // period gives the collect loop time to pick it up before AdvanceRound. + rm.commitmentStream <- testutil.CreateTestCertificationRequest(t, "handoff_failure") + + require.NoError(t, rm.StartNextRoundFromPrecollector(ctx, api.NewBigInt(big.NewInt(2)))) + + rm.roundMutex.RLock() + stillBlocking := rm.precollector == cp + rm.roundMutex.RUnlock() + require.False(t, stillBlocking, + "failed precollector must be discarded on handoff fallback, not left blocking new precollectors") +} + +func TestStartNextRoundFromPrecollectorDiscardsSnapshotOnSetCommitTargetError(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + setTargetErr := errors.New("set target failed") + nextSnapshot := &precollectorErrorSnapshot{} + handoffSnapshot := &precollectorErrorSnapshot{ + forked: nextSnapshot, + setCommitTargetErr: setTargetErr, + } + cp := newChildPrecollector(nil, nil, newTestLogger(t), 10000, nil, nil, "", nil) + cp.Start(ctx, handoffSnapshot) + t.Cleanup(cp.Stop) + + rm := &RoundManager{ + config: &config.Config{ + Processing: config.ProcessingConfig{PrecollectorGracePeriod: 0}, + Storage: config.StorageConfig{UseRedisForCommitments: true}, + Sharding: config.ShardingConfig{Mode: config.ShardingModeStandalone}, + }, + logger: newTestLogger(t), + } + rm.precollector = cp + + err := rm.StartNextRoundFromPrecollector(ctx, api.NewBigIntFromUint64(2)) + + require.ErrorIs(t, err, setTargetErr) + require.Equal(t, 1, handoffSnapshot.discardCount) + require.Equal(t, 0, nextSnapshot.discardCount, "precollector still owns the next snapshot until Stop") +} + func TestStandaloneActivePrecollectorLifecycle(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/internal/round/precollector.go b/internal/round/precollector.go index c904833..58b4afc 100644 --- a/internal/round/precollector.go +++ b/internal/round/precollector.go @@ -3,18 +3,25 @@ package round import ( "context" "fmt" + "math/big" "sync" + "time" + "github.com/google/uuid" "github.com/unicitynetwork/aggregator-go/internal/logger" "github.com/unicitynetwork/aggregator-go/internal/models" smtbackend "github.com/unicitynetwork/aggregator-go/internal/smt/backend" "github.com/unicitynetwork/aggregator-go/internal/storage/interfaces" + "github.com/unicitynetwork/aggregator-go/pkg/api" ) type preCollectionResult struct { - snapshot smtbackend.Snapshot - commitments []*models.CertificationRequest - leaves []smtbackend.LeafInput + snapshot smtbackend.Snapshot + commitments []*models.CertificationRequest + leaves []smtbackend.LeafInput + recordsStaged bool + blockNumber *api.BigInt + proposalID string } type advanceRequest struct { @@ -26,12 +33,35 @@ type advanceResponse struct { err error } +type precollectorPrepareStats struct { + flushCalls int + flushAdded int + flushTotal time.Duration + flushMax time.Duration + stageCalls int + stageAdded int + stageTotal time.Duration + stageMax time.Duration + fork time.Duration + total time.Duration +} + +type precollectorPrepareOutcome struct { + result *preCollectionResult + nextSnapshot smtbackend.Snapshot + stats precollectorPrepareStats + err error +} + type childPrecollector struct { commitmentStream <-chan *models.CertificationRequest commitmentQueue interfaces.CommitmentQueue logger *logger.Logger maxPerRound int markProofPending func([]*models.CertificationRequest) + blockNumber *api.BigInt + proposalID string + stageProposed func(context.Context, *api.BigInt, string, int, []*models.CertificationRequest) error advanceCh chan advanceRequest stopCh chan struct{} @@ -46,16 +76,25 @@ func newChildPrecollector( log *logger.Logger, maxPerRound int, markProofPending func([]*models.CertificationRequest), + blockNumber *api.BigInt, + proposalID string, + stageProposed func(context.Context, *api.BigInt, string, int, []*models.CertificationRequest) error, ) *childPrecollector { if maxPerRound <= 0 { maxPerRound = 10000 } + if proposalID == "" { + proposalID = uuid.NewString() + } return &childPrecollector{ commitmentStream: stream, commitmentQueue: queue, logger: log, maxPerRound: maxPerRound, markProofPending: markProofPending, + blockNumber: cloneBigInt(blockNumber), + proposalID: proposalID, + stageProposed: stageProposed, advanceCh: make(chan advanceRequest), stopCh: make(chan struct{}), doneCh: make(chan struct{}), @@ -104,39 +143,120 @@ func (cp *childPrecollector) Stop() { func (cp *childPrecollector) run(ctx context.Context, snapshot smtbackend.Snapshot) { defer close(cp.doneCh) + commitments := make([]*models.CertificationRequest, 0) + defer func() { if snapshot != nil { snapshot.Discard(ctx) } }() - commitments := make([]*models.CertificationRequest, 0) - leaves := make([]smtbackend.LeafInput, 0) - pending := make([]*models.CertificationRequest, 0, miniBatchSize) - count := 0 - flush := func() error { - if len(pending) == 0 { - return nil + prepareRound := func( + snapshot smtbackend.Snapshot, + blockNumber *api.BigInt, + proposalID string, + rawCommitments []*models.CertificationRequest, + ) precollectorPrepareOutcome { + prepareStart := time.Now() + outcome := precollectorPrepareOutcome{} + + var added []*models.CertificationRequest + var addedLeaves []smtbackend.LeafInput + if len(rawCommitments) > 0 { + start := time.Now() + var err error + added, addedLeaves, err = cp.addBatch(ctx, snapshot, rawCommitments) + elapsed := time.Since(start) + outcome.stats.flushCalls = 1 + outcome.stats.flushAdded = len(added) + outcome.stats.flushTotal = elapsed + outcome.stats.flushMax = elapsed + if err != nil { + outcome.err = err + outcome.stats.total = time.Since(prepareStart) + return outcome + } } - added, addedLeaves, err := cp.addBatch(ctx, snapshot, pending) - if err != nil { - return err + + stageDone := make(chan error, 1) + if len(added) > 0 && cp.stageProposed != nil && blockNumber != nil { + outcome.stats.stageCalls = 1 + outcome.stats.stageAdded = len(added) + go func() { + stageStart := time.Now() + err := cp.stageProposed(ctx, blockNumber, proposalID, 0, added) + stageElapsed := time.Since(stageStart) + outcome.stats.stageTotal = stageElapsed + outcome.stats.stageMax = stageElapsed + stageDone <- err + }() + } else { + stageDone <- nil } + if cp.markProofPending != nil { cp.markProofPending(added) } - commitments = append(commitments, added...) - leaves = append(leaves, addedLeaves...) - count += len(added) - pending = pending[:0] - return nil + + forkStart := time.Now() + nextSnapshot, forkErr := snapshot.Fork(ctx) + outcome.stats.fork = time.Since(forkStart) + stageErr := <-stageDone + if stageErr != nil { + if nextSnapshot != nil { + nextSnapshot.Discard(ctx) + } + outcome.err = stageErr + outcome.stats.total = time.Since(prepareStart) + return outcome + } + if forkErr != nil { + outcome.err = fmt.Errorf("failed to fork next precollector snapshot: %w", forkErr) + outcome.stats.total = time.Since(prepareStart) + return outcome + } + + outcome.result = &preCollectionResult{ + snapshot: snapshot, + commitments: added, + leaves: addedLeaves, + recordsStaged: len(added) > 0 && cp.stageProposed != nil && blockNumber != nil, + blockNumber: cloneBigInt(blockNumber), + proposalID: proposalID, + } + outcome.nextSnapshot = nextSnapshot + outcome.stats.total = time.Since(prepareStart) + return outcome + } + + prepareSynchronously := func() (precollectorPrepareOutcome, error) { + outcome := prepareRound(snapshot, cloneBigInt(cp.blockNumber), cp.proposalID, commitments) + if outcome.err != nil { + return outcome, outcome.err + } + cp.logger.WithContext(ctx).Debug("Precollector prepared", + "commitments", len(outcome.result.commitments), + "leaves", len(outcome.result.leaves), + "maxPerRound", cp.maxPerRound, + "flushCalls", outcome.stats.flushCalls, + "flushAdded", outcome.stats.flushAdded, + "flushTotal", outcome.stats.flushTotal.String(), + "flushMax", outcome.stats.flushMax.String(), + "stageCalls", outcome.stats.stageCalls, + "stageAdded", outcome.stats.stageAdded, + "stageTotal", outcome.stats.stageTotal.String(), + "stageMax", outcome.stats.stageMax.String(), + "fork", outcome.stats.fork.String(), + "total", outcome.stats.total.String()) + return outcome, nil } // streamCh is nil when we've hit maxPerRound so we stop reading streamCh := cp.commitmentStream for { - if count+len(pending) >= cp.maxPerRound { + activeCount := len(commitments) + if activeCount >= cp.maxPerRound { streamCh = nil } else { streamCh = cp.commitmentStream @@ -144,85 +264,61 @@ func (cp *childPrecollector) run(ctx context.Context, snapshot smtbackend.Snapsh select { case commitment := <-streamCh: - pending = append(pending, commitment) - if len(pending) >= miniBatchSize { - if err := flush(); err != nil { - cp.setStopErr(err) - return - } - } + commitments = append(commitments, commitment) case req := <-cp.advanceCh: - if err := drainBufferedCommitments(cp.commitmentStream, cp.maxPerRound, &count, &pending, flush); err != nil { - cp.setStopErr(err) - req.resultCh <- advanceResponse{err: err} - return - } - if err := flush(); err != nil { - cp.setStopErr(err) - req.resultCh <- advanceResponse{err: err} - return - } - result := &preCollectionResult{ - snapshot: snapshot, - commitments: commitments, - leaves: leaves, - } - // Chain new collection from current snapshot - nextSnapshot, err := snapshot.Fork(ctx) + advanceStart := time.Now() + pendingAtAdvance := len(commitments) + outcome, err := prepareSynchronously() if err != nil { - err = fmt.Errorf("failed to fork next precollector snapshot: %w", err) cp.setStopErr(err) req.resultCh <- advanceResponse{err: err} return } - snapshot = nextSnapshot + result := outcome.result + cp.logger.WithContext(ctx).Debug("Precollector advanced", + "commitments", len(result.commitments), + "leaves", len(result.leaves), + "pendingAtAdvance", pendingAtAdvance, + "maxPerRound", cp.maxPerRound, + "advanceFlush", outcome.stats.flushTotal.String(), + "total", time.Since(advanceStart).String(), + "flushCalls", outcome.stats.flushCalls, + "flushAdded", outcome.stats.flushAdded, + "flushTotal", outcome.stats.flushTotal.String(), + "flushMax", outcome.stats.flushMax.String(), + "stageCalls", outcome.stats.stageCalls, + "stageAdded", outcome.stats.stageAdded, + "stageTotal", outcome.stats.stageTotal.String(), + "stageMax", outcome.stats.stageMax.String(), + "fork", outcome.stats.fork.String()) + snapshot = outcome.nextSnapshot + cp.blockNumber = incrementBigInt(cp.blockNumber) + cp.proposalID = uuid.NewString() commitments = make([]*models.CertificationRequest, 0) - leaves = make([]smtbackend.LeafInput, 0) - count = 0 req.resultCh <- advanceResponse{result: result} case <-cp.stopCh: - if err := flush(); err != nil { - cp.setStopErr(err) - } return case <-ctx.Done(): - if err := flush(); err != nil { - cp.setStopErr(err) - } return } } } -// drainBufferedCommitments folds already-buffered commitments into the current -// round before an advance boundary is cut. -func drainBufferedCommitments( - stream <-chan *models.CertificationRequest, - maxPerRound int, - count *int, - pending *[]*models.CertificationRequest, - flush func() error, -) error { - for *count+len(*pending) < maxPerRound { - select { - case commitment, ok := <-stream: - if !ok { - return nil - } - *pending = append(*pending, commitment) - if len(*pending) >= miniBatchSize { - if err := flush(); err != nil { - return err - } - } - default: - return nil - } +func cloneBigInt(v *api.BigInt) *api.BigInt { + if v == nil || v.Int == nil { + return nil + } + return api.NewBigInt(new(big.Int).Set(v.Int)) +} + +func incrementBigInt(v *api.BigInt) *api.BigInt { + if v == nil || v.Int == nil { + return nil } - return nil + return api.NewBigInt(new(big.Int).Add(v.Int, big.NewInt(1))) } func (cp *childPrecollector) addBatch( diff --git a/internal/round/recovery.go b/internal/round/recovery.go index 787902e..7505bca 100644 --- a/internal/round/recovery.go +++ b/internal/round/recovery.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "sort" "github.com/unicitynetwork/aggregator-go/internal/logger" "github.com/unicitynetwork/aggregator-go/internal/models" @@ -18,10 +19,205 @@ type RecoveryResult struct { StateIDs []api.StateID } -// indexedStateID tracks a state ID with its original position in the block. -type indexedStateID struct { - stateID api.StateID - leafIndex int +// FinalizeCertifiedProposal completes a previously stored proposed block when +// BFT returns the UC after this process lost its in-memory proposedBlock. +func (rm *RoundManager) FinalizeCertifiedProposal( + ctx context.Context, + blockNumber *api.BigInt, + rootHash api.HexBytes, + unicityCertificate api.HexBytes, +) (bool, error) { + if rm.smtBackend == nil { + return false, fmt.Errorf("SMT backend not initialized") + } + + block, err := getBlockAnyFinalization(ctx, rm.storage, blockNumber) + if err != nil { + return false, fmt.Errorf("failed to load durable proposal block %s: %w", blockNumber.String(), err) + } + if block == nil { + return false, nil + } + if !bytes.Equal(block.RootHash, rootHash) { + return false, fmt.Errorf("durable proposal block %s root %s does not match certified root %s", + block.Index.String(), block.RootHash.String(), rootHash.String()) + } + if block.Finalized { + return true, nil + } + if block.Status != "" && block.Status != models.FinalityStatusProposed && block.Status != models.FinalityStatusFinalizing { + return false, nil + } + block.UnicityCertificate = unicityCertificate + + records, err := loadAggregatorRecordsForBlockAnyFinalization(ctx, rm.storage, block) + if err != nil { + return false, fmt.Errorf("failed to load durable proposal records %s: %w", block.Index.String(), err) + } + stateIDs, leaves, err := stateIDsAndLeavesFromAggregatorRecords(records) + if err != nil { + return false, fmt.Errorf("failed to derive durable proposal leaves for block %s: %w", block.Index.String(), err) + } + + rm.finalizationMu.Lock() + defer rm.finalizationMu.Unlock() + + snapshot, err := rm.smtBackend.CreateSnapshot(ctx) + if err != nil { + return false, fmt.Errorf("failed to create SMT snapshot for durable proposal %s: %w", block.Index.String(), err) + } + snapshotCommitted := false + defer func() { + if !snapshotCommitted { + snapshot.Discard(ctx) + } + }() + + candidateRoot, err := applyDurableProposalLeaves(ctx, snapshot, leaves) + if err != nil { + return false, fmt.Errorf("failed to apply durable proposal leaves for block %s: %w", block.Index.String(), err) + } + if !bytes.Equal(candidateRoot, rootHash) { + return false, fmt.Errorf("durable proposal block %s candidate root %s does not match certified root %s", + block.Index.String(), api.HexBytes(candidateRoot).String(), rootHash.String()) + } + + if !rm.usesDiskSMTBackend() { + if err := setBlockFinalizingWithCertificate(ctx, rm.storage, block); err != nil { + return false, err + } + block.Finalized = false + block.Status = models.FinalityStatusFinalizing + + smtNodes, err := rm.convertLeavesToNodes(leaves) + if err != nil { + return false, fmt.Errorf("failed to convert durable proposal leaves to SMT nodes: %w", err) + } + if _, err := rm.storeDataParallel(ctx, smtNodes); err != nil { + return false, fmt.Errorf("failed to store durable proposal SMT nodes: %w", err) + } + } + + var preparedProofView smtbackend.PreparedProofView + proofViewPublished := false + defer func() { + if preparedProofView != nil && !proofViewPublished { + preparedProofView.Discard(ctx) + } + }() + if rm.usesDiskSMTBackend() { + if err := setBlockFinalizedWithCertificate(ctx, rm.storage, block); err != nil { + return false, err + } + block.Finalized = true + block.Status = models.FinalityStatusFinalized + } + if publishable, ok := snapshot.(smtbackend.ProofViewPreparingSnapshot); ok { + preparedProofView, err = publishable.CommitAndPrepareProofView(ctx, smtbackend.CommitMetadata{BlockNumber: block.Index, RootHash: rootHash}) + } else { + err = snapshot.Commit(ctx, smtbackend.CommitMetadata{BlockNumber: block.Index, RootHash: rootHash}) + } + if err != nil { + return false, fmt.Errorf("failed to commit durable proposal SMT snapshot: %w", err) + } + snapshotCommitted = true + + if !rm.usesDiskSMTBackend() { + if err := setBlockFinalizedWithCertificate(ctx, rm.storage, block); err != nil { + return false, err + } + block.Finalized = true + block.Status = models.FinalityStatusFinalized + } + + rm.markProofsReady(block, stateIDs, records) + if preparedProofView != nil { + if err := preparedProofView.Publish(ctx); err != nil { + return false, fmt.Errorf("failed to publish durable proposal proof view: %w", err) + } + proofViewPublished = true + } + rm.stateTracker.SetLastSyncedBlock(block.Index.Int) + rm.ackRecoveredProposalCommitments(ctx, block.Index, stateIDs) + rm.logger.WithContext(ctx).Info("Durable proposal finalized", + "blockNumber", block.Index.String(), + "records", len(records)) + return true, nil +} + +// LoadDurableProposal returns a stored uncertified proposal for the requested +// round, if one exists. +func (rm *RoundManager) LoadDurableProposal(ctx context.Context, blockNumber *api.BigInt) (*models.Block, bool, error) { + block, err := getBlockAnyFinalization(ctx, rm.storage, blockNumber) + if err != nil { + return nil, false, fmt.Errorf("failed to load durable proposal block %s: %w", blockNumber.String(), err) + } + if block == nil || block.Finalized || block.Status != models.FinalityStatusProposed { + return nil, false, nil + } + return block, true, nil +} + +type aggregatorRecordAnyFinalizationStorage interface { + GetByStateIDAnyFinalization(ctx context.Context, stateID api.StateID) (*models.AggregatorRecord, error) + GetByBlockNumberAnyFinalization(ctx context.Context, blockNumber *api.BigInt) ([]*models.AggregatorRecord, error) +} + +type aggregatorRecordProposalAnyFinalizationStorage interface { + GetByBlockNumberAndProposalIDAnyFinalization(ctx context.Context, blockNumber *api.BigInt, proposalID string) ([]*models.AggregatorRecord, error) +} + +type aggregatorRecordProposalDeleter interface { + DeleteByBlockNumberAndProposalID(ctx context.Context, blockNumber *api.BigInt, proposalID string) error +} + +type aggregatorRecordSerialBatchStorage interface { + StoreBatchSerial(ctx context.Context, records []*models.AggregatorRecord) error +} + +type blockAnyFinalizationStorage interface { + GetByNumberAnyFinalization(ctx context.Context, blockNumber *api.BigInt) (*models.Block, error) +} + +type blockCertificateFinalizer interface { + SetFinalizedWithCertificate(ctx context.Context, block *models.Block) error +} + +type blockCertificateFinalizingMarker interface { + SetFinalizingWithCertificate(ctx context.Context, block *models.Block) error +} + +type blockStatusUpdater interface { + SetStatus(ctx context.Context, blockNumber *api.BigInt, status string) error +} + +// AbandonDurableProposal removes an uncertified proposed block from the retry path. +func (rm *RoundManager) AbandonDurableProposal(ctx context.Context, blockNumber *api.BigInt, rootHash api.HexBytes) error { + block, err := getBlockAnyFinalization(ctx, rm.storage, blockNumber) + if err != nil { + return fmt.Errorf("failed to load durable proposal block %s: %w", blockNumber.String(), err) + } + if block == nil || block.Finalized || block.Status != models.FinalityStatusProposed { + return nil + } + if !bytes.Equal(block.RootHash, rootHash) { + return fmt.Errorf("refusing to abandon durable proposal %s: stored root %s does not match expected root %s", + block.Index.String(), block.RootHash.String(), rootHash.String()) + } + + return rm.storage.WithTransaction(ctx, func(txCtx context.Context) error { + if updater, ok := rm.storage.BlockStorage().(blockStatusUpdater); ok { + if err := updater.SetStatus(txCtx, block.Index, models.FinalityStatusAbandoned); err != nil { + return err + } + } + if deleter, ok := rm.storage.AggregatorRecordStorage().(aggregatorRecordProposalDeleter); ok { + if err := deleter.DeleteByBlockNumberAndProposalID(txCtx, block.Index, block.ProposalID); err != nil { + return err + } + } + return nil + }) } // RecoverUnfinalizedBlock checks for and completes any unfinalized blocks. @@ -31,6 +227,16 @@ func RecoverUnfinalizedBlock( log *logger.Logger, storage interfaces.Storage, commitmentQueue interfaces.CommitmentQueue, +) (*RecoveryResult, error) { + return recoverUnfinalizedBlock(ctx, log, storage, commitmentQueue, true) +} + +func recoverUnfinalizedBlock( + ctx context.Context, + log *logger.Logger, + storage interfaces.Storage, + commitmentQueue interfaces.CommitmentQueue, + repairSMTNodes bool, ) (*RecoveryResult, error) { log.WithContext(ctx).Info("Checking for unfinalized blocks...") @@ -60,7 +266,7 @@ func RecoverUnfinalizedBlock( "blockNumber", block.Index.String(), "rootHash", block.RootHash.String()) - stateIDs, err := recoverBlock(ctx, log, storage, commitmentQueue, block) + stateIDs, err := recoverBlock(ctx, log, storage, commitmentQueue, block, repairSMTNodes) if err != nil { return nil, fmt.Errorf("failed to recover block %s: %w", block.Index.String(), err) } @@ -79,62 +285,61 @@ func recoverBlock( storage interfaces.Storage, commitmentQueue interfaces.CommitmentQueue, block *models.Block, + repairSMTNodes bool, ) ([]api.StateID, error) { - blockRecords, err := storage.BlockRecordsStorage().GetByBlockNumber(ctx, block.Index) + records, err := loadAggregatorRecordsForBlockAnyFinalization(ctx, storage, block) if err != nil { - return nil, fmt.Errorf("failed to get block records: %w", err) - } - if blockRecords == nil { - return nil, fmt.Errorf("FATAL: block records not found for block %s", block.Index.String()) + return nil, fmt.Errorf("failed to get aggregator records: %w", err) } - - stateIDs := blockRecords.StateIDs - log.WithContext(ctx).Info("Block records found", "stateCount", len(stateIDs)) - - existingRecordIDs, err := storage.AggregatorRecordStorage().GetExistingStateIDs(ctx, stateIDsToStrings(stateIDs)) + stateIDs, _, err := stateIDsAndLeavesFromAggregatorRecords(records) if err != nil { - return nil, fmt.Errorf("failed to check existing records: %w", err) + return nil, err } - smtKeyStrings := make([]string, len(stateIDs)) - for i, stateID := range stateIDs { - keyBytes, err := stateID.GetTreeKey() + log.WithContext(ctx).Info("Aggregator records found", "stateCount", len(stateIDs)) + + if repairSMTNodes { + smtKeyStrings := make([]string, len(stateIDs)) + for i, stateID := range stateIDs { + keyBytes, err := stateID.GetTreeKey() + if err != nil { + return nil, fmt.Errorf("failed to get SMT key for stateID: %w", err) + } + smtKeyStrings[i] = api.HexBytes(keyBytes).String() + } + existingSmtKeys, err := storage.SmtStorage().GetExistingKeys(ctx, smtKeyStrings) if err != nil { - return nil, fmt.Errorf("failed to get SMT key for stateID: %w", err) + return nil, fmt.Errorf("failed to check existing SMT nodes: %w", err) } - smtKeyStrings[i] = api.HexBytes(keyBytes).String() - } - existingSmtKeys, err := storage.SmtStorage().GetExistingKeys(ctx, smtKeyStrings) - if err != nil { - return nil, fmt.Errorf("failed to check existing SMT nodes: %w", err) - } - var missingRecords []indexedStateID - var missingSmtKeys []api.StateID - for i, stateID := range stateIDs { - if !existingRecordIDs[stateID.String()] { - missingRecords = append(missingRecords, indexedStateID{stateID: stateID, leafIndex: i}) - } - if !existingSmtKeys[smtKeyStrings[i]] { - missingSmtKeys = append(missingSmtKeys, stateID) + var missingSmtKeys []api.StateID + for i, stateID := range stateIDs { + if !existingSmtKeys[smtKeyStrings[i]] { + missingSmtKeys = append(missingSmtKeys, stateID) + } } - } - log.WithContext(ctx).Info("Recovery status", - "totalStateIDs", len(stateIDs), - "existingRecords", len(existingRecordIDs), - "existingSmtNodes", len(existingSmtKeys), - "missingRecords", len(missingRecords), - "missingSmtNodes", len(missingSmtKeys)) + log.WithContext(ctx).Info("Recovery status", + "totalStateIDs", len(stateIDs), + "existingRecords", len(records), + "existingSmtNodes", len(existingSmtKeys), + "missingRecords", 0, + "missingSmtNodes", len(missingSmtKeys)) - if len(missingRecords) > 0 || len(missingSmtKeys) > 0 { - if err := recoverMissingData(ctx, log, storage, commitmentQueue, block.Index, missingRecords, missingSmtKeys); err != nil { - return nil, err + if len(missingSmtKeys) > 0 { + if err := recoverMissingSMTNodes(ctx, log, storage, commitmentQueue, missingSmtKeys); err != nil { + return nil, err + } } + } else { + log.WithContext(ctx).Info("Recovery status", + "totalStateIDs", len(stateIDs), + "existingRecords", len(records), + "repairSMTNodes", false) } - if err := storage.BlockStorage().SetFinalized(ctx, block.Index, true); err != nil { - return nil, fmt.Errorf("failed to set block as finalized: %w", err) + if err := setBlockFinalizedWithCertificate(ctx, storage, block); err != nil { + return nil, err } // Ack certification requests in Redis - fetch only the ones we need by state ID. @@ -159,102 +364,244 @@ func recoverBlock( return stateIDs, nil } -func recoverMissingData( +func sortAggregatorRecordsByLeafIndex(records []*models.AggregatorRecord) { + sort.SliceStable(records, func(i, j int) bool { + left := records[i] + right := records[j] + if left == nil || left.LeafIndex == nil || left.LeafIndex.Int == nil { + return right != nil && right.LeafIndex != nil && right.LeafIndex.Int != nil + } + if right == nil || right.LeafIndex == nil || right.LeafIndex.Int == nil { + return false + } + return left.LeafIndex.Int.Cmp(right.LeafIndex.Int) < 0 + }) +} + +func loadAggregatorRecordsForBlockAnyFinalization(ctx context.Context, storage interfaces.Storage, block *models.Block) ([]*models.AggregatorRecord, error) { + if block == nil { + return nil, fmt.Errorf("block is nil") + } + records, err := getAggregatorRecordsByBlockAndProposalAnyFinalization(ctx, storage, block.Index, block.ProposalID) + if err != nil { + return nil, err + } + sortAggregatorRecordsByLeafIndex(records) + return records, nil +} + +func stateIDsAndLeavesFromAggregatorRecords(records []*models.AggregatorRecord) ([]api.StateID, []smtbackend.LeafInput, error) { + stateIDs := make([]api.StateID, len(records)) + leaves := make([]smtbackend.LeafInput, len(records)) + for i, record := range records { + if record == nil { + return nil, nil, fmt.Errorf("nil aggregator record at index %d", i) + } + stateIDs[i] = record.StateID + key, err := record.StateID.GetTreeKey() + if err != nil { + return nil, nil, fmt.Errorf("failed to get SMT key for stateID %s: %w", record.StateID.String(), err) + } + leaves[i] = smtbackend.LeafInput{ + Key: append([]byte(nil), key...), + Value: append([]byte(nil), record.CertificationData.TransactionHash...), + } + } + return stateIDs, leaves, nil +} + +func applyDurableProposalLeaves(ctx context.Context, snapshot smtbackend.Snapshot, leaves []smtbackend.LeafInput) ([]byte, error) { + if len(leaves) == 0 { + return snapshot.RootHashRaw(ctx) + } + result, err := snapshot.AddLeavesClassified(ctx, leaves) + if err != nil { + return nil, err + } + if err := result.ValidateAllAccepted(len(leaves)); err != nil { + return nil, err + } + return result.CandidateRoot, nil +} + +func (rm *RoundManager) ackRecoveredProposalCommitments(ctx context.Context, blockNumber *api.BigInt, stateIDs []api.StateID) { + if rm.commitmentQueue == nil || len(stateIDs) == 0 { + return + } + commitmentMap, err := rm.commitmentQueue.GetByStateIDs(ctx, stateIDs) + if err != nil { + rm.logger.WithContext(ctx).Warn("Failed to load durable proposal commitments for ack", + "blockNumber", blockNumber.String(), + "error", err.Error()) + return + } + if len(commitmentMap) == 0 { + return + } + ackEntries := make([]interfaces.CertificationRequestAck, 0, len(commitmentMap)) + for _, commitment := range commitmentMap { + ackEntries = append(ackEntries, interfaces.CertificationRequestAck{ + StateID: commitment.StateID, + StreamID: commitment.StreamID, + }) + } + if err := rm.commitmentQueue.MarkProcessed(ctx, ackEntries); err != nil { + rm.logger.WithContext(ctx).Warn("Failed to ack durable proposal commitments", + "blockNumber", blockNumber.String(), + "commitments", len(ackEntries), + "error", err.Error()) + } +} + +func recoverMissingSMTNodes( ctx context.Context, log *logger.Logger, storage interfaces.Storage, commitmentQueue interfaces.CommitmentQueue, - blockNumber *api.BigInt, - missingRecords []indexedStateID, missingSmtKeys []api.StateID, ) error { - // Collect all needed state IDs. - neededIDsMap := make(map[string]api.StateID, len(missingRecords)+len(missingSmtKeys)) - for _, missing := range missingRecords { - neededIDsMap[missing.stateID.String()] = missing.stateID - } - for _, stateID := range missingSmtKeys { - neededIDsMap[stateID.String()] = stateID - } - neededIDs := make([]api.StateID, 0, len(neededIDsMap)) - for _, stateID := range neededIDsMap { - neededIDs = append(neededIDs, stateID) + if len(missingSmtKeys) == 0 { + return nil } - // Fetch only the certification requests we need (streams in batches internally). - commitmentMap, err := commitmentQueue.GetByStateIDs(ctx, neededIDs) + commitmentMap, err := commitmentQueue.GetByStateIDs(ctx, missingSmtKeys) if err != nil { return fmt.Errorf("failed to get commitments: %w", err) } - if len(missingRecords) > 0 { - var records []*models.AggregatorRecord - for _, missing := range missingRecords { - commitment, ok := commitmentMap[missing.stateID.String()] - if !ok { - existingRecord, err := storage.AggregatorRecordStorage().GetByStateID(ctx, missing.stateID) - if err != nil { - return fmt.Errorf("failed to check existing record: %w", err) - } - if existingRecord != nil { - continue - } - return fmt.Errorf("FATAL: certification request not found for stateID %s", missing.stateID) + var nodes []*models.SmtNode + for _, stateID := range missingSmtKeys { + commitment, ok := commitmentMap[stateID.String()] + if !ok { + keyBytes, err := stateID.GetTreeKey() + if err != nil { + return fmt.Errorf("failed to get SMT key for stateID: %w", err) } - leafIndex := api.NewBigInt(nil) - leafIndex.SetInt64(int64(missing.leafIndex)) - records = append(records, models.NewAggregatorRecord(commitment, blockNumber, leafIndex)) - } - - if len(records) > 0 { - if err := storage.AggregatorRecordStorage().StoreBatch(ctx, records); err != nil { - return fmt.Errorf("failed to store missing aggregator records: %w", err) + existingNode, err := storage.SmtStorage().GetByKey(ctx, keyBytes) + if err != nil { + return fmt.Errorf("failed to check existing SMT node: %w", err) } - log.WithContext(ctx).Info("Stored missing aggregator records", "count", len(records)) - } - } - - if len(missingSmtKeys) > 0 { - var nodes []*models.SmtNode - for _, stateID := range missingSmtKeys { - commitment, ok := commitmentMap[stateID.String()] - if !ok { - keyBytes, err := stateID.GetTreeKey() - if err != nil { - return fmt.Errorf("failed to get SMT key for stateID: %w", err) - } - existingNode, err := storage.SmtStorage().GetByKey(ctx, keyBytes) - if err != nil { - return fmt.Errorf("failed to check existing SMT node: %w", err) - } - if existingNode != nil { - continue - } - return fmt.Errorf("FATAL: certification request not found for SMT key %s", stateID) + if existingNode != nil { + continue } - - keyBytes, err := commitment.StateID.GetTreeKey() + existingRecord, err := getAggregatorRecordAnyFinalization(ctx, storage, stateID) if err != nil { - return fmt.Errorf("failed to get SMT key for commitment: %w", err) + return fmt.Errorf("failed to check existing aggregator record: %w", err) } - leafValue, err := commitment.LeafValue() - if err != nil { - return fmt.Errorf("failed to create leaf value: %w", err) + if existingRecord == nil { + return fmt.Errorf("FATAL: durable aggregator record not found for SMT key %s", stateID) } - nodes = append(nodes, models.NewSmtNode(keyBytes, leafValue)) + nodes = append(nodes, models.NewSmtNode(keyBytes, append([]byte(nil), existingRecord.CertificationData.TransactionHash...))) + continue } - if len(nodes) > 0 { - if err := storage.SmtStorage().StoreBatch(ctx, nodes); err != nil { - return fmt.Errorf("failed to store missing SMT nodes: %w", err) - } - log.WithContext(ctx).Info("Stored missing SMT nodes", "count", len(nodes)) + keyBytes, err := commitment.StateID.GetTreeKey() + if err != nil { + return fmt.Errorf("failed to get SMT key for commitment: %w", err) + } + leafValue, err := commitment.LeafValue() + if err != nil { + return fmt.Errorf("failed to create leaf value: %w", err) + } + nodes = append(nodes, models.NewSmtNode(keyBytes, leafValue)) + } + + if len(nodes) > 0 { + if err := storage.SmtStorage().StoreBatch(ctx, nodes); err != nil { + return fmt.Errorf("failed to store missing SMT nodes: %w", err) + } + log.WithContext(ctx).Info("Stored missing SMT nodes", "count", len(nodes)) + } + + return nil +} + +func getBlockAnyFinalization(ctx context.Context, storage interfaces.Storage, blockNumber *api.BigInt) (*models.Block, error) { + if reader, ok := storage.BlockStorage().(blockAnyFinalizationStorage); ok { + return reader.GetByNumberAnyFinalization(ctx, blockNumber) + } + block, err := storage.BlockStorage().GetByNumber(ctx, blockNumber) + if err != nil { + return nil, err + } + if block != nil { + return block, nil + } + unfinalized, err := storage.BlockStorage().GetUnfinalized(ctx) + if err != nil { + return nil, err + } + for _, candidate := range unfinalized { + if candidate != nil && candidate.Index.Cmp(blockNumber.Int) == 0 { + return candidate, nil } } + return nil, nil +} +func setBlockFinalizedWithCertificate(ctx context.Context, storage interfaces.Storage, block *models.Block) error { + if updater, ok := storage.BlockStorage().(blockCertificateFinalizer); ok { + if err := updater.SetFinalizedWithCertificate(ctx, block); err != nil { + return fmt.Errorf("failed to set block as finalized: %w", err) + } + return nil + } + if err := storage.BlockStorage().SetFinalized(ctx, block.Index, true); err != nil { + return fmt.Errorf("failed to set block as finalized: %w", err) + } return nil } +func setBlockFinalizingWithCertificate(ctx context.Context, storage interfaces.Storage, block *models.Block) error { + if updater, ok := storage.BlockStorage().(blockCertificateFinalizingMarker); ok { + if err := updater.SetFinalizingWithCertificate(ctx, block); err != nil { + return fmt.Errorf("failed to set block as finalizing: %w", err) + } + return nil + } + return fmt.Errorf("block storage does not support certified finalizing marker") +} + +func getAggregatorRecordAnyFinalization(ctx context.Context, storage interfaces.Storage, stateID api.StateID) (*models.AggregatorRecord, error) { + if reader, ok := storage.AggregatorRecordStorage().(aggregatorRecordAnyFinalizationStorage); ok { + return reader.GetByStateIDAnyFinalization(ctx, stateID) + } + return storage.AggregatorRecordStorage().GetByStateID(ctx, stateID) +} + +func getAggregatorRecordsByBlockAnyFinalization(ctx context.Context, storage interfaces.Storage, blockNumber *api.BigInt) ([]*models.AggregatorRecord, error) { + if reader, ok := storage.AggregatorRecordStorage().(aggregatorRecordAnyFinalizationStorage); ok { + return reader.GetByBlockNumberAnyFinalization(ctx, blockNumber) + } + return storage.AggregatorRecordStorage().GetByBlockNumber(ctx, blockNumber) +} + +func getAggregatorRecordsByBlockAndProposalAnyFinalization( + ctx context.Context, + storage interfaces.Storage, + blockNumber *api.BigInt, + proposalID string, +) ([]*models.AggregatorRecord, error) { + if reader, ok := storage.AggregatorRecordStorage().(aggregatorRecordProposalAnyFinalizationStorage); ok { + return reader.GetByBlockNumberAndProposalIDAnyFinalization(ctx, blockNumber, proposalID) + } + records, err := getAggregatorRecordsByBlockAnyFinalization(ctx, storage, blockNumber) + if err != nil { + return nil, err + } + return filterAggregatorRecordsByProposalID(records, proposalID), nil +} + +func filterAggregatorRecordsByProposalID(records []*models.AggregatorRecord, proposalID string) []*models.AggregatorRecord { + filtered := records[:0] + for _, record := range records { + if record != nil && record.ProposalID == proposalID { + filtered = append(filtered, record) + } + } + return filtered +} + // LoadRecoveredNodesIntoBackend loads SMT nodes for a recovered block into the active SMT backend. // Used in HA mode when a follower becomes leader. func LoadRecoveredNodesIntoBackend( @@ -269,8 +616,10 @@ func LoadRecoveredNodesIntoBackend( return fmt.Errorf("SMT backend not initialized") } var expectedRoot []byte + var block *models.Block if blockNumber != nil { - block, err := storage.BlockStorage().GetByNumber(ctx, blockNumber) + var err error + block, err = storage.BlockStorage().GetByNumber(ctx, blockNumber) if err != nil { return fmt.Errorf("failed to load recovered finalized block %s: %w", blockNumber.String(), err) } @@ -282,28 +631,29 @@ func LoadRecoveredNodesIntoBackend( if blockNumber == nil { return nil } - state, err := backend.CommittedState(ctx) + return loadRecoveredLeavesIntoBackend(ctx, log, backend, blockNumber, expectedRoot, nil) + } + + if usesDiskBackedBackend(backend) { + if blockNumber == nil { + return fmt.Errorf("disk recovered block number is nil") + } + if block == nil { + return fmt.Errorf("failed to load recovered finalized block %s", blockNumber.String()) + } + records, err := loadAggregatorRecordsForBlockAnyFinalization(ctx, storage, block) if err != nil { - return fmt.Errorf("failed to read SMT backend state: %w", err) + return fmt.Errorf("failed to load recovered aggregator records: %w", err) } - snapshot, err := backend.CreateSnapshot(ctx) + recordStateIDs, leaves, err := stateIDsAndLeavesFromAggregatorRecords(records) if err != nil { - return fmt.Errorf("failed to create SMT snapshot: %w", err) - } - defer snapshot.Discard(ctx) - rootHash := state.RootHash - if expectedRoot != nil { - if !bytes.Equal(state.RootHash, expectedRoot) { - return fmt.Errorf("recovered empty block root mismatch: SMT=%s block=%s", - api.HexBytes(state.RootHash).String(), api.HexBytes(expectedRoot).String()) - } - rootHash = expectedRoot + return fmt.Errorf("failed to load recovered leaves from aggregator records: %w", err) } - if err := snapshot.Commit(ctx, smtbackend.CommitMetadata{BlockNumber: blockNumber, RootHash: rootHash}); err != nil { - return fmt.Errorf("failed to advance recovered SMT metadata: %w", err) + if !sameStateIDs(recordStateIDs, stateIDs) { + return fmt.Errorf("recovered aggregator records state IDs mismatch: records=%d recovered=%d", len(recordStateIDs), len(stateIDs)) } - log.WithContext(ctx).Info("Advanced recovered SMT metadata", "blockNumber", blockNumber.String()) - return nil + log.WithContext(ctx).Info("Loading recovered records into disk SMT", "count", len(leaves)) + return loadRecoveredLeavesIntoBackend(ctx, log, backend, blockNumber, expectedRoot, leaves) } log.WithContext(ctx).Info("Loading recovered SMT nodes", "count", len(stateIDs)) @@ -342,23 +692,49 @@ func LoadRecoveredNodesIntoBackend( } } + return loadRecoveredLeavesIntoBackend(ctx, log, backend, blockNumber, expectedRoot, leaves) +} + +func usesDiskBackedBackend(backend smtbackend.Backend) bool { + diskBacked, ok := backend.(smtbackend.DiskBacked) + return ok && diskBacked.IsDiskBackedSMT() +} + +func loadRecoveredLeavesIntoBackend( + ctx context.Context, + log *logger.Logger, + backend smtbackend.Backend, + blockNumber *api.BigInt, + expectedRoot []byte, + leaves []smtbackend.LeafInput, +) error { snapshot, err := backend.CreateSnapshot(ctx) if err != nil { return fmt.Errorf("failed to create SMT snapshot: %w", err) } defer snapshot.Discard(ctx) - result, err := snapshot.AddLeavesClassified(ctx, leaves) - if err != nil { - return fmt.Errorf("failed to add recovered nodes to SMT: %w", err) - } - if err := result.ValidateAllAccepted(len(leaves)); err != nil { - return fmt.Errorf("failed to add recovered nodes to SMT: %w", err) + + var rootHash []byte + if len(leaves) == 0 { + state, err := backend.CommittedState(ctx) + if err != nil { + return fmt.Errorf("failed to read SMT backend state: %w", err) + } + rootHash = state.RootHash + } else { + result, err := snapshot.AddLeavesClassified(ctx, leaves) + if err != nil { + return fmt.Errorf("failed to add recovered nodes to SMT: %w", err) + } + if err := result.ValidateAllAccepted(len(leaves)); err != nil { + return fmt.Errorf("failed to add recovered nodes to SMT: %w", err) + } + rootHash = result.CandidateRoot } - rootHash := result.CandidateRoot if expectedRoot != nil { - if !bytes.Equal(result.CandidateRoot, expectedRoot) { + if !bytes.Equal(rootHash, expectedRoot) { return fmt.Errorf("recovered SMT root mismatch: candidate=%s block=%s", - api.HexBytes(result.CandidateRoot).String(), api.HexBytes(expectedRoot).String()) + api.HexBytes(rootHash).String(), api.HexBytes(expectedRoot).String()) } rootHash = expectedRoot } @@ -366,16 +742,9 @@ func LoadRecoveredNodesIntoBackend( return fmt.Errorf("failed to commit recovered nodes to SMT: %w", err) } - log.WithContext(ctx).Info("Loaded recovered SMT nodes", "count", len(nodes)) + log.WithContext(ctx).Info("Loaded recovered SMT nodes", "count", len(leaves)) return nil } -func stateIDsToStrings(stateIDs []api.StateID) []string { - result := make([]string, len(stateIDs)) - for i, stateID := range stateIDs { - result[i] = stateID.String() - } - return result -} // CleanupProcessedPendingCommitments ACKs pending commitments that are already in AggregatorRecords. // This handles the case where a block was finalized but MarkProcessed failed (e.g., Redis was down). diff --git a/internal/round/recovery_test.go b/internal/round/recovery_test.go index 85be2cf..cfc3502 100644 --- a/internal/round/recovery_test.go +++ b/internal/round/recovery_test.go @@ -1,6 +1,7 @@ package round import ( + "bytes" "context" "math/big" "testing" @@ -13,6 +14,8 @@ import ( redisContainer "github.com/testcontainers/testcontainers-go/modules/redis" "github.com/unicitynetwork/aggregator-go/internal/config" + "github.com/unicitynetwork/aggregator-go/internal/events" + "github.com/unicitynetwork/aggregator-go/internal/ha/state" "github.com/unicitynetwork/aggregator-go/internal/logger" "github.com/unicitynetwork/aggregator-go/internal/models" "github.com/unicitynetwork/aggregator-go/internal/smt" @@ -147,11 +150,12 @@ func (s *RecoveryTestSuite) createTestData(blockNum int64, commitmentCount int, } err := smtTree.AddLeaves(leaves) require.NoError(t, err) - rootHashBytes := smtTree.GetRootHash() + rootHashBytes := smtTree.GetRootHashRaw() // Create block (unfinalized) block := models.NewBlock(blockNumber, "unicity", 0, "1.0", "mainnet", api.HexBytes(rootHashBytes), nil, nil) block.Finalized = false + block.ProposalID = "proposal-" + blockNumber.String() return commitments, block, stateIDs } @@ -181,10 +185,11 @@ func (s *RecoveryTestSuite) storeSmtNodes(commitments []*models.CertificationReq } // Helper to store aggregator records -func (s *RecoveryTestSuite) storeAggregatorRecords(commitments []*models.CertificationRequest, blockNumber *api.BigInt) { +func (s *RecoveryTestSuite) storeAggregatorRecords(commitments []*models.CertificationRequest, block *models.Block) { records := make([]*models.AggregatorRecord, len(commitments)) for i, c := range commitments { - records[i] = models.NewAggregatorRecord(c, blockNumber, api.NewBigInt(big.NewInt(int64(i)))) + records[i] = models.NewAggregatorRecord(c, block.Index, api.NewBigInt(big.NewInt(int64(i)))) + records[i].ProposalID = block.ProposalID } err := s.storage.AggregatorRecordStorage().StoreBatch(s.ctx, records) s.Require().NoError(err) @@ -197,21 +202,16 @@ func (s *RecoveryTestSuite) Test01_NoUnfinalizedBlocks() { t := s.T() // Create a finalized block - commitments, block, stateIDs := s.createTestData(1, 3, "t01") + commitments, block, _ := s.createTestData(1, 3, "t01") block.Finalized = true // Store the finalized block err := s.storage.BlockStorage().Store(s.ctx, block) require.NoError(t, err) - // Store block records - blockRecords := models.NewBlockRecords(block.Index, stateIDs) - err = s.storage.BlockRecordsStorage().Store(s.ctx, blockRecords) - require.NoError(t, err) - // Store SMT nodes and aggregator records s.storeSmtNodes(commitments) - s.storeAggregatorRecords(commitments, block.Index) + s.storeAggregatorRecords(commitments, block) // Run recovery result, err := RecoverUnfinalizedBlock(s.ctx, s.testLogger, s.storage, s.commitmentQueue) @@ -230,20 +230,15 @@ func (s *RecoveryTestSuite) Test02_AllDataPresent() { t := s.T() // Create unfinalized block with all data present - commitments, block, stateIDs := s.createTestData(2, 5, "t02") + commitments, block, _ := s.createTestData(2, 5, "t02") // Store block (unfinalized) err := s.storage.BlockStorage().Store(s.ctx, block) require.NoError(t, err) - // Store block records - blockRecords := models.NewBlockRecords(block.Index, stateIDs) - err = s.storage.BlockRecordsStorage().Store(s.ctx, blockRecords) - require.NoError(t, err) - // Store all SMT nodes and aggregator records s.storeSmtNodes(commitments) - s.storeAggregatorRecords(commitments, block.Index) + s.storeAggregatorRecords(commitments, block) // Store commitments in Redis (for acking) s.storeCommitmentsInRedis(commitments) @@ -264,72 +259,20 @@ func (s *RecoveryTestSuite) Test02_AllDataPresent() { } // ============================================================================ -// Test 3: Missing Aggregator Records (Recover from Redis) -// ============================================================================ -func (s *RecoveryTestSuite) Test03_MissingAggregatorRecords() { - t := s.T() - - // Create unfinalized block - commitments, block, stateIDs := s.createTestData(3, 4, "t03") - - // Store block (unfinalized) - err := s.storage.BlockStorage().Store(s.ctx, block) - require.NoError(t, err) - - // Store block records - blockRecords := models.NewBlockRecords(block.Index, stateIDs) - err = s.storage.BlockRecordsStorage().Store(s.ctx, blockRecords) - require.NoError(t, err) - - // Store ONLY SMT nodes (no aggregator records) - s.storeSmtNodes(commitments) - - // Store commitments in Redis (for recovery) - s.storeCommitmentsInRedis(commitments) - - // Verify no aggregator records exist - count, err := s.storage.AggregatorRecordStorage().Count(s.ctx) - require.NoError(t, err) - countBefore := count - - // Run recovery - result, err := RecoverUnfinalizedBlock(s.ctx, s.testLogger, s.storage, s.commitmentQueue) - require.NoError(t, err) - require.True(t, result.Recovered) - - // Verify aggregator records were recovered - countAfter, err := s.storage.AggregatorRecordStorage().Count(s.ctx) - require.NoError(t, err) - require.Equal(t, countBefore+int64(len(commitments)), countAfter, "Aggregator records should be recovered") - - // Verify block is finalized - finalizedBlock, err := s.storage.BlockStorage().GetByNumber(s.ctx, block.Index) - require.NoError(t, err) - require.NotNil(t, finalizedBlock) - - t.Log("✓ Test03_MissingAggregatorRecords passed") -} - -// ============================================================================ -// Test 4: Missing SMT Nodes (Recover from Redis) +// Test 3: Missing SMT Nodes (Recover from Durable Records) // ============================================================================ func (s *RecoveryTestSuite) Test04_MissingSmtNodes() { t := s.T() // Create unfinalized block - commitments, block, stateIDs := s.createTestData(4, 4, "t04") + commitments, block, _ := s.createTestData(4, 4, "t04") // Store block (unfinalized) err := s.storage.BlockStorage().Store(s.ctx, block) require.NoError(t, err) - // Store block records - blockRecords := models.NewBlockRecords(block.Index, stateIDs) - err = s.storage.BlockRecordsStorage().Store(s.ctx, blockRecords) - require.NoError(t, err) - // Store ONLY aggregator records (no SMT nodes for this block's commitments) - s.storeAggregatorRecords(commitments, block.Index) + s.storeAggregatorRecords(commitments, block) // Store commitments in Redis (for recovery) s.storeCommitmentsInRedis(commitments) @@ -356,236 +299,280 @@ func (s *RecoveryTestSuite) Test04_MissingSmtNodes() { t.Log("✓ Test04_MissingSmtNodes passed") } -// ============================================================================ -// Test 5: Missing Both (Recover from Redis) -// ============================================================================ -func (s *RecoveryTestSuite) Test05_MissingBoth() { +func (s *RecoveryTestSuite) Test04b_MissingSmtNodesRecoverFromDurableRecordsWithoutRedis() { t := s.T() - // Create unfinalized block - commitments, block, stateIDs := s.createTestData(5, 5, "t05") + commitments, block, _ := s.createTestData(40, 4, "t04b") - // Store block (unfinalized) err := s.storage.BlockStorage().Store(s.ctx, block) require.NoError(t, err) - // Store block records - blockRecords := models.NewBlockRecords(block.Index, stateIDs) - err = s.storage.BlockRecordsStorage().Store(s.ctx, blockRecords) - require.NoError(t, err) - - // Store NO SMT nodes and NO aggregator records - // Store commitments in Redis (for recovery) - s.storeCommitmentsInRedis(commitments) + // Durable proposal records must be sufficient to rebuild the certified SMT + // root even if Redis loses the original pending commitments. + s.storeAggregatorRecords(commitments, block) - // Get counts before smtCountBefore, err := s.storage.SmtStorage().Count(s.ctx) require.NoError(t, err) - recordCountBefore, err := s.storage.AggregatorRecordStorage().Count(s.ctx) - require.NoError(t, err) - // Run recovery result, err := RecoverUnfinalizedBlock(s.ctx, s.testLogger, s.storage, s.commitmentQueue) require.NoError(t, err) require.True(t, result.Recovered) - // Verify both were recovered smtCountAfter, err := s.storage.SmtStorage().Count(s.ctx) require.NoError(t, err) - require.Equal(t, smtCountBefore+int64(len(commitments)), smtCountAfter, "SMT nodes should be recovered") + require.Equal(t, smtCountBefore+int64(len(commitments)), smtCountAfter) - recordCountAfter, err := s.storage.AggregatorRecordStorage().Count(s.ctx) - require.NoError(t, err) - require.Equal(t, recordCountBefore+int64(len(commitments)), recordCountAfter, "Aggregator records should be recovered") - - // Verify block is finalized finalizedBlock, err := s.storage.BlockStorage().GetByNumber(s.ctx, block.Index) require.NoError(t, err) require.NotNil(t, finalizedBlock) - - t.Log("✓ Test05_MissingBoth passed") } -// ============================================================================ -// Test 6: Multiple Unfinalized Blocks (FATAL Error) -// ============================================================================ -func (s *RecoveryTestSuite) Test06_MultipleUnfinalizedBlocks() { +func (s *RecoveryTestSuite) Test04c_FinalizeCertifiedProposalFromDurableRecordsWithoutRedis() { t := s.T() - // Create two unfinalized blocks - _, block1, stateIDs1 := s.createTestData(6, 2, "t06a") - _, block2, stateIDs2 := s.createTestData(7, 2, "t06b") + commitments, block, stateIDs := s.createTestData(41, 4, "t04c") + block.Status = models.FinalityStatusProposed + block.Finalized = false + proposalUC := api.NewHexBytes([]byte{0x04, 0x0c}) - // Store both blocks as unfinalized - err := s.storage.BlockStorage().Store(s.ctx, block1) - require.NoError(t, err) - err = s.storage.BlockStorage().Store(s.ctx, block2) - require.NoError(t, err) + records := make([]*models.AggregatorRecord, len(commitments)) + for i, c := range commitments { + record := models.NewAggregatorRecord(c, block.Index, api.NewBigInt(big.NewInt(int64(i)))) + record.ProposalID = block.ProposalID + records[i] = record + } - // Store block records for both - err = s.storage.BlockRecordsStorage().Store(s.ctx, models.NewBlockRecords(block1.Index, stateIDs1)) - require.NoError(t, err) - err = s.storage.BlockRecordsStorage().Store(s.ctx, models.NewBlockRecords(block2.Index, stateIDs2)) + require.NoError(t, s.storage.WithTransaction(s.ctx, func(txCtx context.Context) error { + if err := s.storage.BlockStorage().Store(txCtx, block); err != nil { + return err + } + return s.storage.AggregatorRecordStorage().StoreBatch(txCtx, records) + })) + require.NoError(t, s.redisClient.FlushAll(s.ctx).Err()) + + cfg := config.Config{ + Database: config.DatabaseConfig{Database: "test_durable_proposal_recovery"}, + Processing: config.ProcessingConfig{ + CommitmentStreamBufferSize: 16, + MaxCommitmentsPerRound: 1000, + }, + Sharding: config.ShardingConfig{Mode: config.ShardingModeBFTShard}, + } + rm, err := NewRoundManager( + s.ctx, + &cfg, + s.testLogger, + s.commitmentQueue, + s.storage, + nil, + state.NewSyncStateTracker(), + nil, + events.NewEventBus(s.testLogger), + smt.NewThreadSafeSMT(smt.NewSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits)), + nil, + ) require.NoError(t, err) - // Run recovery - should fail with FATAL error - result, err := RecoverUnfinalizedBlock(s.ctx, s.testLogger, s.storage, s.commitmentQueue) - require.Error(t, err, "Should return error for multiple unfinalized blocks") - require.Nil(t, result) - require.Contains(t, err.Error(), "FATAL") - require.Contains(t, err.Error(), "2 unfinalized blocks") + recovered, err := rm.FinalizeCertifiedProposal(s.ctx, block.Index, block.RootHash, proposalUC) + require.NoError(t, err) + require.True(t, recovered) - // Clean up - finalize both blocks to not affect other tests - _ = s.storage.BlockStorage().SetFinalized(s.ctx, block1.Index, true) - _ = s.storage.BlockStorage().SetFinalized(s.ctx, block2.Index, true) + finalizedBlock, err := s.storage.BlockStorage().GetByNumber(s.ctx, block.Index) + require.NoError(t, err) + require.NotNil(t, finalizedBlock) + require.True(t, finalizedBlock.Finalized) + require.Equal(t, models.FinalityStatusFinalized, finalizedBlock.Status) + require.Equal(t, proposalUC, finalizedBlock.UnicityCertificate) - t.Log("✓ Test06_MultipleUnfinalizedBlocks passed") + for _, stateID := range stateIDs { + record, err := s.storage.AggregatorRecordStorage().GetByStateID(s.ctx, stateID) + require.NoError(t, err) + require.NotNil(t, record) + require.Equal(t, block.ProposalID, record.ProposalID) + } } -// ============================================================================ -// Test 7: Missing Block Records (FATAL Error) -// ============================================================================ -func (s *RecoveryTestSuite) Test07_MissingBlockRecords() { +func (s *RecoveryTestSuite) Test04d_AbandonDurableProposalFreesStateIDsForRestage() { t := s.T() - // Create unfinalized block - _, block, _ := s.createTestData(8, 3, "t07") + commitments, block, _ := s.createTestData(42, 2, "t04d") + block.Status = models.FinalityStatusProposed + block.Finalized = false + block.ProposalID = "proposal-42a" + neighborCommitments, _, _ := s.createTestData(42, 1, "t04d_neighbor") - // Store block (unfinalized) but NO block records - err := s.storage.BlockStorage().Store(s.ctx, block) + records := make([]*models.AggregatorRecord, len(commitments)) + for i, c := range commitments { + record := models.NewAggregatorRecord(c, block.Index, api.NewBigInt(big.NewInt(int64(i)))) + record.ProposalID = block.ProposalID + records[i] = record + } + neighborRecord := models.NewAggregatorRecord(neighborCommitments[0], block.Index, api.NewBigInt(big.NewInt(99))) + neighborRecord.ProposalID = "proposal-42b" + + require.NoError(t, s.storage.WithTransaction(s.ctx, func(txCtx context.Context) error { + if err := s.storage.BlockStorage().Store(txCtx, block); err != nil { + return err + } + if err := s.storage.AggregatorRecordStorage().StoreBatch(txCtx, records); err != nil { + return err + } + return s.storage.AggregatorRecordStorage().Store(txCtx, neighborRecord) + })) + + rm := &RoundManager{storage: s.storage} + require.NoError(t, rm.AbandonDurableProposal(s.ctx, block.Index, block.RootHash)) + + abandonedBlock, err := getBlockAnyFinalization(s.ctx, s.storage, block.Index) require.NoError(t, err) + require.NotNil(t, abandonedBlock) + require.Equal(t, models.FinalityStatusAbandoned, abandonedBlock.Status) - // Run recovery - should fail with FATAL error - result, err := RecoverUnfinalizedBlock(s.ctx, s.testLogger, s.storage, s.commitmentQueue) - require.Error(t, err, "Should return error for missing block records") - require.Nil(t, result) - require.Contains(t, err.Error(), "FATAL") - require.Contains(t, err.Error(), "block records not found") + visibleRecords, err := s.storage.AggregatorRecordStorage().GetByBlockNumber(s.ctx, block.Index) + require.NoError(t, err) + require.Empty(t, visibleRecords) - // Clean up - _ = s.storage.BlockStorage().SetFinalized(s.ctx, block.Index, true) + oldRecords, err := getAggregatorRecordsByBlockAnyFinalization(s.ctx, s.storage, block.Index) + require.NoError(t, err) + require.Len(t, oldRecords, 1) + require.Equal(t, neighborRecord.ProposalID, oldRecords[0].ProposalID) - t.Log("✓ Test07_MissingBlockRecords passed") + restaged := models.NewAggregatorRecord(commitments[0], api.NewBigInt(big.NewInt(43)), api.NewBigInt(big.NewInt(0))) + restaged.ProposalID = "proposal-43" + require.NoError(t, s.storage.AggregatorRecordStorage().Store(s.ctx, restaged)) + + storedRecords, err := getAggregatorRecordsByBlockAnyFinalization(s.ctx, s.storage, restaged.BlockNumber) + require.NoError(t, err) + require.Len(t, storedRecords, 1) + require.Equal(t, "43", storedRecords[0].BlockNumber.String()) + require.Equal(t, restaged.ProposalID, storedRecords[0].ProposalID) } -// ============================================================================ -// Test 8: Certification Request Not Found (FATAL Error) -// ============================================================================ -func (s *RecoveryTestSuite) Test08_CertificationRequestNotFound() { +func (s *RecoveryTestSuite) Test04e_OrphanStagedRecordsDoNotAttachToReusedBlockNumber() { t := s.T() - // Create unfinalized block - commitments, block, stateIDs := s.createTestData(9, 3, "t08") + orphanCommitments, orphanBlock, orphanStateIDs := s.createTestData(45, 3, "t04e_orphan") + orphanBlock.ProposalID = "proposal-45-orphan" + s.storeAggregatorRecords(orphanCommitments, orphanBlock) - // Store block (unfinalized) - err := s.storage.BlockStorage().Store(s.ctx, block) + visibleOrphans, err := s.storage.AggregatorRecordStorage().GetByBlockNumber(s.ctx, orphanBlock.Index) require.NoError(t, err) + require.Empty(t, visibleOrphans, "records without a finalized block must stay invisible") + for _, stateID := range orphanStateIDs { + record, err := s.storage.AggregatorRecordStorage().GetByStateID(s.ctx, stateID) + require.NoError(t, err) + require.Nil(t, record, "orphan staged record must not be visible by stateID") + } - // Store block records - blockRecords := models.NewBlockRecords(block.Index, stateIDs) - err = s.storage.BlockRecordsStorage().Store(s.ctx, blockRecords) + result, err := RecoverUnfinalizedBlock(s.ctx, s.testLogger, s.storage, s.commitmentQueue) require.NoError(t, err) + require.False(t, result.Recovered, "records without a proposed block are not recoverable proposals") - // Store ONLY some commitments in Redis (not all) - // Only store first commitment, missing the rest - err = s.commitmentQueue.Store(s.ctx, commitments[0]) - require.NoError(t, err) - time.Sleep(200 * time.Millisecond) + winnerCommitments, winnerBlock, winnerStateIDs := s.createTestData(45, 4, "t04e_winner") + winnerBlock.Status = models.FinalityStatusProposed + winnerBlock.Finalized = false + winnerBlock.ProposalID = "proposal-45-winner" + require.False(t, bytes.Equal(orphanBlock.RootHash, winnerBlock.RootHash), "test needs distinct proposal roots") - // Run recovery - should fail because certification request data not found - result, err := RecoverUnfinalizedBlock(s.ctx, s.testLogger, s.storage, s.commitmentQueue) - require.Error(t, err, "Should return error when certification request is not found") - require.Nil(t, result) - require.Contains(t, err.Error(), "FATAL") - require.Contains(t, err.Error(), "certification request not found") - - // Clean up - _ = s.storage.BlockStorage().SetFinalized(s.ctx, block.Index, true) + winnerRecords := make([]*models.AggregatorRecord, len(winnerCommitments)) + for i, commitment := range winnerCommitments { + record := models.NewAggregatorRecord(commitment, winnerBlock.Index, api.NewBigInt(big.NewInt(int64(i)))) + record.ProposalID = winnerBlock.ProposalID + winnerRecords[i] = record + } + require.NoError(t, s.storage.WithTransaction(s.ctx, func(txCtx context.Context) error { + if err := s.storage.BlockStorage().Store(txCtx, winnerBlock); err != nil { + return err + } + return s.storage.AggregatorRecordStorage().StoreBatch(txCtx, winnerRecords) + })) - t.Log("✓ Test08_CertificationRequestNotFound passed") + cfg := config.Config{ + Database: config.DatabaseConfig{Database: "test_durable_proposal_orphan_reuse"}, + Processing: config.ProcessingConfig{ + CommitmentStreamBufferSize: 16, + MaxCommitmentsPerRound: 1000, + }, + Sharding: config.ShardingConfig{Mode: config.ShardingModeBFTShard}, + } + rm, err := NewRoundManager( + s.ctx, + &cfg, + s.testLogger, + s.commitmentQueue, + s.storage, + nil, + state.NewSyncStateTracker(), + nil, + events.NewEventBus(s.testLogger), + smt.NewThreadSafeSMT(smt.NewSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits)), + nil, + ) + require.NoError(t, err) + + recovered, err := rm.FinalizeCertifiedProposal(s.ctx, winnerBlock.Index, winnerBlock.RootHash, api.NewHexBytes([]byte{0x04, 0x0e})) + require.NoError(t, err) + require.True(t, recovered) + + visibleWinnerRecords, err := s.storage.AggregatorRecordStorage().GetByBlockNumber(s.ctx, winnerBlock.Index) + require.NoError(t, err) + require.Len(t, visibleWinnerRecords, len(winnerStateIDs)) + for i, record := range visibleWinnerRecords { + require.Equal(t, winnerBlock.ProposalID, record.ProposalID) + require.Equal(t, winnerStateIDs[i], record.StateID) + } + for _, stateID := range orphanStateIDs { + record, err := s.storage.AggregatorRecordStorage().GetByStateID(s.ctx, stateID) + require.NoError(t, err) + require.Nil(t, record, "orphan stateID must remain invisible after same-number finalization") + } } // ============================================================================ -// Test 9: Partial Aggregator Records - Verify LeafIndex Preservation +// Test 5: Multiple Unfinalized Blocks (FATAL Error) // ============================================================================ -func (s *RecoveryTestSuite) Test09_PartialAggregatorRecords_LeafIndexPreserved() { +func (s *RecoveryTestSuite) Test06_MultipleUnfinalizedBlocks() { t := s.T() - // Create unfinalized block with 5 commitments - commitments, block, stateIDs := s.createTestData(10, 5, "t09partial") - - // Store block (unfinalized) - err := s.storage.BlockStorage().Store(s.ctx, block) - require.NoError(t, err) + // Create two unfinalized blocks + _, block1, _ := s.createTestData(6, 2, "t06a") + _, block2, _ := s.createTestData(7, 2, "t06b") - // Store block records - blockRecords := models.NewBlockRecords(block.Index, stateIDs) - err = s.storage.BlockRecordsStorage().Store(s.ctx, blockRecords) + // Store both blocks as unfinalized + err := s.storage.BlockStorage().Store(s.ctx, block1) require.NoError(t, err) - - // Store ONLY aggregator records at positions 0, 1, and 4 (missing 2 and 3) - existingIndices := []int{0, 1, 4} - existingRecords := make([]*models.AggregatorRecord, len(existingIndices)) - for i, idx := range existingIndices { - existingRecords[i] = models.NewAggregatorRecord(commitments[idx], block.Index, api.NewBigInt(big.NewInt(int64(idx)))) - } - err = s.storage.AggregatorRecordStorage().StoreBatch(s.ctx, existingRecords) + err = s.storage.BlockStorage().Store(s.ctx, block2) require.NoError(t, err) - // Store all SMT nodes (so only aggregator records need recovery) - s.storeSmtNodes(commitments) - - // Store ALL commitments in Redis (for recovery) - s.storeCommitmentsInRedis(commitments) - - // Run recovery + // Run recovery - should fail with FATAL error result, err := RecoverUnfinalizedBlock(s.ctx, s.testLogger, s.storage, s.commitmentQueue) - require.NoError(t, err) - require.True(t, result.Recovered) - - // Verify the recovered records have correct leaf indices - // Record at position 2 should have leafIndex=2, record at position 3 should have leafIndex=3 - missingIndices := []int{2, 3} - for _, idx := range missingIndices { - record, err := s.storage.AggregatorRecordStorage().GetByStateID(s.ctx, stateIDs[idx]) - require.NoError(t, err) - require.NotNil(t, record, "Record at position %d should exist after recovery", idx) - require.Equal(t, int64(idx), record.LeafIndex.Int.Int64(), - "Record at position %d should have leafIndex=%d, got %d", idx, idx, record.LeafIndex.Int.Int64()) - } + require.Error(t, err, "Should return error for multiple unfinalized blocks") + require.Nil(t, result) + require.Contains(t, err.Error(), "FATAL") + require.Contains(t, err.Error(), "2 unfinalized blocks") - // Also verify existing records still have correct indices - for _, idx := range existingIndices { - record, err := s.storage.AggregatorRecordStorage().GetByStateID(s.ctx, stateIDs[idx]) - require.NoError(t, err) - require.NotNil(t, record, "Existing record at position %d should still exist", idx) - require.Equal(t, int64(idx), record.LeafIndex.Int.Int64(), - "Existing record at position %d should still have leafIndex=%d", idx, idx) - } + // Clean up - finalize both blocks to not affect other tests + _ = s.storage.BlockStorage().SetFinalized(s.ctx, block1.Index, true) + _ = s.storage.BlockStorage().SetFinalized(s.ctx, block2.Index, true) - t.Log("✓ Test09_PartialAggregatorRecords_LeafIndexPreserved passed") + t.Log("✓ Test06_MultipleUnfinalizedBlocks passed") } // ============================================================================ -// Test 10: Partial SMT Nodes - Verify Correct Detection of Missing Nodes +// Test 6: Partial SMT Nodes - Verify Correct Detection of Missing Nodes // ============================================================================ func (s *RecoveryTestSuite) Test10_PartialSmtNodes_CorrectDetection() { t := s.T() // Create unfinalized block with 5 commitments - commitments, block, stateIDs := s.createTestData(10, 5, "t10partial") + commitments, block, _ := s.createTestData(10, 5, "t10partial") // Store block (unfinalized) err := s.storage.BlockStorage().Store(s.ctx, block) require.NoError(t, err) - // Store block records - blockRecords := models.NewBlockRecords(block.Index, stateIDs) - err = s.storage.BlockRecordsStorage().Store(s.ctx, blockRecords) - require.NoError(t, err) - // Store ALL aggregator records (so only SMT nodes need recovery) - s.storeAggregatorRecords(commitments, block.Index) + s.storeAggregatorRecords(commitments, block) // Store ONLY SMT nodes at positions 0, 1, and 4 (missing 2 and 3) existingIndices := []int{0, 1, 4} @@ -727,13 +714,16 @@ func (s *RecoveryTestSuite) Test14_CleanupAllProcessed() { // Create commitments commitments, block, _ := s.createTestData(14, 3, "t14") + block.Finalized = true + block.Status = models.FinalityStatusFinalized // Store in Redis and make them pending s.storeCommitmentsInRedis(commitments) s.readPendingToMakeThemClaimed() - // Store AggregatorRecords (simulate block was finalized) - s.storeAggregatorRecords(commitments, block.Index) + // Store finalized block and records (simulate block was finalized). + require.NoError(t, s.storage.BlockStorage().Store(s.ctx, block)) + s.storeAggregatorRecords(commitments, block) // Verify we have 3 unprocessed count, err := s.commitmentQueue.CountUnprocessed(s.ctx) @@ -759,16 +749,18 @@ func (s *RecoveryTestSuite) Test15_CleanupMixed() { t := s.T() // Create 4 commitments - commitments := testutil.CreateTestCertificationRequests(t, 4, "t15") - blockNumber := api.NewBigInt(big.NewInt(15)) + commitments, block, _ := s.createTestData(15, 4, "t15") + block.Finalized = true + block.Status = models.FinalityStatusFinalized // Store all 4 in Redis and make them pending s.storeCommitmentsInRedis(commitments) s.readPendingToMakeThemClaimed() - // Store only first 2 in AggregatorRecords (simulate partial processing) + // Store only first 2 in finalized AggregatorRecords (simulate partial processing) processedCommitments := commitments[:2] - s.storeAggregatorRecords(processedCommitments, blockNumber) + require.NoError(t, s.storage.BlockStorage().Store(s.ctx, block)) + s.storeAggregatorRecords(processedCommitments, block) // Verify we have 4 unprocessed count, err := s.commitmentQueue.CountUnprocessed(s.ctx) @@ -794,13 +786,12 @@ func (s *RecoveryTestSuite) Test16_RecoveryCallsCleanup() { t := s.T() // Create a FINALIZED block with records - commitments, block, stateIDs := s.createTestData(16, 3, "t16") + commitments, block, _ := s.createTestData(16, 3, "t16") block.Finalized = true + block.Status = models.FinalityStatusFinalized err := s.storage.BlockStorage().Store(s.ctx, block) require.NoError(t, err) - err = s.storage.BlockRecordsStorage().Store(s.ctx, models.NewBlockRecords(block.Index, stateIDs)) - require.NoError(t, err) - s.storeAggregatorRecords(commitments, block.Index) + s.storeAggregatorRecords(commitments, block) s.storeSmtNodes(commitments) // Store commitments in Redis as pending (simulating MarkProcessed failure) diff --git a/internal/round/round_manager.go b/internal/round/round_manager.go index 697fe81..d8198ed 100644 --- a/internal/round/round_manager.go +++ b/internal/round/round_manager.go @@ -10,6 +10,7 @@ import ( "sync/atomic" "time" + "github.com/google/uuid" "github.com/unicitynetwork/bft-go-base/types" "github.com/unicitynetwork/aggregator-go/internal/bft" @@ -35,7 +36,7 @@ const ( RoundStateFinalizing // Finalizing block ) -const miniBatchSize = 100 // Number of commitments to process per SMT mini-batch +const defaultMiniBatchSize = 500 // Number of commitments to process per SMT mini-batch func (rs RoundState) String() string { switch rs { @@ -72,6 +73,10 @@ type Round struct { // PreCollected is true when this round starts from a precollector snapshot. // These rounds skip the fixed collect window. PreCollected bool + // ProposalRecordsStaged is true when the precollector already wrote the + // round's proposed aggregator records before the certification request. + ProposalRecordsStaged bool + ProposalID string // Timing metrics for this round ProcessingTime time.Duration ProposalTime time.Time // When block was proposed to BFT @@ -289,7 +294,7 @@ func (rm *RoundManager) Start(ctx context.Context) error { // Nothing verified to publish yet; BlockSyncer refreshes the proof view per replayed block. rm.logger.Info("Stale disk follower; skipping unfinalized-block recovery, BlockSyncer will catch up") } else { - recoveryResult, err := RecoverUnfinalizedBlock(ctx, rm.logger, rm.storage, rm.commitmentQueue) + recoveryResult, err := recoverUnfinalizedBlock(ctx, rm.logger, rm.storage, rm.commitmentQueue, false) if err != nil { return fmt.Errorf("failed to recover unfinalized block: %w", err) } @@ -408,6 +413,17 @@ func (rm *RoundManager) GetSMTBackend() smtbackend.Backend { return rm.smtBackend } +func (rm *RoundManager) CommittedRoot(ctx context.Context) ([]byte, *api.BigInt, error) { + if rm.smtBackend == nil { + return nil, nil, fmt.Errorf("SMT backend is not initialized") + } + state, err := rm.smtBackend.CommittedState(ctx) + if err != nil { + return nil, nil, err + } + return state.RootHash, state.BlockNumber, nil +} + // FinalizationReadLock blocks only during the commit+finalize window to avoid root/block mismatches in proofs. func (rm *RoundManager) FinalizationReadLock() func() { rm.finalizationMu.RLock() @@ -491,8 +507,8 @@ func (rm *RoundManager) GetStats() map[string]interface{} { defer rm.roundMutex.RUnlock() stats := map[string]interface{}{ - "totalRounds": rm.totalRounds, - "totalCommitments": rm.totalCommitments, + "totalRounds": atomic.LoadInt64(&rm.totalRounds), + "totalCommitments": atomic.LoadInt64(&rm.totalCommitments), "roundDuration": rm.roundDuration.String(), } @@ -513,13 +529,19 @@ func (rm *RoundManager) GetStats() map[string]interface{} { func (rm *RoundManager) StartNewRound(ctx context.Context, roundNumber *api.BigInt) error { abandonPendingRound := rm.hasPendingRoundToAbandon(roundNumber) if abandonPendingRound { + rm.roundMutex.Lock() + supersededSnapshot := rm.abandonSupersededRoundLocked(roundNumber) + rm.roundMutex.Unlock() + if supersededSnapshot != nil { + supersededSnapshot.Discard(ctx) + } resetDone := rm.discardActivePrecollector(ctx) if !resetDone { rm.clearProofPending() rm.resetRedisPendingSweep(ctx) } } - return rm.StartNewRoundWithSnapshot(ctx, roundNumber, nil, nil, nil) + return rm.StartNewRoundWithSnapshot(ctx, roundNumber, nil, nil, nil, false, "") } func (rm *RoundManager) hasPendingRoundToAbandon(roundNumber *api.BigInt) bool { @@ -548,6 +570,8 @@ func (rm *RoundManager) StartNewRoundWithSnapshot( snapshot smtbackend.Snapshot, commitments []*models.CertificationRequest, leaves []smtbackend.LeafInput, + proposalRecordsStaged bool, + proposalID string, ) error { rm.logger.WithContext(ctx).Info("StartNewRound called", "roundNumber", roundNumber.String(), @@ -632,24 +656,29 @@ func (rm *RoundManager) StartNewRoundWithSnapshot( supersededSnapshot := rm.abandonSupersededRoundLocked(roundNumber) processCtx, processCancel := context.WithCancel(roundCtx) + if proposalID == "" { + proposalID = uuid.NewString() + } round := &Round{ - Number: roundNumber, - StartTime: time.Now(), - State: RoundStateProcessing, - Commitments: commitments, - Cancel: processCancel, - Snapshot: snapshot, - PendingLeaves: leaves, - PendingCommitments: commitments, // In child mode, commitments are already filtered by pre-collection - PreCollected: preCollected, + Number: roundNumber, + StartTime: time.Now(), + State: RoundStateProcessing, + Commitments: commitments, + Cancel: processCancel, + Snapshot: snapshot, + PendingLeaves: leaves, + PendingCommitments: commitments, // In child mode, commitments are already filtered by pre-collection + PreCollected: preCollected, + ProposalRecordsStaged: proposalRecordsStaged, + ProposalID: proposalID, } rm.currentRound = round // Start precollector on the first child-mode round so it begins collecting // commitments into a chained snapshot while the current round processes. if rm.config.Sharding.Mode.IsChild() && rm.precollector == nil && !rm.precollectorDisabled { - rm.startPrecollectorLocked(roundCtx, snapshot) + rm.startPrecollectorLocked(roundCtx, snapshot, nil) } rm.roundWG.Go(func() { @@ -801,7 +830,15 @@ func (rm *RoundManager) processRound(ctx context.Context, round *Round) error { if collectDuration <= 0 { collectDuration = 200 * time.Millisecond } + miniBatchSize := rm.collectMiniBatchSize() deadline := time.Now().Add(collectDuration) + flushCalls := 0 + flushAdded := 0 + var flushTotal time.Duration + var flushMax time.Duration + var finalFlush time.Duration + finalFlushAdded := 0 + collectedCount := 0 for time.Now().Before(deadline) { select { @@ -813,10 +850,20 @@ func (rm *RoundManager) processRound(ctx context.Context, round *Round) error { } round.Commitments = append(round.Commitments, commitment) var dropped []interfaces.CertificationRequestAck - if len(round.Commitments)%100 == 0 { - batch := round.Commitments[len(round.Commitments)-100:] + if len(round.Commitments)%miniBatchSize == 0 { + batch := round.Commitments[len(round.Commitments)-miniBatchSize:] var err error + pendingBefore := len(round.PendingCommitments) + flushStart := time.Now() dropped, err = rm.processMiniBatchForRound(ctx, round, batch) + flushElapsed := time.Since(flushStart) + added := len(round.PendingCommitments) - pendingBefore + flushCalls++ + flushAdded += added + flushTotal += flushElapsed + if flushElapsed > flushMax { + flushMax = flushElapsed + } if err != nil { rm.roundMutex.Unlock() return err @@ -836,19 +883,43 @@ func (rm *RoundManager) processRound(ctx context.Context, round *Round) error { rm.roundMutex.Unlock() return nil } - remaining := len(round.Commitments) % 100 + remaining := len(round.Commitments) % miniBatchSize var dropped []interfaces.CertificationRequestAck if remaining > 0 { batch := round.Commitments[len(round.Commitments)-remaining:] var err error + pendingBefore := len(round.PendingCommitments) + flushStart := time.Now() dropped, err = rm.processMiniBatchForRound(ctx, round, batch) + finalFlush = time.Since(flushStart) + finalFlushAdded = len(round.PendingCommitments) - pendingBefore + flushCalls++ + flushAdded += finalFlushAdded + flushTotal += finalFlush + if finalFlush > flushMax { + flushMax = finalFlush + } if err != nil { rm.roundMutex.Unlock() return err } } + collectedCount = len(round.Commitments) rm.roundMutex.Unlock() ackDroppedCommitments(ctx, rm.logger, rm.commitmentQueue, dropped) + if flushCalls > 0 { + rm.logger.WithContext(ctx).Debug("Round collection flush", + "roundNumber", roundNumber.String(), + "commitments", collectedCount, + "miniBatchSize", miniBatchSize, + "flushCalls", flushCalls, + "flushAdded", flushAdded, + "flushTotal", flushTotal.String(), + "flushMax", flushMax.String(), + "finalFlush", finalFlush.String(), + "finalFlushAdded", finalFlushAdded, + "collectDuration", collectDuration.String()) + } } rm.startActivePrecollectorIfNeeded(ctx, round) @@ -888,8 +959,8 @@ func (rm *RoundManager) processRound(ctx context.Context, round *Round) error { } proposed = true - rm.totalRounds++ - rm.totalCommitments += int64(commitmentCount) + atomic.AddInt64(&rm.totalRounds, 1) + atomic.AddInt64(&rm.totalCommitments, int64(commitmentCount)) rm.logger.WithContext(ctx).Info("Round completed", "roundNumber", roundNumber.String(), @@ -1140,6 +1211,16 @@ func (rm *RoundManager) restoreSmtFromStorage(ctx context.Context) (*api.BigInt, func (rm *RoundManager) Activate(ctx context.Context) error { rm.logger.WithContext(ctx).Info("Activating round manager") + rm.roundMutex.RLock() + alreadyActive := rm.activeCancel != nil + rm.roundMutex.RUnlock() + if alreadyActive { + rm.logger.WithContext(ctx).Warn("Round manager already active, deactivating previous activation") + if err := rm.Deactivate(ctx); err != nil { + return fmt.Errorf("failed to deactivate previous activation: %w", err) + } + } + activeCtx, activeCancel := context.WithCancel(ctx) activated := false defer func() { @@ -1147,18 +1228,13 @@ func (rm *RoundManager) Activate(ctx context.Context) error { return } activeCancel() - rm.roundMutex.Lock() - if rm.activeCtx == activeCtx { - rm.activeCtx = nil - rm.activeCancel = nil + if err := rm.Deactivate(ctx); err != nil { + rm.logger.WithContext(ctx).Error("Failed to deactivate after activation failure", + "error", err.Error()) } - rm.roundMutex.Unlock() }() rm.roundMutex.Lock() - if rm.activeCancel != nil { - rm.activeCancel() - } rm.activeCtx = activeCtx rm.activeCancel = activeCancel rm.precollectorDisabled = false @@ -1168,7 +1244,7 @@ func (rm *RoundManager) Activate(ctx context.Context) error { rm.roundMutex.Unlock() if rm.config.HA.Enabled { - recoveryResult, err := RecoverUnfinalizedBlock(activeCtx, rm.logger, rm.storage, rm.commitmentQueue) + recoveryResult, err := recoverUnfinalizedBlock(activeCtx, rm.logger, rm.storage, rm.commitmentQueue, !rm.usesDiskSMTBackend()) if err != nil { return fmt.Errorf("failed to recover unfinalized block on activation: %w", err) } @@ -1311,9 +1387,6 @@ func (rm *RoundManager) Deactivate(ctx context.Context) error { } func (rm *RoundManager) usesActivePrecollector() bool { - if rm.config.Processing.PrecollectorGracePeriod <= 0 { - return false - } if !rm.config.Storage.UseRedisForCommitments { return false } @@ -1397,7 +1470,7 @@ func (rm *RoundManager) startActivePrecollectorIfNeeded(ctx context.Context, rou if rm.activeCtx != nil { precollectorCtx = rm.activeCtx } - rm.startPrecollectorLocked(precollectorCtx, round.Snapshot) + rm.startPrecollectorLocked(precollectorCtx, round.Snapshot, incrementRoundNumber(round.Number)) rm.roundMutex.Unlock() } @@ -1405,32 +1478,58 @@ func (rm *RoundManager) startActivePrecollectorIfNeeded(ctx context.Context, rou // the owned fork. The fork is taken under roundMutex so the base snapshot cannot // be discarded while the precollector forks it. Caller must hold roundMutex; on // fork failure the precollector is not started. -func (rm *RoundManager) startPrecollectorLocked(ctx context.Context, base smtbackend.Snapshot) { +func (rm *RoundManager) startPrecollectorLocked(ctx context.Context, base smtbackend.Snapshot, blockNumber *api.BigInt) { forked, err := base.Fork(ctx) if err != nil { rm.logger.WithContext(ctx).Error("Failed to fork precollector snapshot", "error", err.Error()) return } - cp := rm.newActivePrecollector() + cp := rm.newActivePrecollector(blockNumber) rm.precollector = cp cp.Start(ctx, forked) } -func (rm *RoundManager) newActivePrecollector() *childPrecollector { +func (rm *RoundManager) newActivePrecollector(blockNumber *api.BigInt) *childPrecollector { cp := newChildPrecollector( rm.commitmentStream, rm.commitmentQueue, rm.logger, rm.config.Processing.MaxCommitmentsPerRound, rm.markProofsPending, + blockNumber, + "", + rm.stageProposedCommitments, ) return cp } +func (rm *RoundManager) collectMiniBatchSize() int { + if rm == nil || rm.config == nil || rm.config.Processing.CollectMiniBatchSize <= 0 { + return defaultMiniBatchSize + } + return rm.config.Processing.CollectMiniBatchSize +} + func (rm *RoundManager) advancePrecollectorForHandoff(cp *childPrecollector) (*preCollectionResult, error) { return cp.AdvanceRound() } +func validatePrecollectorBlockNumber(preResult *preCollectionResult, roundNumber *api.BigInt) error { + if preResult == nil || !preResult.recordsStaged { + return nil + } + if preResult.blockNumber == nil || preResult.blockNumber.Int == nil { + return fmt.Errorf("precollector staged records without a block number") + } + if roundNumber == nil || roundNumber.Int == nil { + return fmt.Errorf("precollector staged records for block %s cannot start a nil round", preResult.blockNumber.String()) + } + if preResult.blockNumber.Cmp(roundNumber.Int) != 0 { + return fmt.Errorf("precollector staged records for block %s cannot start round %s", preResult.blockNumber.String(), roundNumber.String()) + } + return nil +} + // StartNextRoundFromPrecollector starts the next standalone/bft-shard round // from the active precollector snapshot. If precollection is disabled or no // precollector is available, it falls back to the fixed collect path. @@ -1484,13 +1583,22 @@ func (rm *RoundManager) StartNextRoundFromPrecollector(ctx context.Context, roun rm.logger.WithContext(ctx).Warn("Failed to advance precollector, falling back to fixed collect round", "error", err.Error(), "advanceDuration", advanceDuration.String()) + // The precollector failed to hand off; discard it here so a fresh one + // can start. StartNewRound only discards when abandoning a pending + // round, which is not the case on the post-finalization fallback path. + rm.discardActivePrecollector(ctx) return rm.StartNewRound(ctx, roundNumber) } + if err := validatePrecollectorBlockNumber(preResult, roundNumber); err != nil { + preResult.snapshot.Discard(ctx) + return err + } if err := preResult.snapshot.SetCommitTarget(ctx, rm.smtBackend); err != nil { + preResult.snapshot.Discard(ctx) return fmt.Errorf("failed to set precollector commit target: %w", err) } - if err := rm.StartNewRoundWithSnapshot(ctx, roundNumber, preResult.snapshot, preResult.commitments, preResult.leaves); err != nil { + if err := rm.StartNewRoundWithSnapshot(ctx, roundNumber, preResult.snapshot, preResult.commitments, preResult.leaves, preResult.recordsStaged, preResult.proposalID); err != nil { if errors.Is(err, ErrDeactivated) { return nil } @@ -1499,6 +1607,13 @@ func (rm *RoundManager) StartNextRoundFromPrecollector(ctx context.Context, roun return nil } +func incrementRoundNumber(roundNumber *api.BigInt) *api.BigInt { + if roundNumber == nil || roundNumber.Int == nil { + return nil + } + return api.NewBigInt(new(big.Int).Add(roundNumber.Int, big.NewInt(1))) +} + func (rm *RoundManager) waitBeforePrecollectorHandoff(ctx context.Context, done <-chan struct{}) error { grace := rm.config.Processing.PrecollectorGracePeriod if grace <= 0 { diff --git a/internal/round/round_process_regression_test.go b/internal/round/round_process_regression_test.go index 432383f..0332a31 100644 --- a/internal/round/round_process_regression_test.go +++ b/internal/round/round_process_regression_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/unicitynetwork/aggregator-go/internal/bft" "github.com/unicitynetwork/aggregator-go/internal/config" "github.com/unicitynetwork/aggregator-go/internal/events" "github.com/unicitynetwork/aggregator-go/internal/ha/state" @@ -63,6 +64,24 @@ func (c *recordingBFTClient) snapshot() []recordedBFTProposal { return append([]recordedBFTProposal(nil), c.proposals...) } +type staleBFTClient struct { + called chan struct{} +} + +func (c *staleBFTClient) Start(context.Context) error { return nil } +func (c *staleBFTClient) Stop() {} +func (c *staleBFTClient) WaitForInitialized(context.Context) error { + return nil +} + +func (c *staleBFTClient) CertificationRequest(context.Context, *models.Block) error { + select { + case c.called <- struct{}{}: + default: + } + return bft.ErrStaleCertificationRound +} + func TestRoundProcessingUsesScheduledRoundSnapshot(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -127,6 +146,8 @@ func TestRoundProcessingUsesScheduledRoundSnapshot(t *testing.T) { roundTwoSnapshot, []*models.CertificationRequest{roundTwoCommitment}, []smtbackend.LeafInput{roundTwoLeaf}, + false, + "", )) require.Eventually(t, func() bool { @@ -146,6 +167,28 @@ func TestRoundProcessingUsesScheduledRoundSnapshot(t *testing.T) { require.Len(t, proposals, 1) require.EqualValues(t, 2, proposals[0].blockNumber) require.True(t, bytes.Equal(proposals[0].rootHash, roundTwoRoot)) + + finalizedBlock, err := storage.BlockStorage().GetByNumber(ctx, api.NewBigInt(big.NewInt(2))) + require.NoError(t, err) + require.Nil(t, finalizedBlock) + + proposedBlock, err := getBlockAnyFinalization(ctx, storage, api.NewBigInt(big.NewInt(2))) + require.NoError(t, err) + require.NotNil(t, proposedBlock) + require.Equal(t, models.FinalityStatusProposed, proposedBlock.Status) + + visibleRecord, err := storage.AggregatorRecordStorage().GetByStateID(ctx, roundTwoCommitment.StateID) + require.NoError(t, err) + require.Nil(t, visibleRecord) + + proposedRecord, err := getAggregatorRecordAnyFinalization(ctx, storage, roundTwoCommitment.StateID) + require.NoError(t, err) + require.NotNil(t, proposedRecord) + require.Equal(t, proposedBlock.ProposalID, proposedRecord.ProposalID) + + visibleRecords, err := storage.AggregatorRecordStorage().GetByBlockNumber(ctx, api.NewBigInt(big.NewInt(2))) + require.NoError(t, err) + require.Empty(t, visibleRecords) } func TestStartNewRoundAbandonsSupersededPendingRound(t *testing.T) { @@ -213,6 +256,84 @@ func TestStartNewRoundAbandonsSupersededPendingRound(t *testing.T) { require.False(t, pending) } +func TestStaleCertificationRequestAbandonsStoredDurableProposal(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + cfg := config.Config{ + Database: config.DatabaseConfig{ + Database: "test_stale_certification_request_abandons_durable_proposal", + }, + Processing: config.ProcessingConfig{ + CollectPhaseDuration: time.Hour, + CommitmentStreamBufferSize: 16, + MaxCommitmentsPerRound: 1000, + }, + Sharding: config.ShardingConfig{Mode: config.ShardingModeBFTShard}, + } + storage := testutil.SetupTestStorage(t, cfg) + testLogger := newTestLogger(t) + threadSafeSMT := smt.NewThreadSafeSMT(smt.NewSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits)) + rm, err := NewRoundManager( + ctx, + &cfg, + testLogger, + storage.CommitmentQueue(), + storage, + nil, + state.NewSyncStateTracker(), + nil, + events.NewEventBus(testLogger), + threadSafeSMT, + nil, + ) + require.NoError(t, err) + client := &staleBFTClient{called: make(chan struct{}, 1)} + rm.bftClient = client + defer func() { + cancel() + rm.roundWG.Wait() + }() + + commitment := testutil.CreateTestCertificationRequest(t, "stale_durable_proposal") + leaf, err := commitmentLeafInput(commitment) + require.NoError(t, err) + snapshot, err := rm.smtBackend.CreateSnapshot(ctx) + require.NoError(t, err) + result, err := snapshot.AddLeavesClassified(ctx, []smtbackend.LeafInput{leaf}) + require.NoError(t, err) + require.NoError(t, result.ValidateAllAccepted(1)) + + require.NoError(t, rm.StartNewRoundWithSnapshot( + ctx, + api.NewBigInt(big.NewInt(1)), + snapshot, + []*models.CertificationRequest{commitment}, + []smtbackend.LeafInput{leaf}, + false, + "", + )) + + select { + case <-client.called: + case <-time.After(2 * time.Second): + t.Fatal("expected stale BFT client to be called") + } + require.Eventually(t, func() bool { + block, err := getBlockAnyFinalization(ctx, storage, api.NewBigInt(big.NewInt(1))) + require.NoError(t, err) + return block != nil && block.Status == models.FinalityStatusAbandoned + }, 2*time.Second, 10*time.Millisecond) + + visibleRecord, err := storage.AggregatorRecordStorage().GetByStateID(ctx, commitment.StateID) + require.NoError(t, err) + require.Nil(t, visibleRecord) + + _, found, err := rm.LoadDurableProposal(ctx, api.NewBigInt(big.NewInt(1))) + require.NoError(t, err) + require.False(t, found) +} + func TestStartNewRoundRetriesEqualFinalizingRoundProposal(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) @@ -330,7 +451,14 @@ func TestProposeBlockLinksToLatestFinalizedBlockAcrossRoundGap(t *testing.T) { require.NoError(t, storage.BlockStorage().Store(ctx, block1)) rootHash := api.HexBytes(bytes.Repeat([]byte{0x33}, 32)) - require.NoError(t, rm.proposeBlock(ctx, nil, api.NewBigInt(big.NewInt(3)), rootHash)) + round := &Round{ + Number: api.NewBigInt(big.NewInt(3)), + StartTime: time.Now(), + State: RoundStateProcessing, + ProposalID: "gap-proposal-3", + } + rm.currentRound = round + require.NoError(t, rm.proposeBlock(ctx, round, api.NewBigInt(big.NewInt(3)), rootHash)) proposals := recorder.snapshot() require.Len(t, proposals, 1) diff --git a/internal/round/smt_persistence_integration_test.go b/internal/round/smt_persistence_integration_test.go index aa1ca70..5d7a62a 100644 --- a/internal/round/smt_persistence_integration_test.go +++ b/internal/round/smt_persistence_integration_test.go @@ -230,6 +230,7 @@ func TestCompleteWorkflowWithRestart(t *testing.T) { api.HexBytes{}, nil, ) + storeDurableProposalForCurrentRound(t, ctx, rm, block) err = rm.FinalizeBlock(ctx, block) require.NoError(t, err, "FinalizeBlock should succeed") diff --git a/internal/service/aggregate_test.go b/internal/service/aggregate_test.go index d7903a6..d156bd4 100644 --- a/internal/service/aggregate_test.go +++ b/internal/service/aggregate_test.go @@ -78,7 +78,6 @@ func TestGetBlockTotalCommitments(t *testing.T) { BlockNumber: api.NewBigInt(big.NewInt(1)), LeafIndex: api.NewBigInt(big.NewInt(0)), CreatedAt: api.Now(), - FinalizedAt: api.Now(), } apiRecord := modelToAPIAggregatorRecord(modelRecord) diff --git a/internal/service/proof_diagnostics.go b/internal/service/proof_diagnostics.go index 3420327..82b4b8d 100644 --- a/internal/service/proof_diagnostics.go +++ b/internal/service/proof_diagnostics.go @@ -2,10 +2,11 @@ package service import ( "context" - "sync/atomic" + "errors" "time" "github.com/unicitynetwork/aggregator-go/internal/metrics" + smtbackend "github.com/unicitynetwork/aggregator-go/internal/smt/backend" ) const ( @@ -21,102 +22,50 @@ const ( proofPathError = "error" ) -type proofDiagnostics struct { - lastLogSecond atomic.Int64 +const ( + proofBuildSourcePublishedView = "published_view" + proofBuildSourceLiveTree = "live_tree" - requests atomic.Int64 - knownNotReady atomic.Int64 - precomputedHit atomic.Int64 - metadataHit atomic.Int64 - metadataMiss atomic.Int64 - mongoBlock atomic.Int64 - mongoRecord atomic.Int64 - emptyResponses atomic.Int64 - liveCert atomic.Int64 - errors atomic.Int64 -} + proofBuildResultOK = "ok" + proofBuildResultNotFound = "not_found" + proofBuildResultRootChanged = "root_changed" + proofBuildResultError = "error" +) -func (as *AggregatorService) recordProofPath(ctx context.Context, path string) { +func (as *AggregatorService) recordProofPath(_ context.Context, path string) { if as == nil { return } metrics.InclusionProofPathTotal.WithLabelValues(path).Inc() +} - switch path { - case proofPathRequest: - as.proofDiag.requests.Add(1) - case proofPathKnownNotReady: - as.proofDiag.knownNotReady.Add(1) - case proofPathPrecomputedHit: - as.proofDiag.precomputedHit.Add(1) - case proofPathMetadataHit: - as.proofDiag.metadataHit.Add(1) - case proofPathMetadataMiss: - as.proofDiag.metadataMiss.Add(1) - case proofPathMongoBlock: - as.proofDiag.mongoBlock.Add(1) - case proofPathMongoRecord: - as.proofDiag.mongoRecord.Add(1) - case proofPathEmpty: - as.proofDiag.emptyResponses.Add(1) - case proofPathLiveCert: - as.proofDiag.liveCert.Add(1) - case proofPathError: - as.proofDiag.errors.Add(1) +func (as *AggregatorService) recordProofBuildDuration(_ context.Context, source, result string, duration time.Duration) { + if as == nil { + return } - - now := time.Now().Unix() - if !as.proofDiag.lastLogSecond.CompareAndSwap(now-1, now) { - last := as.proofDiag.lastLogSecond.Load() - if last == now || !as.proofDiag.lastLogSecond.CompareAndSwap(last, now) { - return - } + if duration < 0 { + duration = 0 } - as.logProofPathSnapshot(ctx) + metrics.SMTInclusionCertBuildDuration.WithLabelValues(source, result).Observe(duration.Seconds()) } -func (as *AggregatorService) logProofPathSnapshot(ctx context.Context) { - requests := as.proofDiag.requests.Swap(0) - knownNotReady := as.proofDiag.knownNotReady.Swap(0) - precomputedHit := as.proofDiag.precomputedHit.Swap(0) - metadataHit := as.proofDiag.metadataHit.Swap(0) - metadataMiss := as.proofDiag.metadataMiss.Swap(0) - mongoBlock := as.proofDiag.mongoBlock.Swap(0) - mongoRecord := as.proofDiag.mongoRecord.Swap(0) - emptyResponses := as.proofDiag.emptyResponses.Swap(0) - liveCert := as.proofDiag.liveCert.Swap(0) - errors := as.proofDiag.errors.Swap(0) - - if requests == 0 && - knownNotReady == 0 && - precomputedHit == 0 && - metadataHit == 0 && - metadataMiss == 0 && - mongoBlock == 0 && - mongoRecord == 0 && - emptyResponses == 0 && - liveCert == 0 && - errors == 0 { - return +func proofBuildResultFromPublishedErr(err error) string { + if err == nil { + return proofBuildResultOK } - if as.logger == nil { - return + if errors.Is(err, smtbackend.ErrPublishedProofLeafNotFound) { + return proofBuildResultNotFound } - pending, records, blocks := as.roundManager.GetProofCacheStats() + if errors.Is(err, smtbackend.ErrPublishedProofRootChanged) { + return proofBuildResultRootChanged + } + return proofBuildResultError +} - as.logger.WithContext(ctx).Info("PERF: Proof path", - "requests", requests, - "knownNotReady", knownNotReady, - "precomputedHit", precomputedHit, - "metadataCacheHit", metadataHit, - "metadataCacheMiss", metadataMiss, - "mongoBlockLookup", mongoBlock, - "mongoRecordLookup", mongoRecord, - "emptyResponses", emptyResponses, - "liveCert", liveCert, - "errors", errors, - "proofPending", pending, - "proofCacheRecords", records, - "proofCacheBlocks", blocks) +func proofBuildResultFromErr(err error) string { + if err == nil { + return proofBuildResultOK + } + return proofBuildResultError } diff --git a/internal/service/service.go b/internal/service/service.go index 475e60c..7fc9bba 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "strconv" + "time" "github.com/unicitynetwork/bft-go-base/types" @@ -75,7 +76,6 @@ type AggregatorService struct { leaderSelector LeaderSelector certificationRequestValidator *signing.CertificationRequestValidator trustBaseValidator *trustbase.TrustBaseValidator - proofDiag proofDiagnostics } type LeaderSelector interface { @@ -97,7 +97,6 @@ func modelToAPIAggregatorRecord(modelRecord *models.AggregatorRecord) *api.Aggre BlockNumber: modelRecord.BlockNumber, LeafIndex: modelRecord.LeafIndex, CreatedAt: modelRecord.CreatedAt, - FinalizedAt: modelRecord.FinalizedAt, } } @@ -286,7 +285,9 @@ func (as *AggregatorService) GetInclusionProofV2(ctx context.Context, req *api.G if err != nil { return nil, fmt.Errorf("failed to get published SMT root hash: %w", err) } + certStart := time.Now() childCert, err = publishedProofReader.GetPublishedInclusionCertAtRoot(ctx, rootHash, key) + as.recordProofBuildDuration(ctx, proofBuildSourcePublishedView, proofBuildResultFromPublishedErr(err), time.Since(certStart)) if errors.Is(err, smtbackend.ErrPublishedProofRootChanged) || errors.Is(err, smtbackend.ErrPublishedProofLeafNotFound) { rootHashRaw := api.HexBytes(rootHash) block, ok := as.roundManager.GetProofReadyBlockByRoot(rootHashRaw) @@ -346,7 +347,7 @@ func (as *AggregatorService) GetInclusionProofV2(ctx context.Context, req *api.G return nil, fmt.Errorf("failed to get aggregator record: %w", err) } } - if record == nil || record.BlockNumber.Cmp(block.Index.Int) > 0 { + if record == nil || record.BlockNumber == nil || record.BlockNumber.Cmp(block.Index.Int) > 0 { // Non-inclusion is not implemented yet. Return an empty v2 proof // payload so verifiers short-circuit with ErrExclusionNotImpl. as.recordProofPath(ctx, proofPathEmpty) @@ -358,7 +359,9 @@ func (as *AggregatorService) GetInclusionProofV2(ctx context.Context, req *api.G } if childCert == nil { + certStart := time.Now() childCert, err = smtBackend.GetInclusionCert(ctx, key) + as.recordProofBuildDuration(ctx, proofBuildSourceLiveTree, proofBuildResultFromErr(err), time.Since(certStart)) if err != nil { return nil, fmt.Errorf("failed to build inclusion cert for state ID %s: %w", req.StateID, err) } diff --git a/internal/service/service_test.go b/internal/service/service_test.go index 33c9214..7aaf5e6 100644 --- a/internal/service/service_test.go +++ b/internal/service/service_test.go @@ -419,7 +419,6 @@ func TestGetInclusionProofV2Child_ComposesParentFragment(t *testing.T) { BlockNumber: api.NewBigIntFromUint64(1), LeafIndex: api.NewBigIntFromUint64(0), CreatedAt: api.Now(), - FinalizedAt: api.Now(), } service := newAggregatorServiceForTest(t, shardingCfg, childTree) @@ -663,7 +662,6 @@ func TestGetInclusionProofUsesCachedProofMetadata(t *testing.T) { BlockNumber: api.NewBigIntFromUint64(9), LeafIndex: api.NewBigIntFromUint64(0), CreatedAt: api.Now(), - FinalizedAt: api.Now(), } blockStorage := &testBlockStorage{latestByRoot: map[string]*models.Block{rootHash.String(): block}} @@ -1156,14 +1154,13 @@ type testStorage struct { func (s *testStorage) AggregatorRecordStorage() interfaces.AggregatorRecordStorage { return s.recordStorage } -func (s *testStorage) BlockStorage() interfaces.BlockStorage { return s.blockStorage } -func (s *testStorage) SmtStorage() interfaces.SmtStorage { return nil } -func (s *testStorage) BlockRecordsStorage() interfaces.BlockRecordsStorage { return nil } -func (s *testStorage) LeadershipStorage() interfaces.LeadershipStorage { return nil } -func (s *testStorage) TrustBaseStorage() interfaces.TrustBaseStorage { return nil } -func (s *testStorage) Initialize(context.Context) error { return nil } -func (s *testStorage) Ping(context.Context) error { return nil } -func (s *testStorage) Close(context.Context) error { return nil } +func (s *testStorage) BlockStorage() interfaces.BlockStorage { return s.blockStorage } +func (s *testStorage) SmtStorage() interfaces.SmtStorage { return nil } +func (s *testStorage) LeadershipStorage() interfaces.LeadershipStorage { return nil } +func (s *testStorage) TrustBaseStorage() interfaces.TrustBaseStorage { return nil } +func (s *testStorage) Initialize(context.Context) error { return nil } +func (s *testStorage) Ping(context.Context) error { return nil } +func (s *testStorage) Close(context.Context) error { return nil } func (s *testStorage) WithTransaction(ctx context.Context, fn func(context.Context) error) error { return fn(ctx) } @@ -1184,6 +1181,9 @@ func (s *testBlockStorage) Count(context.Context) (int64, error) func (s *testBlockStorage) GetRange(context.Context, *api.BigInt, *api.BigInt) ([]*models.Block, error) { return nil, nil } +func (s *testBlockStorage) GetNextFinalizedAfter(context.Context, *api.BigInt, *api.BigInt) (*models.Block, error) { + return nil, nil +} func (s *testBlockStorage) SetFinalized(context.Context, *api.BigInt, bool) error { return nil } func (s *testBlockStorage) GetUnfinalized(context.Context) ([]*models.Block, error) { return nil, nil diff --git a/internal/smt/backend/backend.go b/internal/smt/backend/backend.go index 87a1027..00fdf36 100644 --- a/internal/smt/backend/backend.go +++ b/internal/smt/backend/backend.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "time" "github.com/unicitynetwork/aggregator-go/internal/smt" "github.com/unicitynetwork/aggregator-go/pkg/api" @@ -54,6 +55,21 @@ type ProofViewPreparingSnapshot interface { CommitAndPrepareProofView(ctx context.Context, meta CommitMetadata) (PreparedProofView, error) } +type CommitTimingProvider interface { + LastCommitTiming() CommitTiming +} + +type CommitTiming struct { + CollectDuration time.Duration + TombstoneDuration time.Duration + BatchBuildDuration time.Duration + RootHashDuration time.Duration + EngineWriteDuration time.Duration + CacheUpdateDuration time.Duration + NodeWrites int + NodeDeletes int +} + type ProofViewPublisher interface { // RefreshPublishedProofView publishes the backend's current committed root as // the served proof view. expectedRoot, when non-nil, must equal the backend's diff --git a/internal/smt/backend/disk_backend.go b/internal/smt/backend/disk_backend.go index b8b1aea..082c2d1 100644 --- a/internal/smt/backend/disk_backend.go +++ b/internal/smt/backend/disk_backend.go @@ -22,9 +22,9 @@ type DiskBackend struct { store storage.Store tree *persist.Tree - mu sync.Mutex - lastCommitStats persist.CommitStats - snapshotWarning sync.Once + lastCommitStatsMu sync.RWMutex + lastCommitStats persist.CommitStats + snapshotWarning sync.Once proofViewMu sync.RWMutex proofView *diskProofView @@ -57,8 +57,6 @@ func (b *DiskBackend) RootHashRaw(context.Context) ([]byte, error) { if b == nil || b.tree == nil { return nil, fmt.Errorf("disk SMT backend not initialized") } - b.mu.Lock() - defer b.mu.Unlock() return hashBytes(b.tree.RootHash()), nil } @@ -66,8 +64,6 @@ func (b *DiskBackend) CommittedState(context.Context) (CommittedState, error) { if b == nil || b.store == nil { return CommittedState{}, fmt.Errorf("disk SMT backend not initialized") } - b.mu.Lock() - defer b.mu.Unlock() state, err := b.store.CommittedState() if err != nil { return CommittedState{}, err @@ -82,8 +78,6 @@ func (b *DiskBackend) CreateSnapshot(context.Context) (Snapshot, error) { if b == nil || b.tree == nil { return nil, fmt.Errorf("disk SMT backend not initialized") } - b.mu.Lock() - defer b.mu.Unlock() snapshot, err := b.tree.CreateSnapshot() if err != nil { return nil, err @@ -252,9 +246,9 @@ func (b *DiskBackend) Stats(ctx context.Context) BackendStats { if b == nil { return BackendStats{RootHash: root} } - b.mu.Lock() + b.lastCommitStatsMu.RLock() lastCommitStats := b.lastCommitStats - b.mu.Unlock() + b.lastCommitStatsMu.RUnlock() raw := map[string]any{ "last_commit": lastCommitStats, @@ -289,20 +283,16 @@ func (b *DiskBackend) RefreshPublishedProofView(_ context.Context, expectedRoot if b == nil || b.tree == nil { return fmt.Errorf("disk SMT backend not initialized") } - b.mu.Lock() root := b.tree.RootHash() if expectedRoot != nil && !bytes.Equal(hashBytes(root), expectedRoot) { - b.mu.Unlock() return fmt.Errorf("refusing to publish proof view: disk SMT root %s does not match expected finalized root %s", api.HexBytes(hashBytes(root)).String(), api.HexBytes(expectedRoot).String()) } view, err := b.prepareProofViewLocked(root) if err != nil { - b.mu.Unlock() return err } old := b.swapProofView(view) - b.mu.Unlock() if old != nil { old.close() } @@ -375,15 +365,12 @@ func (p *preparedDiskProofView) Publish(context.Context) error { return fmt.Errorf("disk SMT prepared proof view is not initialized") } // Refuse to publish if the disk advanced since prepare; that would roll the served view backward. - p.target.mu.Lock() current := p.target.tree.RootHash() if current != p.view.root { - p.target.mu.Unlock() return fmt.Errorf("refusing to publish prepared proof view: disk SMT root advanced to %x since the view was prepared at %x", current, p.view.root) } old := p.target.swapProofView(p.view) p.view = nil - p.target.mu.Unlock() if old != nil { old.close() } @@ -406,8 +393,6 @@ func (s *DiskSnapshot) AddLeavesClassified(_ context.Context, inputs []LeafInput if s == nil || s.snapshot == nil || s.target == nil { return BatchApplyResult{}, fmt.Errorf("disk SMT snapshot not initialized") } - s.target.mu.Lock() - defer s.target.mu.Unlock() result, err := s.snapshot.AddLeaves(toDiskLeafInputs(inputs)) if err != nil { return BatchApplyResult{}, err @@ -426,8 +411,6 @@ func (s *DiskSnapshot) Fork(context.Context) (Snapshot, error) { if s == nil || s.snapshot == nil || s.target == nil { return nil, fmt.Errorf("disk SMT snapshot not initialized") } - s.target.mu.Lock() - defer s.target.mu.Unlock() child, err := s.snapshot.Fork() if err != nil { return nil, err @@ -449,8 +432,6 @@ func (s *DiskSnapshot) SetCommitTarget(_ context.Context, target Backend) error if diskTarget == nil || diskTarget.tree == nil { return fmt.Errorf("disk snapshot commit target is not initialized") } - diskTarget.mu.Lock() - defer diskTarget.mu.Unlock() if err := s.snapshot.SetCommitTarget(diskTarget.tree); err != nil { return err } @@ -467,12 +448,27 @@ func (s *DiskSnapshot) CommitAndPrepareProofView(_ context.Context, meta CommitM return s.commit(meta, true) } +func (s *DiskSnapshot) LastCommitTiming() CommitTiming { + if s == nil || s.snapshot == nil { + return CommitTiming{} + } + stats := s.snapshot.LastCommitStats() + return CommitTiming{ + CollectDuration: stats.CollectDuration, + TombstoneDuration: stats.TombstoneDuration, + BatchBuildDuration: stats.BatchBuildDuration, + RootHashDuration: stats.RootHashDuration, + EngineWriteDuration: stats.EngineWriteDuration, + CacheUpdateDuration: stats.CacheUpdateDuration, + NodeWrites: stats.NodeWrites, + NodeDeletes: stats.NodeDeletes, + } +} + func (s *DiskSnapshot) commit(meta CommitMetadata, prepareProofView bool) (PreparedProofView, error) { if s == nil || s.snapshot == nil || s.target == nil { return nil, fmt.Errorf("disk SMT snapshot not initialized") } - s.target.mu.Lock() - defer s.target.mu.Unlock() root := hashBytes(s.snapshot.RootHash()) if meta.RootHash != nil && !bytes.Equal(root, meta.RootHash) { return nil, fmt.Errorf("disk SMT snapshot root %x does not match expected root %x", root, meta.RootHash) @@ -480,7 +476,9 @@ func (s *DiskSnapshot) commit(meta CommitMetadata, prepareProofView bool) (Prepa if err := s.snapshot.Commit(meta.BlockNumber); err != nil { return nil, err } + s.target.lastCommitStatsMu.Lock() s.target.lastCommitStats = s.snapshot.LastCommitStats() + s.target.lastCommitStatsMu.Unlock() if !prepareProofView { return nil, nil } @@ -495,10 +493,6 @@ func (s *DiskSnapshot) Discard(context.Context) { if s == nil || s.snapshot == nil { return } - if s.target != nil { - s.target.mu.Lock() - defer s.target.mu.Unlock() - } s.snapshot.Discard() } diff --git a/internal/smt/disk/persist/tree.go b/internal/smt/disk/persist/tree.go index 8b504e2..1ee902d 100644 --- a/internal/smt/disk/persist/tree.go +++ b/internal/smt/disk/persist/tree.go @@ -25,6 +25,7 @@ type Tree struct { materializeMode string frontierReadMode string materializeWorkers int + trackApplyStats bool nodeCacheDepth int nodeCache map[disk.NodeKey]cachedNode nodeCacheBytes int64 @@ -62,6 +63,9 @@ type Options struct { // MaterializeWorkers bounds concurrent branch materialization in // MaterializeParallel mode. Zero uses GOMAXPROCS. MaterializeWorkers int + // DisableApplyStats skips detailed per-node materialization counters. Keep + // false unless a performance run only needs commit timings. + DisableApplyStats bool } type NodeCacheStats struct { @@ -147,6 +151,7 @@ func OpenWithOptions(store storage.Store, opts Options) (*Tree, error) { materializeMode: opts.MaterializeMode, frontierReadMode: opts.FrontierReadMode, materializeWorkers: opts.MaterializeWorkers, + trackApplyStats: !opts.DisableApplyStats, nodeCacheDepth: opts.NodeCacheDepth, } if opts.NodeCacheDepth >= 0 { @@ -592,42 +597,50 @@ func (s *Snapshot) LastCommitStats() CommitStats { return s.commitStats } -// Fork creates a child snapshot over this snapshot's overlay. The caller must -// preserve the precollector invariant: at most one uncommitted parent snapshot -// may sit between this child and the committed backend. +// Fork creates a child snapshot over this snapshot's overlay. If this snapshot +// already depends on an uncommitted parent overlay, flatten the inherited +// overlays into the child so chained precollection can continue without +// requiring every intermediate snapshot to commit first. func (s *Snapshot) Fork() (*Snapshot, error) { - if s == nil || s.tree == nil { + if s == nil { return nil, fmt.Errorf("disk SMT persist: nil snapshot") } if s.closed { return nil, fmt.Errorf("disk SMT persist: snapshot is closed") } - if s.parentOverlay != nil && s.parent.RootHash() != s.baseRoot { - return nil, fmt.Errorf("disk SMT persist: cannot fork snapshot with uncommitted grandparent overlay") + if s.tree == nil { + return nil, fmt.Errorf("disk SMT persist: nil snapshot") } if s.rootHash != s.baseRoot && len(s.ownOverlay) == 0 { if err := s.rebuildOverlay(); err != nil { return nil, err } } + parentOverlay := cloneOverlay(s.ownOverlay) + if s.parentOverlay != nil && s.parent.RootHash() != s.baseRoot { + parentOverlay = mergeOverlays(s.parentOverlay, s.ownOverlay) + } return &Snapshot{ parent: s.parent, tree: disk.NewTreeWithRoot(stubForRoot(s.rootHash)), baseRoot: s.rootHash, rootHash: s.rootHash, ownOverlay: make(nodeOverlay), - parentOverlay: cloneOverlay(s.ownOverlay), + parentOverlay: parentOverlay, loadedKeys: make(map[disk.NodeKey]struct{}), }, nil } func (s *Snapshot) SetCommitTarget(target *Tree) error { - if s == nil || s.tree == nil { + if s == nil { return fmt.Errorf("disk SMT persist: nil snapshot") } if s.closed { return fmt.Errorf("disk SMT persist: snapshot is closed") } + if s.tree == nil { + return fmt.Errorf("disk SMT persist: nil snapshot") + } if target == nil { return fmt.Errorf("disk SMT persist: nil commit target") } @@ -641,12 +654,15 @@ func (s *Snapshot) SetCommitTarget(target *Tree) error { } func (s *Snapshot) AddLeaves(inputs []disk.LeafInput) (disk.ApplyResult, error) { - if s == nil || s.tree == nil { + if s == nil { return disk.ApplyResult{}, fmt.Errorf("disk SMT persist: nil snapshot") } if s.closed { return disk.ApplyResult{}, fmt.Errorf("disk SMT persist: snapshot is closed") } + if s.tree == nil { + return disk.ApplyResult{}, fmt.Errorf("disk SMT persist: nil snapshot") + } materializedBefore := s.stats.MaterializedNodes nodeReadsBefore := s.stats.NodeReads @@ -713,12 +729,15 @@ func (s *Snapshot) AddLeaves(inputs []disk.LeafInput) (disk.ApplyResult, error) } func (s *Snapshot) Commit(blockNumber *api.BigInt) error { - if s == nil || s.tree == nil || s.parent == nil { + if s == nil { return fmt.Errorf("disk SMT persist: nil snapshot") } if s.closed { return fmt.Errorf("disk SMT persist: snapshot is closed") } + if s.tree == nil || s.parent == nil { + return fmt.Errorf("disk SMT persist: nil snapshot") + } var stats CommitStats stats.CollectDuration = s.commitStats.CollectDuration @@ -785,15 +804,28 @@ func (s *Snapshot) Commit(blockNumber *api.BigInt) error { s.parent.applyNodeCacheUpdate(s.cacheDeletes, s.cacheWrites) stats.CacheUpdateDuration = time.Since(cacheUpdateStart) s.closed = true + s.releaseBuffers() return nil } func (s *Snapshot) Discard() { if s != nil { s.closed = true + s.releaseBuffers() } } +func (s *Snapshot) releaseBuffers() { + s.tree = nil + s.ownOverlay = nil + s.parentOverlay = nil + s.overlayWrites = nil + s.overlayDeletes = nil + s.loadedKeys = nil + s.cacheWrites = nil + s.cacheDeletes = nil +} + func (s *Snapshot) rebuildOverlay() error { if s == nil || s.tree == nil || s.parent == nil { return fmt.Errorf("disk SMT persist: nil snapshot") @@ -809,27 +841,21 @@ func (s *Snapshot) rebuildOverlay() error { collectDuration := time.Since(collectStart) tombstoneStart := time.Now() - deletes := make([]disk.NodeKey, 0, len(s.loadedKeys)) - for key := range s.loadedKeys { - if _, written := writeSet[key]; written { - continue - } - deletes = append(deletes, key) - } + // Disk SMT node storage is append-only from the tree's perspective: updated + // path nodes are overwritten, and obsolete path nodes are left unreachable. + // Keeping old nodes lets speculative child snapshots materialize safely while + // their parent snapshot is being committed. tombstoneDuration := time.Since(tombstoneStart) - overlay := make(nodeOverlay, len(writes)+len(deletes)) - for _, key := range deletes { - overlay[key] = overlayEntry{delete: true} - } + overlay := make(nodeOverlay, len(writes)) for _, write := range writes { overlay[write.Key] = overlayEntry{value: write.Value} } s.ownOverlay = overlay s.overlayWrites = writes - s.overlayDeletes = deletes + s.overlayDeletes = nil s.cacheWrites = cacheWrites - s.cacheDeletes = s.parent.prepareNodeCacheDeletes(s.loadedKeys, writeSet) + s.cacheDeletes = nil s.commitStats.CollectDuration = collectDuration s.commitStats.TombstoneDuration = tombstoneDuration return nil @@ -850,6 +876,24 @@ func cloneOverlay(in nodeOverlay) nodeOverlay { return out } +func mergeOverlays(base, override nodeOverlay) nodeOverlay { + out := cloneOverlay(base) + if len(override) == 0 { + return out + } + if out == nil { + out = make(nodeOverlay, len(override)) + } + for key, entry := range override { + next := overlayEntry{delete: entry.delete} + if entry.value != nil { + next.value = append([]byte(nil), entry.value...) + } + out[key] = next + } + return out +} + func (t *Tree) shouldCacheNode(key disk.NodeKey) bool { return t != nil && t.nodeCacheDepth >= 0 && key.DepthBits() <= t.nodeCacheDepth } @@ -888,20 +932,6 @@ func (t *Tree) newNodeCacheWrites() map[disk.NodeKey]cachedNode { return make(map[disk.NodeKey]cachedNode) } -func (t *Tree) prepareNodeCacheDeletes(loadedKeys map[disk.NodeKey]struct{}, writeSet map[disk.NodeKey]struct{}) []disk.NodeKey { - if t == nil || t.nodeCacheDepth < 0 { - return nil - } - deletes := make([]disk.NodeKey, 0) - for key := range loadedKeys { - if _, written := writeSet[key]; written || !t.shouldCacheNode(key) { - continue - } - deletes = append(deletes, key) - } - return deletes -} - func (t *Tree) applyNodeCacheUpdate(deletes []disk.NodeKey, updates map[disk.NodeKey]cachedNode) { if t == nil || t.nodeCacheDepth < 0 { return @@ -947,11 +977,17 @@ func (s *Snapshot) materializeInputs(inputs []disk.LeafInput) error { seen[key] = struct{}{} keys = append(keys, key) } - sortStart := time.Now() + trackStats := s.trackApplyStats() + var sortStart time.Time + if trackStats { + sortStart = time.Now() + } sort.Slice(keys, func(i, j int) bool { return keyPathLess(keys[i], keys[j]) }) - s.stats.MaterializeSortDuration += time.Since(sortStart) + if trackStats { + s.stats.MaterializeSortDuration += time.Since(sortStart) + } if s.parent != nil && (s.parent.materializeMode == MaterializeFrontier || s.parent.materializeMode == MaterializeParallel) { if s.parent.materializeMode == MaterializeParallel { return s.materializeInputsParallel(keys) @@ -1051,9 +1087,15 @@ func (s *Snapshot) materializeBranchParallel(branch *disk.Branch, nodeKey disk.N return nil, fmt.Errorf("disk SMT persist: malformed internal node") } - routeStart := time.Now() + trackStats := s.trackApplyStats() + var routeStart time.Time + if trackStats { + routeStart = time.Now() + } leftKeys, rightKeys, err := routeSortedKeysForInternal(keys, node.Path, startBit) - s.recordMaterializeRoute(time.Since(routeStart)) + if trackStats { + s.recordMaterializeRoute(time.Since(routeStart)) + } if err != nil { return nil, err } @@ -1181,9 +1223,12 @@ func (s *Snapshot) loadBranchFromOverlay(key disk.NodeKey, expectedHash disk.Has if lockedStats { s.recordMaterialized(key, branch.Kind, int64(len(entry.value)), false, 0, 0, 0, 0, addLoaded) } else { - if addLoaded { + if addLoaded && s.trackLoadedKeys() { s.loadedKeys[key] = struct{}{} } + if !s.trackApplyStats() { + return branch, true, nil + } s.stats.MaterializedNodes++ s.stats.MaterializedBytes += int64(len(entry.value)) switch branch.Kind { @@ -1215,11 +1260,19 @@ func (s *Snapshot) recordParallelMaterialized(key disk.NodeKey, kind disk.Branch } func (s *Snapshot) recordMaterialized(key disk.NodeKey, kind disk.BranchKind, encodedBytes int64, cacheHit bool, cacheBytes int64, readDuration, decodeDuration, hashDuration time.Duration, addLoaded bool) { + trackStats := s.trackApplyStats() + trackLoaded := addLoaded && s.trackLoadedKeys() + if !trackStats && !trackLoaded { + return + } s.statsMu.Lock() defer s.statsMu.Unlock() - if addLoaded { + if trackLoaded { s.loadedKeys[key] = struct{}{} } + if !trackStats { + return + } s.stats.MaterializedNodes++ s.stats.MaterializedBytes += encodedBytes switch kind { @@ -1239,23 +1292,40 @@ func (s *Snapshot) recordMaterialized(key disk.NodeKey, kind disk.BranchKind, en } func (s *Snapshot) recordNodeReads(count int) { + if !s.trackApplyStats() { + return + } s.statsMu.Lock() defer s.statsMu.Unlock() s.stats.NodeReads += count } func (s *Snapshot) recordMaterializeRoute(duration time.Duration) { + if !s.trackApplyStats() { + return + } s.statsMu.Lock() defer s.statsMu.Unlock() s.stats.MaterializeRouteDuration += duration } func (s *Snapshot) recordMaterializeFork() { + if !s.trackApplyStats() { + return + } s.statsMu.Lock() defer s.statsMu.Unlock() s.stats.MaterializeParallelForks++ } +func (s *Snapshot) trackApplyStats() bool { + return s != nil && s.parent != nil && s.parent.trackApplyStats +} + +func (s *Snapshot) trackLoadedKeys() bool { + return s != nil && s.parent != nil && s.parent.nodeCacheDepth >= 0 +} + func recordMaterializedDepth(stats *disk.ApplyStats, depth int) { switch { case depth <= 8: diff --git a/internal/smt/disk/persist/tree_test.go b/internal/smt/disk/persist/tree_test.go index e515f9d..e6eb46d 100644 --- a/internal/smt/disk/persist/tree_test.go +++ b/internal/smt/disk/persist/tree_test.go @@ -280,7 +280,7 @@ func TestSnapshotDiscardDoesNotWriteStore(t *testing.T) { require.Equal(t, disk.EmptyRootHash(), tree.RootHash()) } -func TestSnapshotCommitTombstonesLoadedKeysWithoutFinalWrites(t *testing.T) { +func TestSnapshotCommitLeavesLoadedKeysAppendOnly(t *testing.T) { store := openTestStore(t, t.TempDir()) defer store.Close() tree := openPersistTree(t, store) @@ -297,7 +297,8 @@ func TestSnapshotCommitTombstonesLoadedKeysWithoutFinalWrites(t *testing.T) { snapshot.loadedKeys[staleKey] = struct{}{} require.NoError(t, snapshot.rebuildOverlay()) require.NoError(t, snapshot.Commit(api.NewBigIntFromUint64(3))) - requireNodeMissing(t, store, staleKey) + requireNodeExists(t, store, staleKey) + require.Equal(t, disk.EmptyRootHash(), tree.RootHash()) } func TestSnapshotCommitWriteWinsOverLoadedKeyTombstone(t *testing.T) { @@ -423,23 +424,47 @@ func TestSnapshotForkUsesParentOverlayWithoutPersistingParent(t *testing.T) { require.Equal(t, childResult.CandidateRoot, tree.RootHash()) } -func TestSnapshotForkRejectsUncommittedGrandparentOverlay(t *testing.T) { +func TestSnapshotForkFlattensUncommittedGrandparentOverlay(t *testing.T) { store := openTestStore(t, t.TempDir()) defer store.Close() tree := openPersistTree(t, store) + k1 := keyWithFirstByte(0x01) + k2 := keyWithFirstByte(0x03) + k3 := keyWithFirstByte(0x07) + v1 := []byte("value-one") + v2 := []byte("value-two") + v3 := []byte("value-three") + parent, err := tree.CreateSnapshot() require.NoError(t, err) - _, err = parent.AddLeaves([]disk.LeafInput{leafInput(keyWithFirstByte(0x01), []byte("value-one"))}) + parentResult, err := parent.AddLeaves([]disk.LeafInput{leafInput(k1, v1)}) require.NoError(t, err) child, err := parent.Fork() require.NoError(t, err) - _, err = child.AddLeaves([]disk.LeafInput{leafInput(keyWithFirstByte(0x03), []byte("value-two"))}) + childResult, err := child.AddLeaves([]disk.LeafInput{leafInput(k2, v2)}) require.NoError(t, err) - _, err = child.Fork() - require.ErrorContains(t, err, "uncommitted grandparent overlay") + grandchild, err := child.Fork() + require.NoError(t, err) + grandchildResult, err := grandchild.AddLeaves([]disk.LeafInput{leafInput(k3, v3)}) + require.NoError(t, err) + + require.NoError(t, parent.Commit(api.NewBigIntFromUint64(1))) + require.Equal(t, parentResult.CandidateRoot, tree.RootHash()) + + require.NoError(t, child.SetCommitTarget(tree)) + require.NoError(t, child.Commit(api.NewBigIntFromUint64(2))) + require.Equal(t, childResult.CandidateRoot, tree.RootHash()) + + require.NoError(t, grandchild.SetCommitTarget(tree)) + require.NoError(t, grandchild.Commit(api.NewBigIntFromUint64(3))) + require.Equal(t, memoryRootAfterLeaves(t, + memoryLeaf{key: k1, value: v1}, + memoryLeaf{key: k2, value: v2}, + memoryLeaf{key: k3, value: v3}, + ), grandchildResult.CandidateRoot) } func TestSnapshotForkAfterOlderParentCommitDropsGrandparentOverlay(t *testing.T) { diff --git a/internal/smt/disk/rocksstore/store.go b/internal/smt/disk/rocksstore/store.go index c1a5b64..63b7346 100644 --- a/internal/smt/disk/rocksstore/store.go +++ b/internal/smt/disk/rocksstore/store.go @@ -760,7 +760,9 @@ const ( columnFamilyMeta ) -var proofResponseKeyPrefix = []byte("proof:") +var ( + proofResponseKeyPrefix = []byte("proof:") +) func (s *Store) columnFamily(cf columnFamilyID) *C.rocksdb_column_family_handle_t { switch cf { @@ -1035,13 +1037,17 @@ func nodeKey(key disk.NodeKey) []byte { } func proofResponseKey(stateID api.StateID) ([]byte, error) { + return prefixedStateKey(proofResponseKeyPrefix, stateID) +} + +func prefixedStateKey(prefix []byte, stateID api.StateID) ([]byte, error) { key, err := stateID.GetTreeKey() if err != nil { return nil, err } - out := make([]byte, len(proofResponseKeyPrefix)+len(key)) - copy(out, proofResponseKeyPrefix) - copy(out[len(proofResponseKeyPrefix):], key) + out := make([]byte, len(prefix)+len(key)) + copy(out, prefix) + copy(out[len(prefix):], key) return out, nil } diff --git a/internal/smt/golden_vectors_test.go b/internal/smt/golden_vectors_test.go index 2e326d6..2173f47 100644 --- a/internal/smt/golden_vectors_test.go +++ b/internal/smt/golden_vectors_test.go @@ -20,7 +20,7 @@ func TestGoldenVector_RootMatches(t *testing.T) { require.NoError(t, tree.AddLeaf(k1, []byte("value-one"))) require.NoError(t, tree.AddLeaf(k2, []byte("value-two"))) - const expectedRoot = "000020563433422d651813394a07697b9c09f9c2ab2ddb95eaa8ed2dc3211de3e869" + const expectedRoot = "20563433422d651813394a07697b9c09f9c2ab2ddb95eaa8ed2dc3211de3e869" require.Equal(t, expectedRoot, tree.GetRootHashHex()) } @@ -39,7 +39,7 @@ func TestGoldenVector_ProofBitmapAndSiblingsMatch(t *testing.T) { require.NoError(t, tree.AddLeaf(k2, v2)) require.NoError(t, tree.AddLeaf(k3, v3)) - const expectedRoot = "0000b08cae8f98a168a4b39dced99fc3ea2833291c8c53a0eb447e0056044dee598a" + const expectedRoot = "b08cae8f98a168a4b39dced99fc3ea2833291c8c53a0eb447e0056044dee598a" require.Equal(t, expectedRoot, tree.GetRootHashHex()) path, err := tree.GetPath(k2) diff --git a/internal/smt/inclusion_cert_test.go b/internal/smt/inclusion_cert_test.go index be69cd9..973ffcf 100644 --- a/internal/smt/inclusion_cert_test.go +++ b/internal/smt/inclusion_cert_test.go @@ -91,7 +91,7 @@ func TestGetInclusionCert_GoldenVector(t *testing.T) { addLeaf(t, tree, k2, v2) addLeaf(t, tree, k3, v3) - const expectedRoot = "0000b08cae8f98a168a4b39dced99fc3ea2833291c8c53a0eb447e0056044dee598a" + const expectedRoot = "b08cae8f98a168a4b39dced99fc3ea2833291c8c53a0eb447e0056044dee598a" require.Equal(t, expectedRoot, tree.GetRootHashHex()) cert, err := tree.GetInclusionCert(k2) @@ -307,8 +307,8 @@ func TestGetShardInclusionFragment_SkipsNilSiblingAfterNonNilSibling(t *testing. require.NoError(t, fragment.Verify(0b100, 2, leaf4, root, api.SHA256)) } -// TestGetRootHashRaw_MatchesHex confirms that GetRootHashRaw produces -// the 32-byte hash portion of GetRootHashHex. +// TestGetRootHashRaw_MatchesHex confirms that GetRootHashHex is the raw root +// hex encoding, with no algorithm/imprint prefix. func TestGetRootHashRaw_MatchesHex(t *testing.T) { tree := NewSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits) addLeaf(t, tree, @@ -322,7 +322,7 @@ func TestGetRootHashRaw_MatchesHex(t *testing.T) { require.Len(t, rawHex, 64, "raw root must be 32 bytes hex-encoded") fullHex := tree.GetRootHashHex() - require.Equal(t, "0000"+rawHex, fullHex, "raw root must match the hash portion of the hex root") + require.Equal(t, rawHex, fullHex, "hex root must be the raw 32-byte root") } // addLeaf is a small helper that converts a 32-byte key to its path form diff --git a/internal/smt/legacy_path_test.go b/internal/smt/legacy_path_test.go new file mode 100644 index 0000000..4c04d93 --- /dev/null +++ b/internal/smt/legacy_path_test.go @@ -0,0 +1,7 @@ +package smt + +import "github.com/unicitynetwork/aggregator-go/pkg/api" + +func legacyPathRootHex(tree *SparseMerkleTree) string { + return api.NewDataHash(tree.algorithm, tree.GetRootHashRaw()).ToHex() +} diff --git a/internal/smt/smt.go b/internal/smt/smt.go index f04095a..358adac 100644 --- a/internal/smt/smt.go +++ b/internal/smt/smt.go @@ -36,9 +36,8 @@ func hashLen(algo api.HashAlgorithm) int { } } -// buildImprint constructs the hash imprint used for SMT root hashes. -// Used at API boundaries (GetRootHash, GetPath) where a full DataHash object -// is not needed. +// buildImprint supports the legacy MerkleTreePath proof API. New v2 proof and +// UC paths use raw roots via GetRootHashRaw/GetRootHashHex. func buildImprint(algo api.HashAlgorithm, raw []byte) []byte { alg := uint(algo) out := make([]byte, len(raw)+2) @@ -199,12 +198,12 @@ func (snapshot *SmtSnapshot) AddLeaves(leaves []*Leaf) error { return snapshot.SparseMerkleTree.AddLeaves(leaves) } -// GetRootHash returns the current root hash of the snapshot +// GetRootHash returns the current raw root hash of the snapshot. func (snapshot *SmtSnapshot) GetRootHash() []byte { return snapshot.SparseMerkleTree.GetRootHash() } -// GetRootHashHex returns the current root hash of the snapshot as hex string +// GetRootHashHex returns the current raw root hash of the snapshot as a hex string. func (snapshot *SmtSnapshot) GetRootHashHex() string { return snapshot.SparseMerkleTree.GetRootHashHex() } @@ -518,18 +517,14 @@ func (smt *SparseMerkleTree) ensureHashes() { smt.root.calculateHash(hasher) } -// GetRootHash returns the root hash as imprint +// GetRootHash returns the raw 32-byte root hash. func (smt *SparseMerkleTree) GetRootHash() []byte { - // Create a new hasher to ensure thread safety - hasher := api.NewDataHasher(smt.algorithm) - return buildImprint(smt.algorithm, smt.root.calculateHash(hasher)) + return smt.GetRootHashRaw() } -// GetRootHashHex returns the root hash as hex string +// GetRootHashHex returns the raw 32-byte root hash as a hex string. func (smt *SparseMerkleTree) GetRootHashHex() string { - // Create a new hasher to ensure thread safety - hasher := api.NewDataHasher(smt.algorithm) - return fmt.Sprintf("%x", buildImprint(smt.algorithm, smt.root.calculateHash(hasher))) + return api.HexBytes(smt.GetRootHashRaw()).String() } // GetRootHashRaw returns the raw 32-byte root hash without the algorithm diff --git a/internal/smt/smt_debug_test.go b/internal/smt/smt_debug_test.go index a0446c0..54b5dc7 100644 --- a/internal/smt/smt_debug_test.go +++ b/internal/smt/smt_debug_test.go @@ -40,7 +40,7 @@ func TestAddLeaves_DebugInvalidPath(t *testing.T) { require.True(t, res.PathIncluded) require.True(t, res.PathValid) - rh := tree.GetRootHashHex() + rh := legacyPathRootHex(tree) require.Equal(t, rh, merkleTreePath.Root) return rh diff --git a/internal/smt/smt_test.go b/internal/smt/smt_test.go index ba69d5c..f454ca8 100644 --- a/internal/smt/smt_test.go +++ b/internal/smt/smt_test.go @@ -19,7 +19,7 @@ func TestSMTGetRoot(t *testing.T) { // v2 reference values for basic tree shapes. t.Run("EmptyTree", func(t *testing.T) { smt := NewSparseMerkleTree(api.SHA256, 2) - expected := "000047dc540c94ceb704a23875c11273e16bb0b8a87aed84de911f2133568115f254" + expected := "47dc540c94ceb704a23875c11273e16bb0b8a87aed84de911f2133568115f254" require.Equal(t, expected, smt.GetRootHashHex()) }) @@ -27,7 +27,7 @@ func TestSMTGetRoot(t *testing.T) { smt := NewSparseMerkleTree(api.SHA256, 2) smt.AddLeaf(big.NewInt(0b100), []byte{0x61}) - expected := "0000d4cb5334dcabbcaff56bfc78706f041b72c0d29337db87d8c85d4e1aaf9fea3a" + expected := "d4cb5334dcabbcaff56bfc78706f041b72c0d29337db87d8c85d4e1aaf9fea3a" require.Equal(t, expected, smt.GetRootHashHex()) }) @@ -35,7 +35,7 @@ func TestSMTGetRoot(t *testing.T) { smt := NewSparseMerkleTree(api.SHA256, 2) smt.AddLeaf(big.NewInt(0b111), []byte{0x62}) - expected := "000064a2f31a60210df058e75a10312c486538f8874e4681de085e3e2d9985b5fd50" + expected := "64a2f31a60210df058e75a10312c486538f8874e4681de085e3e2d9985b5fd50" require.Equal(t, expected, smt.GetRootHashHex()) }) @@ -44,7 +44,7 @@ func TestSMTGetRoot(t *testing.T) { smt.AddLeaf(big.NewInt(0b100), []byte{0x61}) smt.AddLeaf(big.NewInt(0b111), []byte{0x62}) - expected := "0000f0698f0230044b700c1e5e433f7776b8af113199905b6122b19504274dd77111" + expected := "f0698f0230044b700c1e5e433f7776b8af113199905b6122b19504274dd77111" require.Equal(t, expected, smt.GetRootHashHex()) }) @@ -55,7 +55,7 @@ func TestSMTGetRoot(t *testing.T) { smt.AddLeaf(big.NewInt(0b1011), []byte{0x63}) smt.AddLeaf(big.NewInt(0b1111), []byte{0x64}) - expected := "0000728a4e5f71d239df87b57bdf1e3bd5ca3383d2b0d16758a9b3f2aedff02e4c24" + expected := "728a4e5f71d239df87b57bdf1e3bd5ca3383d2b0d16758a9b3f2aedff02e4c24" require.Equal(t, expected, smt.GetRootHashHex()) }) } @@ -65,7 +65,7 @@ func TestChildSMTGetRoot(t *testing.T) { smt := NewChildSparseMerkleTree(api.SHA256, 2, 0b10) smt.AddLeaf(big.NewInt(0b100), []byte{0x61}) - expected := "0000d4cb5334dcabbcaff56bfc78706f041b72c0d29337db87d8c85d4e1aaf9fea3a" + expected := "d4cb5334dcabbcaff56bfc78706f041b72c0d29337db87d8c85d4e1aaf9fea3a" require.Equal(t, expected, smt.GetRootHashHex()) }) @@ -73,7 +73,7 @@ func TestChildSMTGetRoot(t *testing.T) { smt := NewChildSparseMerkleTree(api.SHA256, 2, 0b11) smt.AddLeaf(big.NewInt(0b111), []byte{0x62}) - expected := "000064a2f31a60210df058e75a10312c486538f8874e4681de085e3e2d9985b5fd50" + expected := "64a2f31a60210df058e75a10312c486538f8874e4681de085e3e2d9985b5fd50" require.Equal(t, expected, smt.GetRootHashHex()) }) @@ -82,7 +82,7 @@ func TestChildSMTGetRoot(t *testing.T) { smt.AddLeaf(big.NewInt(0b10010), []byte{0x61}) smt.AddLeaf(big.NewInt(0b11010), []byte{0x62}) - expected := "0000564b213cf6cee27badc130c7b9c7f06c27b76e8bbe25149e1412646d24027d2d" + expected := "564b213cf6cee27badc130c7b9c7f06c27b76e8bbe25149e1412646d24027d2d" require.Equal(t, expected, smt.GetRootHashHex()) }) @@ -91,7 +91,7 @@ func TestChildSMTGetRoot(t *testing.T) { smt.AddLeaf(big.NewInt(0b10101), []byte{0x63}) smt.AddLeaf(big.NewInt(0b11101), []byte{0x64}) - expected := "0000c5f0538e97bb172a7e423848673faa84141b2201cc803b328f1824299f24dd7f" + expected := "c5f0538e97bb172a7e423848673faa84141b2201cc803b328f1824299f24dd7f" require.Equal(t, expected, smt.GetRootHashHex()) }) } @@ -104,7 +104,7 @@ func TestParentSMTGetRoot(t *testing.T) { smt.AddLeaf(big.NewInt(0b10), left) smt.AddLeaf(big.NewInt(0b11), right) - expected := "0000245915b6e866e0dfa36eb5c1323325c6663bd0ea7fe9ea7c60efe54700901577" + expected := "245915b6e866e0dfa36eb5c1323325c6663bd0ea7fe9ea7c60efe54700901577" require.Equal(t, expected, smt.GetRootHashHex()) }) @@ -115,7 +115,7 @@ func TestParentSMTGetRoot(t *testing.T) { smt.AddLeaf(big.NewInt(0b110), left) smt.AddLeaf(big.NewInt(0b101), right) - expected := "00001f52283972b0b30de79673b0a889357af74859504f70a657c7516ab77b698302" + expected := "1f52283972b0b30de79673b0a889357af74859504f70a657c7516ab77b698302" require.Equal(t, expected, smt.GetRootHashHex()) }) } @@ -227,7 +227,7 @@ func TestSMTBatchOperations(t *testing.T) { // TestSMTRootHashRegressionFixture pins an implementation reference root hash // for a fixed leaf set, so refactors cannot accidentally change hash behavior. func TestSMTRootHashRegressionFixture(t *testing.T) { - const expectedRoot = "00008f12d069a0a8d02649dae4485d97ea1d98f2742b5c22de64a4f331b6f0b7b7dd" + const expectedRoot = "8f12d069a0a8d02649dae4485d97ea1d98f2742b5c22de64a4f331b6f0b7b7dd" leaves := []*Leaf{ NewLeaf(big.NewInt(0b110010000), []byte("value00010000")), // 400 @@ -254,9 +254,9 @@ func TestSMTRootHashRegressionFixture(t *testing.T) { // deterministic child-root inputs. func TestParentSMTRootHashRegressionFixture(t *testing.T) { const ( - expectedEmpty = "0000cd123cd6893ea82539bbce16cd69f196ad3770a1a16806acd624061684f04c22" - expectedOneUpdate = "0000b3bf509ebc9114647fd69f72b817b257b07b8bd32ed82f6b85b8f5b19dedcfc8" - expectedTwoUpdates = "00000eb669e4b5572cd9c2cf3b4a18b491354f67c0591d549cc2879a63defe0a7759" + expectedEmpty = "cd123cd6893ea82539bbce16cd69f196ad3770a1a16806acd624061684f04c22" + expectedOneUpdate = "b3bf509ebc9114647fd69f72b817b257b07b8bd32ed82f6b85b8f5b19dedcfc8" + expectedTwoUpdates = "0eb669e4b5572cd9c2cf3b4a18b491354f67c0591d549cc2879a63defe0a7759" ) make32 := func(start byte) []byte { @@ -552,7 +552,7 @@ func TestSMTGetPath(t *testing.T) { merklePath, err := smt.GetPath(path) require.NoError(t, err) require.NotNil(t, merklePath, "GetPath should return a path") - require.Equal(t, smt.GetRootHashHex(), merklePath.Root, "Root hash should match expected value") + require.Equal(t, legacyPathRootHex(smt), merklePath.Root, "Root hash should match expected value") require.NotNil(t, merklePath.Steps, "Steps should not be nil") require.Equal(t, 2, len(merklePath.Steps), "There should be exactly two steps in the path") // First step should be the LeafNode hash step @@ -585,7 +585,7 @@ func TestSMTGetPath(t *testing.T) { t.Logf("✅ GetPath for existing leaf - Root: %s, Steps: %d", path.Root, len(path.Steps)) // Verify the root hash matches the tree's root - expectedRoot := smt.GetRootHashHex() + expectedRoot := legacyPathRootHex(smt) require.Equal(t, expectedRoot, path.Root, "Path root should match tree root") }) @@ -659,7 +659,7 @@ func TestSMTGetPathComprehensive(t *testing.T) { path, err := smt.GetPath(leafPath) require.NoError(t, err) require.NotNil(t, path, "GetPath should return a path") - require.Equal(t, smt.GetRootHashHex(), path.Root, "Path root should match tree root") + require.Equal(t, legacyPathRootHex(smt), path.Root, "Path root should match tree root") step := path.Steps[0] require.Equal(t, leafPath.String(), step.Path, "Step path should match leaf path") @@ -686,14 +686,14 @@ func TestSMTGetPathComprehensive(t *testing.T) { merkPath1, err := smt.GetPath(path1) require.NoError(t, err) require.NotNil(t, merkPath1, "GetPath should return a path") - require.Equal(t, smt.GetRootHashHex(), merkPath1.Root, "Path root should match tree root") + require.Equal(t, legacyPathRootHex(smt), merkPath1.Root, "Path root should match tree root") require.NotEmpty(t, merkPath1.Steps, "Should have steps") // Get path for second leaf merkPath2, err := smt.GetPath(path2) require.NoError(t, err) require.NotNil(t, merkPath2, "GetPath should return a path") - require.Equal(t, smt.GetRootHashHex(), merkPath2.Root, "Path root should match tree root") + require.Equal(t, legacyPathRootHex(smt), merkPath2.Root, "Path root should match tree root") require.NotEmpty(t, merkPath2.Steps, "Should have steps") // Both paths should have the same root but different steps @@ -728,7 +728,7 @@ func TestSMTGetPathComprehensive(t *testing.T) { merkPath, err := smt.GetPath(nonExistentPath) require.NoError(t, err) require.NotNil(t, merkPath, "GetPath should return a path even for non-existent paths") - require.Equal(t, smt.GetRootHashHex(), merkPath.Root, "Path root should match tree root") + require.Equal(t, legacyPathRootHex(smt), merkPath.Root, "Path root should match tree root") require.NotEmpty(t, merkPath.Steps, "Should have steps even for non-existent path") t.Logf("✅ Non-existent path: Root=%s, Steps=%d", @@ -761,7 +761,7 @@ func TestSMTGetPathComprehensive(t *testing.T) { } // Get paths for all leaves and verify they're consistent - rootHash := smt.GetRootHashHex() + rootHash := legacyPathRootHex(smt) for i, path := range testPaths { merkPath, err := smt.GetPath(path) diff --git a/internal/storage/interfaces/interfaces.go b/internal/storage/interfaces/interfaces.go index c1beffe..0889d4c 100644 --- a/internal/storage/interfaces/interfaces.go +++ b/internal/storage/interfaces/interfaces.go @@ -105,6 +105,9 @@ type BlockStorage interface { // GetRange retrieves blocks in a range GetRange(ctx context.Context, fromBlock, toBlock *api.BigInt) ([]*models.Block, error) + // GetNextFinalizedAfter retrieves the first finalized block after afterBlock, up to toBlock. + GetNextFinalizedAfter(ctx context.Context, afterBlock, toBlock *api.BigInt) (*models.Block, error) + // SetFinalized marks a block as finalized or unfinalized SetFinalized(ctx context.Context, blockNumber *api.BigInt, finalized bool) error @@ -151,25 +154,6 @@ type SmtStorage interface { GetExistingKeys(ctx context.Context, keys []string) (map[string]bool, error) } -// BlockRecordsStorage handles block to state ID mappings -type BlockRecordsStorage interface { - // Store stores a new block records entry - Store(ctx context.Context, records *models.BlockRecords) error - - // GetByBlockNumber retrieves block records by block number - GetByBlockNumber(ctx context.Context, blockNumber *api.BigInt) (*models.BlockRecords, error) - - // Count returns the total number of block records - Count(ctx context.Context) (int64, error) - - // GetNextBlock retrieves the first block after the given block number. - // If blockNumber is nil then returns the very first block. - GetNextBlock(ctx context.Context, blockNumber *api.BigInt) (*models.BlockRecords, error) - - // GetLatestBlockNumber retrieves the latest block - GetLatestBlockNumber(ctx context.Context) (*api.BigInt, error) -} - // LeadershipStorage handles high availability leadership state type LeadershipStorage interface { // TryAcquireLock attempts to acquire the leadership lock, @@ -207,7 +191,6 @@ type Storage interface { AggregatorRecordStorage() AggregatorRecordStorage BlockStorage() BlockStorage SmtStorage() SmtStorage - BlockRecordsStorage() BlockRecordsStorage LeadershipStorage() LeadershipStorage TrustBaseStorage() TrustBaseStorage diff --git a/internal/storage/mongodb/aggregator_record.go b/internal/storage/mongodb/aggregator_record.go index 3f11fc0..775be11 100644 --- a/internal/storage/mongodb/aggregator_record.go +++ b/internal/storage/mongodb/aggregator_record.go @@ -18,8 +18,9 @@ const aggregatorRecordCollection = "aggregator_records" // AggregatorRecordStorage implements aggregator record storage for MongoDB type AggregatorRecordStorage struct { - collection *mongo.Collection - insertOpts finalizationInsertOptions + collection *mongo.Collection + blockCollection *mongo.Collection + insertOpts finalizationInsertOptions } // NewAggregatorRecordStorage creates a new aggregator record storage instance @@ -29,8 +30,9 @@ func NewAggregatorRecordStorage(db *mongo.Database, insertOpts ...finalizationIn opts = insertOpts[0] } return &AggregatorRecordStorage{ - collection: db.Collection(aggregatorRecordCollection), - insertOpts: opts, + collection: db.Collection(aggregatorRecordCollection), + blockCollection: db.Collection(blockCollection), + insertOpts: opts, } } @@ -49,6 +51,18 @@ func (ars *AggregatorRecordStorage) Store(ctx context.Context, record *models.Ag // StoreBatch stores multiple aggregator records using InsertMany. // Duplicate key errors are ignored (duplicates are simply skipped). func (ars *AggregatorRecordStorage) StoreBatch(ctx context.Context, records []*models.AggregatorRecord) error { + return ars.storeBatch(ctx, records, ars.insertOpts) +} + +// StoreBatchSerial stores records without worker parallelism. It is safe to use +// with a Mongo session transaction. +func (ars *AggregatorRecordStorage) StoreBatchSerial(ctx context.Context, records []*models.AggregatorRecord) error { + opts := ars.insertOpts + opts.workers = 1 + return ars.storeBatch(ctx, records, opts) +} + +func (ars *AggregatorRecordStorage) storeBatch(ctx context.Context, records []*models.AggregatorRecord, insertOpts finalizationInsertOptions) error { if len(records) == 0 { return nil } @@ -62,7 +76,7 @@ func (ars *AggregatorRecordStorage) StoreBatch(ctx context.Context, records []*m docs[i] = recordBSON } - err := insertManyFinalizationBatch(ctx, ars.collection, docs, ars.insertOpts) + err := insertManyFinalizationBatch(ctx, ars.collection, docs, insertOpts) if err != nil { return fmt.Errorf("failed to store aggregator records batch: %w", err) } @@ -72,11 +86,24 @@ func (ars *AggregatorRecordStorage) StoreBatch(ctx context.Context, records []*m // GetExistingStateIDs returns which of the given state IDs already exist in the database. // Used to filter duplicates before inserting. func (ars *AggregatorRecordStorage) GetExistingStateIDs(ctx context.Context, stateIDs []string) (map[string]bool, error) { + return ars.getExistingStateIDs(ctx, stateIDs, true) +} + +// GetExistingStateIDsAnyFinalization returns matching state IDs regardless of finality status. +func (ars *AggregatorRecordStorage) GetExistingStateIDsAnyFinalization(ctx context.Context, stateIDs []string) (map[string]bool, error) { + return ars.getExistingStateIDs(ctx, stateIDs, false) +} + +func (ars *AggregatorRecordStorage) getExistingStateIDs(ctx context.Context, stateIDs []string, finalizedOnly bool) (map[string]bool, error) { if len(stateIDs) == 0 { return make(map[string]bool), nil } - cursor, err := ars.collection.Find(ctx, bson.M{"stateId": bson.M{"$in": stateIDs}}, + if finalizedOnly { + return ars.getExistingFinalizedStateIDs(ctx, stateIDs) + } + cursor, err := ars.collection.Find(ctx, + bson.M{"stateId": bson.M{"$in": stateIDs}}, options.Find().SetProjection(bson.M{"stateId": 1})) if err != nil { return nil, fmt.Errorf("failed to query existing state IDs: %w", err) @@ -95,8 +122,58 @@ func (ars *AggregatorRecordStorage) GetExistingStateIDs(ctx context.Context, sta return existing, nil } +func (ars *AggregatorRecordStorage) getExistingFinalizedStateIDs(ctx context.Context, stateIDs []string) (map[string]bool, error) { + pipeline := mongo.Pipeline{ + {{Key: "$match", Value: bson.M{"stateId": bson.M{"$in": stateIDs}}}}, + {{Key: "$lookup", Value: bson.M{ + "from": blockCollection, + "localField": "blockNumber", + "foreignField": "index", + "as": "block", + }}}, + {{Key: "$unwind", Value: "$block"}}, + {{Key: "$match", Value: bson.M{ + "block.finalized": true, + "$expr": bson.M{"$eq": bson.A{"$proposalId", "$block.proposalId"}}, + }}}, + {{Key: "$project", Value: bson.M{"stateId": 1}}}, + } + cursor, err := ars.collection.Aggregate(ctx, pipeline) + if err != nil { + return nil, fmt.Errorf("failed to query finalized existing state IDs: %w", err) + } + defer cursor.Close(ctx) + + existing := make(map[string]bool) + for cursor.Next(ctx) { + var doc bson.M + if err := cursor.Decode(&doc); err == nil { + if stateID, ok := doc["stateId"].(string); ok { + existing[stateID] = true + } + } + } + if err := cursor.Err(); err != nil { + return nil, fmt.Errorf("cursor error: %w", err) + } + return existing, nil +} + // GetByStateID retrieves an aggregator record by state ID func (ars *AggregatorRecordStorage) GetByStateID(ctx context.Context, stateID api.StateID) (*models.AggregatorRecord, error) { + return ars.getByStateID(ctx, stateID, true) +} + +// GetByStateIDAnyFinalization retrieves an aggregator record regardless of its finality status. +func (ars *AggregatorRecordStorage) GetByStateIDAnyFinalization(ctx context.Context, stateID api.StateID) (*models.AggregatorRecord, error) { + return ars.getByStateID(ctx, stateID, false) +} + +func (ars *AggregatorRecordStorage) getByStateID(ctx context.Context, stateID api.StateID, finalizedOnly bool) (*models.AggregatorRecord, error) { + if finalizedOnly { + return ars.getByStateIDFromFinalizedBlock(ctx, stateID) + } + var raw bson.Raw filter := bson.M{"stateId": stateID.String()} if err := ars.collection.FindOne(ctx, filter).Decode(&raw); err != nil { @@ -108,9 +185,97 @@ func (ars *AggregatorRecordStorage) GetByStateID(ctx context.Context, stateID ap return ars.decodeRecord(raw) } +func (ars *AggregatorRecordStorage) getByStateIDFromFinalizedBlock(ctx context.Context, stateID api.StateID) (*models.AggregatorRecord, error) { + pipeline := mongo.Pipeline{ + {{Key: "$match", Value: bson.M{"stateId": stateID.String()}}}, + {{Key: "$lookup", Value: bson.M{ + "from": blockCollection, + "localField": "blockNumber", + "foreignField": "index", + "as": "block", + }}}, + {{Key: "$unwind", Value: "$block"}}, + {{Key: "$match", Value: bson.M{ + "block.finalized": true, + "$expr": bson.M{"$eq": bson.A{"$proposalId", "$block.proposalId"}}, + }}}, + {{Key: "$sort", Value: bson.M{"blockNumber": -1}}}, + {{Key: "$limit", Value: 1}}, + {{Key: "$project", Value: bson.M{"block": 0}}}, + } + cursor, err := ars.collection.Aggregate(ctx, pipeline) + if err != nil { + return nil, fmt.Errorf("failed to get finalized aggregator record by state ID: %w", err) + } + defer cursor.Close(ctx) + if !cursor.Next(ctx) { + if err := cursor.Err(); err != nil { + return nil, fmt.Errorf("cursor error: %w", err) + } + return nil, nil + } + var raw bson.Raw + if err := cursor.Decode(&raw); err != nil { + return nil, fmt.Errorf("failed to decode aggregator record: %w", err) + } + return ars.decodeRecord(raw) +} + // GetByBlockNumber retrieves all records for a specific block func (ars *AggregatorRecordStorage) GetByBlockNumber(ctx context.Context, blockNumber *api.BigInt) ([]*models.AggregatorRecord, error) { + return ars.getByBlockNumber(ctx, blockNumber, true) +} + +// GetByBlockNumberAnyFinalization retrieves records for a block regardless of finality status. +func (ars *AggregatorRecordStorage) GetByBlockNumberAnyFinalization(ctx context.Context, blockNumber *api.BigInt) ([]*models.AggregatorRecord, error) { + return ars.getByBlockNumber(ctx, blockNumber, false) +} + +// GetByBlockNumberAndProposalIDAnyFinalization retrieves records for one durable proposal. +func (ars *AggregatorRecordStorage) GetByBlockNumberAndProposalIDAnyFinalization( + ctx context.Context, + blockNumber *api.BigInt, + proposalID string, +) ([]*models.AggregatorRecord, error) { + filter := bson.M{ + "blockNumber": bigIntToDecimal128(blockNumber), + "proposalId": proposalID, + } + return ars.findByBlockFilter(ctx, filter, false) +} + +// DeleteByBlockNumberAndProposalID deletes records for one durable proposal. +func (ars *AggregatorRecordStorage) DeleteByBlockNumberAndProposalID( + ctx context.Context, + blockNumber *api.BigInt, + proposalID string, +) error { + filter := bson.M{ + "blockNumber": bigIntToDecimal128(blockNumber), + "proposalId": proposalID, + } + if _, err := ars.collection.DeleteMany(ctx, filter); err != nil { + return fmt.Errorf("failed to delete aggregator records by block and proposal: %w", err) + } + return nil +} + +func (ars *AggregatorRecordStorage) getByBlockNumber(ctx context.Context, blockNumber *api.BigInt, finalizedOnly bool) ([]*models.AggregatorRecord, error) { filter := bson.M{"blockNumber": bigIntToDecimal128(blockNumber)} + if finalizedOnly { + proposalID, finalized, err := ars.finalizedBlockProposalID(ctx, blockNumber) + if err != nil { + return nil, err + } + if !finalized { + return nil, nil + } + filter["proposalId"] = proposalID + } + return ars.findByBlockFilter(ctx, filter, true) +} + +func (ars *AggregatorRecordStorage) findByBlockFilter(ctx context.Context, filter bson.M, sortByLeafIndex bool) ([]*models.AggregatorRecord, error) { cursor, err := ars.collection.Find(ctx, filter) if err != nil { return nil, fmt.Errorf("failed to find records by block number: %w", err) @@ -134,23 +299,42 @@ func (ars *AggregatorRecordStorage) GetByBlockNumber(ctx context.Context, blockN return nil, fmt.Errorf("cursor error: %w", err) } - // Preserve get_block_records ordering after dropping the write-heavy - // {blockNumber, leafIndex} Mongo index. - sort.SliceStable(records, func(i, j int) bool { - left := records[i].LeafIndex - right := records[j].LeafIndex - if left == nil || left.Int == nil { - return right != nil && right.Int != nil - } - if right == nil || right.Int == nil { - return false - } - return left.Int.Cmp(right.Int) < 0 - }) + if sortByLeafIndex { + // Preserve get_block_records ordering after dropping the write-heavy + // {blockNumber, leafIndex} Mongo index. + sort.SliceStable(records, func(i, j int) bool { + left := records[i].LeafIndex + right := records[j].LeafIndex + if left == nil || left.Int == nil { + return right != nil && right.Int != nil + } + if right == nil || right.Int == nil { + return false + } + return left.Int.Cmp(right.Int) < 0 + }) + } return records, nil } +func (ars *AggregatorRecordStorage) finalizedBlockProposalID(ctx context.Context, blockNumber *api.BigInt) (string, bool, error) { + var result struct { + ProposalID string `bson:"proposalId"` + } + err := ars.blockCollection.FindOne(ctx, + bson.M{"index": bigIntToDecimal128(blockNumber), "finalized": true}, + options.FindOne().SetProjection(bson.M{"proposalId": 1}), + ).Decode(&result) + if errors.Is(err, mongo.ErrNoDocuments) { + return "", false, nil + } + if err != nil { + return "", false, fmt.Errorf("failed to load finalized block proposal %s: %w", blockNumber.String(), err) + } + return result.ProposalID, true, nil +} + type versionProbe struct { Version uint32 `bson:"version"` } @@ -186,14 +370,13 @@ func (ars *AggregatorRecordStorage) Count(ctx context.Context) (int64, error) { // CreateIndexes creates the necessary indexes needed by the submit, proof, and // block-record lookup paths. func (ars *AggregatorRecordStorage) CreateIndexes(ctx context.Context) error { + if err := ars.ensureNoLegacyUniqueStateIDIndex(ctx); err != nil { + return err + } + indexes := []mongo.IndexModel{ - { - Keys: bson.D{{Key: "stateId", Value: 1}}, - Options: options.Index().SetUnique(true), - }, - { - Keys: bson.D{{Key: "blockNumber", Value: 1}}, - }, + {Keys: bson.D{{Key: "stateId", Value: "hashed"}}}, + {Keys: bson.D{{Key: "blockNumber", Value: 1}, {Key: "proposalId", Value: 1}}}, } _, err := ars.collection.Indexes().CreateMany(ctx, indexes) @@ -203,3 +386,35 @@ func (ars *AggregatorRecordStorage) CreateIndexes(ctx context.Context) error { return nil } + +// ensureNoLegacyUniqueStateIDIndex fails startup fast if the database still +// carries the pre-proposalId unique stateId index. The current model writes the +// same stateId across proposal attempts (visibility is gated by finalized block +// + proposalId), so a surviving unique stateId index would make StoreBatch +// silently drop re-staged records (it swallows duplicate-key errors), corrupting +// recovery. This branch requires a clean database; we enforce that here rather +// than rely on the operational note alone. +func (ars *AggregatorRecordStorage) ensureNoLegacyUniqueStateIDIndex(ctx context.Context) error { + cursor, err := ars.collection.Indexes().List(ctx) + if err != nil { + return fmt.Errorf("failed to list aggregator record indexes: %w", err) + } + defer cursor.Close(ctx) + + for cursor.Next(ctx) { + var idx struct { + Name string `bson:"name"` + Key bson.D `bson:"key"` + Unique bool `bson:"unique"` + } + if err := cursor.Decode(&idx); err != nil { + return fmt.Errorf("failed to decode aggregator record index: %w", err) + } + if idx.Unique && len(idx.Key) == 1 && idx.Key[0].Key == "stateId" { + return fmt.Errorf("legacy unique index %q on %s detected; this branch requires a clean database "+ + "(drop the unique stateId index or recreate the database before deploying)", + idx.Name, aggregatorRecordCollection) + } + } + return cursor.Err() +} diff --git a/internal/storage/mongodb/aggregator_record_test.go b/internal/storage/mongodb/aggregator_record_test.go index 853adc7..5ba173f 100644 --- a/internal/storage/mongodb/aggregator_record_test.go +++ b/internal/storage/mongodb/aggregator_record_test.go @@ -88,7 +88,35 @@ func createTestAggregatorRecord(stateID string, blockNumber int64, leafIndex int blockIndex := api.NewBigInt(big.NewInt(blockNumber)) leafIdx := api.NewBigInt(big.NewInt(leafIndex)) - return models.NewAggregatorRecord(commitment, blockIndex, leafIdx) + record := models.NewAggregatorRecord(commitment, blockIndex, leafIdx) + record.ProposalID = testProposalID(blockNumber) + return record +} + +func testProposalID(blockNumber int64) string { + return "proposal-" + api.NewBigInt(big.NewInt(blockNumber)).String() +} + +func storeTestBlock(t *testing.T, ctx context.Context, db *mongo.Database, blockNumber int64, finalized bool) { + t.Helper() + block := models.NewBlock( + api.NewBigInt(big.NewInt(blockNumber)), + "test-chain", + 0, + "1", + "test", + api.HexBytes(make([]byte, api.SiblingSize)), + nil, + nil, + ) + block.Finalized = finalized + block.ProposalID = testProposalID(blockNumber) + if finalized { + block.Status = models.FinalityStatusFinalized + } else { + block.Status = models.FinalityStatusProposed + } + require.NoError(t, NewBlockStorage(db).Store(ctx, block)) } func TestAggregatorRecordStorage_StoreBatch_DuplicateHandling(t *testing.T) { @@ -123,15 +151,16 @@ func TestAggregatorRecordStorage_StoreBatch_DuplicateHandling(t *testing.T) { err = storage.StoreBatch(ctx, records1) require.NoError(t, err, "First StoreBatch should not return an error") - // Store second batch with duplicates - duplicates are ignored - // With SetOrdered(false), non-duplicate inserts still happen + // Store second batch with duplicate state IDs. Mongo no longer enforces + // stateID uniqueness; normal round processing filters duplicates in the SMT + // apply path before staging records. err = storage.StoreBatch(ctx, records2) - require.NoError(t, err, "StoreBatch should ignore duplicate key errors") + require.NoError(t, err, "StoreBatch should allow duplicate state IDs") + storeTestBlock(t, ctx, db, 1, true) - // With SetOrdered(false), state4 was still inserted despite duplicates count, err := storage.Count(ctx) require.NoError(t, err, "Count should not return an error") - assert.Equal(t, int64(4), count, "Should have 4 records (3 original + 1 new, duplicates failed)") + assert.Equal(t, int64(6), count, "Should have all records; duplicate filtering is not a Mongo storage concern") // Test GetExistingStateIDs to filter duplicates before inserting stateIDs := []string{state1, state2, state4, state5} @@ -149,10 +178,10 @@ func TestAggregatorRecordStorage_StoreBatch_DuplicateHandling(t *testing.T) { err = storage.StoreBatch(ctx, newRecords) require.NoError(t, err, "StoreBatch with only new records should succeed") - // Verify all 5 records now exist + // Verify all 7 stored records now exist. count, err = storage.Count(ctx) require.NoError(t, err, "Count should not return an error") - assert.Equal(t, int64(5), count, "Should have exactly 5 records now") + assert.Equal(t, int64(7), count, "Should have exactly 7 records now") } func TestAggregatorRecordStorage_StoreBatch_ChunkedDuplicateHandling(t *testing.T) { @@ -189,11 +218,12 @@ func TestAggregatorRecordStorage_StoreBatch_ChunkedDuplicateHandling(t *testing. require.NoError(t, err, "Initial StoreBatch should not return an error") err = storage.StoreBatch(ctx, mixedRecords) - require.NoError(t, err, "Chunked StoreBatch should ignore duplicate key errors") + require.NoError(t, err, "Chunked StoreBatch should allow duplicate state IDs") + storeTestBlock(t, ctx, db, 1, true) count, err := storage.Count(ctx) require.NoError(t, err, "Count should not return an error") - assert.Equal(t, int64(5), count, "Should have exactly 5 unique records") + assert.Equal(t, int64(7), count, "Should have all records; duplicate filtering is not a Mongo storage concern") record4, err := storage.GetByStateID(ctx, api.RequireNewImprintV2(state4)) require.NoError(t, err, "GetByStateID should not return an error for state4") @@ -234,10 +264,14 @@ func TestAggregatorRecordStorage_GetByBlockNumber(t *testing.T) { } err = storage.StoreBatch(ctx, records) require.NoError(t, err, "StoreBatch should not return an error") + storeTestBlock(t, ctx, db, 0, true) + storeTestBlock(t, ctx, db, 100, true) + storeTestBlock(t, ctx, db, 101, true) largeBlockNumberRecord := createTestAggregatorRecord("1000", 99999999999999999, 0) err = storage.Store(ctx, largeBlockNumberRecord) require.NoError(t, err, "Store should not return an error for large block number") + storeTestBlock(t, ctx, db, 99999999999999999, true) t.Run("should return records for a specific block number", func(t *testing.T) { blockNum := api.NewBigInt(big.NewInt(100)) @@ -264,6 +298,7 @@ func TestAggregatorRecordStorage_GetByBlockNumber(t *testing.T) { } err := storage.StoreBatch(ctx, unorderedRecords) require.NoError(t, err, "StoreBatch should not return an error") + storeTestBlock(t, ctx, db, 102, true) blockNum := api.NewBigInt(big.NewInt(102)) retrieved, err := storage.GetByBlockNumber(ctx, blockNum) @@ -292,6 +327,68 @@ func TestAggregatorRecordStorage_GetByBlockNumber(t *testing.T) { }) } +func TestAggregatorRecordStorage_FinalityVisibility(t *testing.T) { + db := setupAggregatorRecordTestDB(t) + storage := NewAggregatorRecordStorage(db) + ctx := context.Background() + + require.NoError(t, storage.CreateIndexes(ctx)) + + finalizedState := testStateIDHex("0f01") + proposedState := testStateIDHex("0f02") + finalizedRecord := createTestAggregatorRecord(finalizedState, 700, 0) + proposedRecord := createTestAggregatorRecord(proposedState, 701, 1) + + require.NoError(t, storage.StoreBatch(ctx, []*models.AggregatorRecord{finalizedRecord, proposedRecord})) + storeTestBlock(t, ctx, db, 700, true) + storeTestBlock(t, ctx, db, 701, false) + + visibleByState, err := storage.GetByStateID(ctx, api.RequireNewImprintV2(finalizedState)) + require.NoError(t, err) + require.NotNil(t, visibleByState) + + hiddenByState, err := storage.GetByStateID(ctx, api.RequireNewImprintV2(proposedState)) + require.NoError(t, err) + require.Nil(t, hiddenByState) + + anyByState, err := storage.GetByStateIDAnyFinalization(ctx, api.RequireNewImprintV2(proposedState)) + require.NoError(t, err) + require.NotNil(t, anyByState) + require.Equal(t, proposedRecord.ProposalID, anyByState.ProposalID) + + finalizedBlockNumber := api.NewBigInt(big.NewInt(700)) + visibleByBlock, err := storage.GetByBlockNumber(ctx, finalizedBlockNumber) + require.NoError(t, err) + require.Len(t, visibleByBlock, 1) + require.Equal(t, api.RequireNewImprintV2(finalizedState), visibleByBlock[0].StateID) + + proposedBlockNumber := api.NewBigInt(big.NewInt(701)) + hiddenByBlock, err := storage.GetByBlockNumber(ctx, proposedBlockNumber) + require.NoError(t, err) + require.Empty(t, hiddenByBlock) + + anyByBlock, err := storage.GetByBlockNumberAnyFinalization(ctx, proposedBlockNumber) + require.NoError(t, err) + require.Len(t, anyByBlock, 1) + + existing, err := storage.GetExistingStateIDs(ctx, []string{finalizedState, proposedState}) + require.NoError(t, err) + require.True(t, existing[finalizedState]) + require.False(t, existing[proposedState]) + + existingAny, err := storage.GetExistingStateIDsAnyFinalization(ctx, []string{finalizedState, proposedState}) + require.NoError(t, err) + require.True(t, existingAny[finalizedState]) + require.True(t, existingAny[proposedState]) + + require.NoError(t, NewBlockStorage(db).SetFinalized(ctx, proposedBlockNumber, true)) + + nowVisible, err := storage.GetByStateID(ctx, api.RequireNewImprintV2(proposedState)) + require.NoError(t, err) + require.NotNil(t, nowVisible) + require.Equal(t, proposedRecord.ProposalID, nowVisible.ProposalID) +} + func TestAggregatorRecordStorage_RoundTrip(t *testing.T) { db := setupAggregatorRecordTestDB(t) storage := NewAggregatorRecordStorage(db) @@ -300,6 +397,7 @@ func TestAggregatorRecordStorage_RoundTrip(t *testing.T) { stateIDHex := "00004d1b938134c52340952357dd89c4c270b9b0b523bd69c03c1774fed907f1" record := createTestAggregatorRecord(stateIDHex, 500, 5) require.NoError(t, storage.Store(ctx, record)) + storeTestBlock(t, ctx, db, 500, true) retrieved, err := storage.GetByStateID(ctx, api.RequireNewImprintV2(stateIDHex)) require.NoError(t, err) diff --git a/internal/storage/mongodb/block.go b/internal/storage/mongodb/block.go index 332db43..009e7f2 100644 --- a/internal/storage/mongodb/block.go +++ b/internal/storage/mongodb/block.go @@ -17,6 +17,14 @@ import ( const blockCollection = "blocks" +func finalizingBlockFilter() bson.M { + return bson.M{"$or": []bson.M{ + {"status": bson.M{"$exists": false}}, + {"status": ""}, + {"status": models.FinalityStatusFinalizing}, + }} +} + // BlockStorage implements block storage for MongoDB type BlockStorage struct { collection *mongo.Collection @@ -57,9 +65,21 @@ func (bs *BlockStorage) Store(ctx context.Context, block *models.Block) error { // GetByNumber retrieves a block by number func (bs *BlockStorage) GetByNumber(ctx context.Context, blockNumber *api.BigInt) (*models.Block, error) { + return bs.getByNumber(ctx, blockNumber, true) +} + +// GetByNumberAnyFinalization retrieves a block regardless of its finality status. +func (bs *BlockStorage) GetByNumberAnyFinalization(ctx context.Context, blockNumber *api.BigInt) (*models.Block, error) { + return bs.getByNumber(ctx, blockNumber, false) +} + +func (bs *BlockStorage) getByNumber(ctx context.Context, blockNumber *api.BigInt, finalizedOnly bool) (*models.Block, error) { var blockBSON models.BlockBSON indexDecimal := bigIntToDecimal128(blockNumber) - filter := bson.M{"index": indexDecimal, "finalized": true} + filter := bson.M{"index": indexDecimal} + if finalizedOnly { + filter["finalized"] = true + } err := bs.collection.FindOne(ctx, filter).Decode(&blockBSON) if err != nil { if errors.Is(err, mongo.ErrNoDocuments) { @@ -194,6 +214,36 @@ func (bs *BlockStorage) GetRange(ctx context.Context, fromBlock, toBlock *api.Bi return blocks, nil } +// GetNextFinalizedAfter retrieves the first finalized block after afterBlock, up to toBlock. +func (bs *BlockStorage) GetNextFinalizedAfter(ctx context.Context, afterBlock, toBlock *api.BigInt) (*models.Block, error) { + afterDecimal := bigIntToDecimal128(afterBlock) + toDecimal := bigIntToDecimal128(toBlock) + + filter := bson.M{ + "index": bson.M{ + "$gt": afterDecimal, + "$lte": toDecimal, + }, + "finalized": true, + } + opts := options.FindOne().SetSort(bson.M{"index": 1}) + + var blockBSON models.BlockBSON + err := bs.collection.FindOne(ctx, filter, opts).Decode(&blockBSON) + if err != nil { + if errors.Is(err, mongo.ErrNoDocuments) { + return nil, nil + } + return nil, fmt.Errorf("failed to find next finalized block after %s: %w", afterBlock.String(), err) + } + + block, err := blockBSON.FromBSON() + if err != nil { + return nil, fmt.Errorf("failed to convert from BSON: %w", err) + } + return block, nil +} + // CreateIndexes creates necessary indexes for the block collection func (bs *BlockStorage) CreateIndexes(ctx context.Context) error { indexes := []mongo.IndexModel{ @@ -228,6 +278,15 @@ func (bs *BlockStorage) CreateIndexes(ctx context.Context) error { if err != nil { return fmt.Errorf("failed to migrate blocks finalized field: %w", err) } + _, err = bs.collection.UpdateMany(ctx, + bson.M{"status": bson.M{"$exists": false}}, + []bson.M{ + {"$set": bson.M{"status": bson.M{"$cond": []interface{}{"$finalized", models.FinalityStatusFinalized, models.FinalityStatusFinalizing}}}}, + }, + ) + if err != nil { + return fmt.Errorf("failed to migrate blocks status field: %w", err) + } return nil } @@ -236,7 +295,11 @@ func (bs *BlockStorage) CreateIndexes(ctx context.Context) error { func (bs *BlockStorage) SetFinalized(ctx context.Context, blockNumber *api.BigInt, finalized bool) error { indexDecimal := bigIntToDecimal128(blockNumber) filter := bson.M{"index": indexDecimal} - update := bson.M{"$set": bson.M{"finalized": finalized}} + status := models.FinalityStatusFinalizing + if finalized { + status = models.FinalityStatusFinalized + } + update := bson.M{"$set": bson.M{"finalized": finalized, "status": status}} result, err := bs.collection.UpdateOne(ctx, filter, update) if err != nil { @@ -248,9 +311,70 @@ func (bs *BlockStorage) SetFinalized(ctx context.Context, blockNumber *api.BigIn return nil } +func (bs *BlockStorage) SetStatus(ctx context.Context, blockNumber *api.BigInt, status string) error { + filter := bson.M{"index": bigIntToDecimal128(blockNumber)} + update := bson.M{"status": status, "finalized": status == models.FinalityStatusFinalized} + result, err := bs.collection.UpdateOne(ctx, filter, bson.M{"$set": update}) + if err != nil { + return fmt.Errorf("failed to set block status: %w", err) + } + if result.MatchedCount == 0 { + return fmt.Errorf("block %s not found", blockNumber.String()) + } + return nil +} + +// SetFinalizingWithCertificate marks a certified block as finalizing and +// persists the UC before non-disk SMT node writes begin. +func (bs *BlockStorage) SetFinalizingWithCertificate(ctx context.Context, block *models.Block) error { + if block == nil { + return errors.New("block is nil") + } + filter := bson.M{"index": bigIntToDecimal128(block.Index)} + update := bson.M{"$set": bson.M{ + "finalized": false, + "status": models.FinalityStatusFinalizing, + "unicityCertificate": block.UnicityCertificate.String(), + }} + + result, err := bs.collection.UpdateOne(ctx, filter, update) + if err != nil { + return fmt.Errorf("failed to set block as finalizing with certificate: %w", err) + } + if result.MatchedCount == 0 { + return fmt.Errorf("block %s not found", block.Index.String()) + } + return nil +} + +// SetFinalizedWithCertificate marks a block finalized and persists the UC that +// may have arrived after the proposed block was first stored. +func (bs *BlockStorage) SetFinalizedWithCertificate(ctx context.Context, block *models.Block) error { + if block == nil { + return errors.New("block is nil") + } + indexDecimal := bigIntToDecimal128(block.Index) + filter := bson.M{"index": indexDecimal} + update := bson.M{"$set": bson.M{ + "finalized": true, + "status": models.FinalityStatusFinalized, + "unicityCertificate": block.UnicityCertificate.String(), + }} + + result, err := bs.collection.UpdateOne(ctx, filter, update) + if err != nil { + return fmt.Errorf("failed to set block as finalized with certificate: %w", err) + } + if result.MatchedCount == 0 { + return fmt.Errorf("block %s not found", block.Index.String()) + } + return nil +} + // GetUnfinalized returns all unfinalized blocks (should be at most 1 in normal operation) func (bs *BlockStorage) GetUnfinalized(ctx context.Context) ([]*models.Block, error) { - filter := bson.M{"finalized": false} + filter := finalizingBlockFilter() + filter["finalized"] = false opts := options.Find().SetSort(bson.M{"index": 1}) cursor, err := bs.collection.Find(ctx, filter, opts) diff --git a/internal/storage/mongodb/block_records.go b/internal/storage/mongodb/block_records.go deleted file mode 100644 index 0991afd..0000000 --- a/internal/storage/mongodb/block_records.go +++ /dev/null @@ -1,145 +0,0 @@ -package mongodb - -import ( - "context" - "errors" - "fmt" - - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/unicitynetwork/aggregator-go/internal/models" - "github.com/unicitynetwork/aggregator-go/internal/storage/interfaces" - "github.com/unicitynetwork/aggregator-go/pkg/api" -) - -const blockRecordsCollection = "block_records" - -// BlockRecordsStorage implements block records storage for MongoDB -type BlockRecordsStorage struct { - collection *mongo.Collection -} - -// NewBlockRecordsStorage creates a new block records storage instance -func NewBlockRecordsStorage(db *mongo.Database) *BlockRecordsStorage { - return &BlockRecordsStorage{ - collection: db.Collection(blockRecordsCollection), - } -} - -// Store stores a new block records entry -func (brs *BlockRecordsStorage) Store(ctx context.Context, records *models.BlockRecords) error { - if records == nil { - return errors.New("block records is nil") - } - recordsBSON, err := records.ToBSON() - if err != nil { - return fmt.Errorf("failed to convert block records to BSON: %w", err) - } - _, err = brs.collection.InsertOne(ctx, recordsBSON) - if err != nil { - if mongo.IsDuplicateKeyError(err) { - return fmt.Errorf("block records already exists: %w", interfaces.ErrDuplicateKey) - } - return fmt.Errorf("failed to store block records: %w", err) - } - return nil -} - -// GetByBlockNumber retrieves block records by block number -func (brs *BlockRecordsStorage) GetByBlockNumber(ctx context.Context, blockNumber *api.BigInt) (*models.BlockRecords, error) { - var result models.BlockRecordsBSON - filter := bson.M{"blockNumber": bigIntToDecimal128(blockNumber)} - err := brs.collection.FindOne(ctx, filter).Decode(&result) - if err != nil { - if errors.Is(err, mongo.ErrNoDocuments) { - return nil, nil - } - return nil, fmt.Errorf("failed to get block records by block number: %w", err) - } - - blockRecords, err := result.FromBSON() - if err != nil { - return nil, fmt.Errorf("failed to convert from BSON: %w", err) - } - return blockRecords, nil -} - -// Count returns the total number of block records -func (brs *BlockRecordsStorage) Count(ctx context.Context) (int64, error) { - count, err := brs.collection.CountDocuments(ctx, bson.M{}) - if err != nil { - return 0, fmt.Errorf("failed to count block records: %w", err) - } - return count, nil -} - -// GetNextBlock retrieves the next block record after the given block number. -// If blockNumber is nil then returns the very first block. -func (brs *BlockRecordsStorage) GetNextBlock(ctx context.Context, blockNumber *api.BigInt) (*models.BlockRecords, error) { - var filter bson.M - if blockNumber != nil { - filter = bson.M{"blockNumber": bson.M{"$gt": bigIntToDecimal128(blockNumber)}} - } else { - filter = bson.M{} - } - opts := options.FindOne().SetSort(bson.D{{Key: "blockNumber", Value: 1}}) - - var result models.BlockRecordsBSON - err := brs.collection.FindOne(ctx, filter, opts).Decode(&result) - if err != nil { - if errors.Is(err, mongo.ErrNoDocuments) { - return nil, nil - } - return nil, fmt.Errorf("failed to get next block record: %w", err) - } - - blockRecord, err := result.FromBSON() - if err != nil { - return nil, fmt.Errorf("failed to convert from BSON: %w", err) - } - return blockRecord, nil -} - -// GetLatestBlockNumber retrieves the latest block number -func (brs *BlockRecordsStorage) GetLatestBlockNumber(ctx context.Context) (*api.BigInt, error) { - opts := options.FindOne(). - SetProjection(bson.M{"blockNumber": 1}). - SetSort(bson.D{{Key: "blockNumber", Value: -1}}) - - var result struct { - BlockNumber primitive.Decimal128 `bson:"blockNumber"` - } - - if err := brs.collection.FindOne(ctx, bson.M{}, opts).Decode(&result); err != nil { - if errors.Is(err, mongo.ErrNoDocuments) { - return nil, nil - } - return nil, fmt.Errorf("failed to get latest block record number: %w", err) - } - - blockNumber, _, err := result.BlockNumber.BigInt() - if err != nil { - return nil, fmt.Errorf("failed to parse blockNumber: %w", err) - } - return api.NewBigInt(blockNumber), nil -} - -// CreateIndexes creates the necessary indexes needed by block recovery and HA -// block sync. -func (brs *BlockRecordsStorage) CreateIndexes(ctx context.Context) error { - indexes := []mongo.IndexModel{ - { - Keys: bson.D{{Key: "blockNumber", Value: 1}}, - Options: options.Index().SetUnique(true), - }, - } - - _, err := brs.collection.Indexes().CreateMany(ctx, indexes) - if err != nil { - return fmt.Errorf("failed to create block records indexes: %w", err) - } - return nil -} diff --git a/internal/storage/mongodb/block_records_test.go b/internal/storage/mongodb/block_records_test.go deleted file mode 100644 index 14062d9..0000000 --- a/internal/storage/mongodb/block_records_test.go +++ /dev/null @@ -1,959 +0,0 @@ -package mongodb - -/* -BlockRecordsStorage Tests - -This file contains comprehensive tests for the BlockRecordsStorage.Store() method. - -Test Categories: -1. Unit Tests (TestBlockRecordsStorage_Store_Unit): Test data validation and structure without requiring MongoDB -2. BSON Tests (TestBlockRecordsStorage_Store_BSON): Test BSON marshaling/unmarshaling of BlockRecords -3. Integration Tests (TestBlockRecordsStorage_Store): Test actual database operations using Testcontainers - -Running Tests: -- Unit/BSON tests: go test ./internal/storage/mongodb -v -run "TestBlockRecordsStorage_Store_Unit|TestBlockRecordsStorage_Store_BSON" -- All tests: go test ./internal/storage/mongodb -v (includes integration tests with containerized MongoDB) - -MongoDB Setup: -Integration tests use Testcontainers to spin up a MongoDB container automatically. -No external MongoDB setup is required. Docker must be available to run integration tests. -*/ - -import ( - "context" - "fmt" - "math/big" - "sort" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/testcontainers/testcontainers-go/modules/mongodb" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/unicitynetwork/aggregator-go/internal/models" - "github.com/unicitynetwork/aggregator-go/internal/storage/interfaces" - "github.com/unicitynetwork/aggregator-go/pkg/api" -) - -// Test configuration -const ( - testDBName = "test_aggregator_db" - testTimeout = 30 * time.Second -) - -// setupTestDB creates a test database connection using Testcontainers -func setupTestDB(t *testing.T) *mongo.Database { - ctx := t.Context() - - // Create MongoDB container - mongoContainer, err := mongodb.Run(ctx, "mongo:7.0") - if err != nil { - t.Skipf("Skipping MongoDB tests - cannot start MongoDB container: %v", err) - } - - // Get connection URI - mongoURI, err := mongoContainer.ConnectionString(ctx) - if err != nil { - t.Fatalf("Failed to get MongoDB connection string: %v", err) - } - - // Connect to MongoDB - connectCtx, cancel := context.WithTimeout(ctx, testTimeout) - defer cancel() - - client, err := mongo.Connect(connectCtx, options.Client().ApplyURI(mongoURI)) - if err != nil { - t.Fatalf("Failed to connect to MongoDB: %v", err) - } - - // Ping to verify connection - if err := client.Ping(connectCtx, nil); err != nil { - t.Fatalf("Failed to ping MongoDB: %v", err) - } - - // Create test database - db := client.Database(testDBName) - - // Cleanup function - cleanup := func() { - ctx, cancel := context.WithTimeout(context.Background(), testTimeout) - defer cancel() - - // Drop the test database - if err := db.Drop(ctx); err != nil { - t.Logf("Failed to drop test database: %v", err) - } - - // Disconnect client - if err := client.Disconnect(ctx); err != nil { - t.Logf("Failed to disconnect MongoDB client: %v", err) - } - - // Terminate the container - if err := mongoContainer.Terminate(ctx); err != nil { - t.Logf("Failed to terminate MongoDB container: %v", err) - } - } - t.Cleanup(cleanup) - - return db -} - -// createTestBlockRecords creates a test BlockRecords instance -func createTestBlockRecords(blockNumber *api.BigInt, stateIDs []api.StateID) *models.BlockRecords { - return &models.BlockRecords{ - BlockNumber: blockNumber, - StateIDs: stateIDs, - CreatedAt: api.Now(), - } -} - -func TestBlockRecordsStorage_Store(t *testing.T) { - db := setupTestDB(t) - storage := NewBlockRecordsStorage(db) - ctx := context.Background() - - t.Run("should store valid block records", func(t *testing.T) { - // Create test data - blockNumber := api.NewBigInt(big.NewInt(12345)) - stateIDs := []api.StateID{ - api.RequireNewImprintV2("ea659cdc838619b3767c057fdf8e6d99fde2680c5d8517eb06761c0878d40c40"), - api.RequireNewImprintV2("1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"), - } - - records := createTestBlockRecords(blockNumber, stateIDs) - - // Store the records - err := storage.Store(ctx, records) - require.NoError(t, err, "Store should not return an error") - - // Verify the record was stored by retrieving it - storedRecord, err := storage.GetByBlockNumber(ctx, blockNumber) - require.NoError(t, err, "Should be able to retrieve stored record") - - // Verify the stored data matches - assert.Equal(t, blockNumber.String(), storedRecord.BlockNumber.String()) - assert.Equal(t, len(stateIDs), len(storedRecord.StateIDs)) - for i, stateID := range stateIDs { - assert.Equal(t, stateID, storedRecord.StateIDs[i]) - } - assert.NotNil(t, storedRecord.CreatedAt) - }) - - t.Run("should store block records with empty state IDs", func(t *testing.T) { - // Create test data with empty state IDs - blockNumber := api.NewBigInt(big.NewInt(54321)) - stateIDs := []api.StateID{} - - records := createTestBlockRecords(blockNumber, stateIDs) - - // Store the records - err := storage.Store(ctx, records) - require.NoError(t, err, "Store should not return an error") - - // Verify the record was stored - storedRecord, err := storage.GetByBlockNumber(ctx, blockNumber) - require.NoError(t, err, "Should be able to retrieve stored record") - - // Verify the stored data - assert.Equal(t, blockNumber.String(), storedRecord.BlockNumber.String()) - assert.Equal(t, 0, len(storedRecord.StateIDs)) - assert.NotNil(t, storedRecord.CreatedAt) - }) - - t.Run("should store block records with large block number", func(t *testing.T) { - // Create test data with large block number - largeNumber, ok := new(big.Int).SetString("999999999999999999999", 10) - require.True(t, ok, "Should be able to create large big.Int") - - blockNumber := api.NewBigInt(largeNumber) - stateIDs := []api.StateID{ - api.RequireNewImprintV2("ffff123456789abcffff123456789abcffff123456789abcffff123456789abc"), - } - - records := createTestBlockRecords(blockNumber, stateIDs) - - // Store the records - err := storage.Store(ctx, records) - require.NoError(t, err, "Store should not return an error") - - // Verify the record was stored - storedRecord, err := storage.GetByBlockNumber(ctx, blockNumber) - require.NoError(t, err, "Should be able to retrieve stored record") - - // Verify the stored data - assert.Equal(t, 0, blockNumber.Cmp(storedRecord.BlockNumber.Int)) - assert.Equal(t, len(stateIDs), len(storedRecord.StateIDs)) - assert.Equal(t, stateIDs[0], storedRecord.StateIDs[0]) - }) - - t.Run("should store multiple block records", func(t *testing.T) { - // Create multiple test records - testCases := []struct { - blockNumber *api.BigInt - stateIDs []api.StateID - }{ - { - blockNumber: api.NewBigInt(big.NewInt(1001)), - stateIDs: []api.StateID{ - api.RequireNewImprintV2("0000000000000000000000000000000000000000000000000000000000000001"), - }, - }, - { - blockNumber: api.NewBigInt(big.NewInt(1002)), - stateIDs: []api.StateID{ - api.RequireNewImprintV2("0000000000000000000000000000000000000000000000000000000000000002"), - api.RequireNewImprintV2("0000000000000000000000000000000000000000000000000000000000000003"), - }, - }, - { - blockNumber: api.NewBigInt(big.NewInt(1003)), - stateIDs: []api.StateID{ - api.RequireNewImprintV2("0000000000000000000000000000000000000000000000000000000000000004"), - api.RequireNewImprintV2("0000000000000000000000000000000000000000000000000000000000000005"), - api.RequireNewImprintV2("0000000000000000000000000000000000000000000000000000000000000006"), - }, - }, - } - - // Store all records - for _, tc := range testCases { - records := createTestBlockRecords(tc.blockNumber, tc.stateIDs) - err := storage.Store(ctx, records) - require.NoError(t, err, "Store should not return an error for block %s", tc.blockNumber.String()) - } - - // Verify all records were stored - for _, tc := range testCases { - storedRecord, err := storage.GetByBlockNumber(ctx, tc.blockNumber) - require.NoError(t, err, "Should be able to retrieve stored record for block %s", tc.blockNumber.String()) - - assert.Equal(t, tc.blockNumber.String(), storedRecord.BlockNumber.String()) - assert.Equal(t, len(tc.stateIDs), len(storedRecord.StateIDs)) - for i, stateID := range tc.stateIDs { - assert.Equal(t, stateID.String(), storedRecord.StateIDs[i].String()) - } - } - }) - - t.Run("should store block records with zero block number", func(t *testing.T) { - // Create test data with zero block number - blockNumber := api.NewBigInt(big.NewInt(0)) - stateIDs := []api.StateID{ - api.RequireNewImprintV2("0000000000000000000000000000000000000000000000000000000000000000"), - } - - records := createTestBlockRecords(blockNumber, stateIDs) - - // Store the records - err := storage.Store(ctx, records) - require.NoError(t, err, "Store should not return an error") - - // Verify the record was stored - storedRecord, err := storage.GetByBlockNumber(ctx, blockNumber) - require.NoError(t, err, "Should be able to retrieve stored record") - - // Verify the stored data - assert.Equal(t, "0", storedRecord.BlockNumber.String()) - assert.Equal(t, len(stateIDs), len(storedRecord.StateIDs)) - }) - - t.Run("should handle context cancellation", func(t *testing.T) { - // Create a context that is already cancelled - cancelledCtx, cancel := context.WithCancel(context.Background()) - cancel() - - // Create test data - blockNumber := api.NewBigInt(big.NewInt(99999)) - stateIDs := []api.StateID{ - api.RequireNewImprintV2("0000000000000000000000000000000000000000000000000000000000099999"), - } - - records := createTestBlockRecords(blockNumber, stateIDs) - - // Attempt to store with cancelled context - err := storage.Store(cancelledCtx, records) - assert.Error(t, err, "Store should return an error when context is cancelled") - assert.Contains(t, err.Error(), "failed to store block records") - }) - - t.Run("should handle context timeout", func(t *testing.T) { - // Create a context with very short timeout - timeoutCtx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) - defer cancel() - - // Wait for timeout to trigger - time.Sleep(10 * time.Millisecond) - - // Create test data - blockNumber := api.NewBigInt(big.NewInt(88888)) - stateIDs := []api.StateID{ - api.RequireNewImprintV2("0000000000000000000000000000000000000000000000000000000000088888"), - } - - records := createTestBlockRecords(blockNumber, stateIDs) - - // Attempt to store with timed out context - err := storage.Store(timeoutCtx, records) - assert.Error(t, err, "Store should return an error when context times out") - assert.Contains(t, err.Error(), "failed to store block records") - }) - - t.Run("should handle nil BlockRecords", func(t *testing.T) { - // Attempt to store nil records - err := storage.Store(ctx, nil) - assert.Error(t, err, "Store should return an error when records is nil") - assert.ErrorContainsf(t, err, "block records is nil", "Store should return an error when records is nil") - }) - - t.Run("should store block records with very long state ID list", func(t *testing.T) { - // Create test data with many state IDs - blockNumber := api.NewBigInt(big.NewInt(77777)) - stateIDs := make([]api.StateID, 1000) - for i := 0; i < 1000; i++ { - stateIDs[i] = api.RequireNewImprintV2(fmt.Sprintf("0000%060d", i)) - } - - records := createTestBlockRecords(blockNumber, stateIDs) - - // Store the records - err := storage.Store(ctx, records) - require.NoError(t, err, "Store should not return an error") - - // Verify the record was stored - storedRecord, err := storage.GetByBlockNumber(ctx, blockNumber) - require.NoError(t, err, "Should be able to retrieve stored record") - - // Verify the stored data - assert.Equal(t, blockNumber.String(), storedRecord.BlockNumber.String()) - assert.Equal(t, 1000, len(storedRecord.StateIDs)) - - // Verify a few random state IDs - assert.Equal(t, stateIDs[0], storedRecord.StateIDs[0]) - assert.Equal(t, stateIDs[500], storedRecord.StateIDs[500]) - assert.Equal(t, stateIDs[999], storedRecord.StateIDs[999]) - }) - - t.Run("should handle decimal128 conversion correctly", func(t *testing.T) { - // Test various number formats - testCases := []int64{0, 1, 100, 999, 1000, 99999, 1000000} - - for _, num := range testCases { - blockNumber := api.NewBigInt(big.NewInt(num)) - stateIDs := []api.StateID{ - api.RequireNewImprintV2("ea659cdc838619b3767c057fdf8e6d99fde2680c5d8517eb06761c0878d40c40"), - api.RequireNewImprintV2("1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"), - } - blockRecords := createTestBlockRecords(blockNumber, stateIDs) - - err := storage.Store(ctx, blockRecords) - require.NoError(t, err, "Should store block with number %d", num) - - retrieved, err := storage.GetByBlockNumber(ctx, blockNumber) - require.NoError(t, err, "Should retrieve block with number %d", num) - require.NotNil(t, retrieved, "Retrieved block should not be nil for number %d", num) - assert.Equal(t, 0, blockNumber.Cmp(retrieved.BlockNumber.Int), "Number should match for %d", num) - } - }) -} - -func TestBlockRecordsStorage_GetLatestBlock(t *testing.T) { - db := setupTestDB(t) - storage := NewBlockRecordsStorage(db) - ctx := context.Background() - - err := storage.CreateIndexes(ctx) - require.NoError(t, err) - - stateIDs := []api.StateID{ - api.RequireNewImprintV2("0000000000000000000000000000000000000000000000000000000000000000"), - } - - t.Run("should return nil when no block records exist", func(t *testing.T) { - num, err := storage.GetLatestBlockNumber(ctx) - require.NoError(t, err, "GetLatestBlockNumber should not return an error when empty") - assert.Nil(t, num, "GetLatestBlockNumber should return nil when no records exist") - }) - - t.Run("should return latest block with single record", func(t *testing.T) { - // Store single record - blockNumber := api.NewBigInt(big.NewInt(5)) - records := createTestBlockRecords(blockNumber, stateIDs) - err := storage.Store(ctx, records) - require.NoError(t, err) - - // Get latest - latestNum, err := storage.GetLatestBlockNumber(ctx) - require.NoError(t, err, "GetLatestNumber should not return an error") - require.NotNil(t, latestNum, "Latest number should not be nil") - - assert.Equal(t, 0, blockNumber.Cmp(records.BlockNumber.Int), "Block number should match") - }) - - t.Run("should return latest block number with multiple records", func(t *testing.T) { - // Store multiple block records in non-sequential order - blockNumbers := []*big.Int{ - big.NewInt(1), - big.NewInt(99), - big.NewInt(110), - big.NewInt(125), - big.NewInt(115), - big.NewInt(130), - big.NewInt(105), - } - - for i, bn := range blockNumbers { - records := createTestBlockRecords(api.NewBigInt(bn), stateIDs) - err := storage.Store(ctx, records) - require.NoError(t, err, "Should store record %d", i) - } - - // Get latest - should be block number 130 - latestBlock, err := storage.GetLatestBlockNumber(ctx) - require.NoError(t, err, "GetLatestNumber should not return an error") - require.NotNil(t, latestBlock, "Latest number should not be nil") - - expectedLatest := api.NewBigInt(big.NewInt(130)) - assert.Equal(t, 0, expectedLatest.Cmp(latestBlock.Int), "Should get latest block number") - }) - - t.Run("should handle decimal128 sorting correctly for large numbers", func(t *testing.T) { - // Test with large numbers to ensure decimal128 sorts correctly - largeNumbers := []string{ - "1000000000000000000000", - "999999999999999999999", - "1000000000000000000001", - "2000000000000000000000", - } - - var expectedLatest *api.BigInt - for _, numStr := range largeNumbers { - bi := new(big.Int) - bi.SetString(numStr, 10) - blockNumber := api.NewBigInt(bi) - records := createTestBlockRecords(blockNumber, stateIDs) - - err := storage.Store(ctx, records) - require.NoError(t, err, "Should store large number record") - - // Track expected latest (largest number) - if expectedLatest == nil || blockNumber.Cmp(expectedLatest.Int) > 0 { - expectedLatest = blockNumber - } - } - - // Get latest - latestBlock, err := storage.GetLatestBlockNumber(ctx) - require.NoError(t, err, "GetLatestNumber should not return an error") - require.NotNil(t, latestBlock, "Latest number should not be nil") - - assert.Equal(t, 0, expectedLatest.Cmp(latestBlock.Int), "Should get latest block number") - }) -} - -func TestBlockRecordsStorage_GetNextBlock(t *testing.T) { - db := setupTestDB(t) - - storage := NewBlockRecordsStorage(db) - ctx := context.Background() - - err := storage.CreateIndexes(ctx) - require.NoError(t, err) - - stateIDs := []api.StateID{ - api.RequireNewImprintV2("0000000000000000000000000000000000000000000000000000000000000000"), - } - - t.Run("should return nil when no block records exist", func(t *testing.T) { - next, err := storage.GetNextBlock(ctx, nil) - require.NoError(t, err, "GetNextBlock should not return an error when empty") - assert.Nil(t, next, "GetNextBlock should return nil when no records exist") - }) - - t.Run("should return the very first block when blockNumber is nil", func(t *testing.T) { - blockNumbers := []*big.Int{big.NewInt(10), big.NewInt(5), big.NewInt(20)} - - for _, bn := range blockNumbers { - records := createTestBlockRecords(api.NewBigInt(bn), stateIDs) - err := storage.Store(ctx, records) - require.NoError(t, err) - } - - first, err := storage.GetNextBlock(ctx, nil) - require.NoError(t, err, "GetNextBlock should not return error") - require.NotNil(t, first, "Should return a record") - - assert.Equal(t, int64(5), first.BlockNumber.Int.Int64(), "Should return the smallest block number") - }) - - t.Run("should return the next block after given blockNumber", func(t *testing.T) { - blockNumbers := []*big.Int{ - big.NewInt(1), - big.NewInt(99), - big.NewInt(100), - big.NewInt(110), - big.NewInt(120), - big.NewInt(130), - } - - for _, bn := range blockNumbers { - records := createTestBlockRecords(api.NewBigInt(bn), stateIDs) - err := storage.Store(ctx, records) - require.NoError(t, err) - } - - // Ask for next after 110 -> should return 120 - start := api.NewBigInt(big.NewInt(110)) - next, err := storage.GetNextBlock(ctx, start) - require.NoError(t, err, "GetNextBlock should not return error") - require.NotNil(t, next, "Next record should not be nil") - - assert.Equal(t, int64(120), next.BlockNumber.Int.Int64()) - }) - - t.Run("should return nil if no higher block exists", func(t *testing.T) { - blockNumbers := []*big.Int{big.NewInt(200), big.NewInt(210)} - - for _, bn := range blockNumbers { - records := createTestBlockRecords(api.NewBigInt(bn), stateIDs) - err := storage.Store(ctx, records) - require.NoError(t, err) - } - - // Ask for next after 210 -> should return nil - start := api.NewBigInt(big.NewInt(210)) - next, err := storage.GetNextBlock(ctx, start) - require.NoError(t, err, "GetNextBlock should not return error") - assert.Nil(t, next, "Should return nil if there is no higher block") - }) - - t.Run("should handle decimal128 sorting correctly for large numbers", func(t *testing.T) { - largeNumbers := []string{ - "1000000000000000000000", - "999999999999999999999", - "2000000000000000000000", - } - - var stored []*api.BigInt - for _, numStr := range largeNumbers { - bi := new(big.Int) - bi.SetString(numStr, 10) - blockNumber := api.NewBigInt(bi) - stored = append(stored, blockNumber) - - records := createTestBlockRecords(blockNumber, stateIDs) - err := storage.Store(ctx, records) - require.NoError(t, err) - } - - // Sort manually to check expected "next" - sort.Slice(stored, func(i, j int) bool { - return stored[i].Int.Cmp(stored[j].Int) < 0 - }) - - // Ask for next after smallest - next, err := storage.GetNextBlock(ctx, stored[0]) - require.NoError(t, err) - require.NotNil(t, next, "Next record should not be nil") - - assert.Equal(t, 0, stored[1].Cmp(next.BlockNumber.Int), "Should return the correct next block") - }) -} - -func TestBlockRecordsStorage_Store_Integration(t *testing.T) { - db := setupTestDB(t) - - storage := NewBlockRecordsStorage(db) - ctx := context.Background() - - t.Run("should create indexes and store records", func(t *testing.T) { - // Create indexes first - err := storage.CreateIndexes(ctx) - require.NoError(t, err, "CreateIndexes should not return an error") - - // Create test data - blockNumber := api.NewBigInt(big.NewInt(555555)) - stateIDs := []api.StateID{ - api.RequireNewImprintV2("0000000000000000000000000000000000000000000000000000000000555555"), - } - - records := createTestBlockRecords(blockNumber, stateIDs) - - // Store the records - err = storage.Store(ctx, records) - require.NoError(t, err, "Store should not return an error") - - // Verify the record was stored and can be retrieved using the GetByBlockNumber method - retrievedRecords, err := storage.GetByBlockNumber(ctx, blockNumber) - require.NoError(t, err, "GetByBlockNumber should not return an error") - require.NotNil(t, retrievedRecords, "Retrieved records should not be nil") - - // Verify the data - assert.Equal(t, blockNumber.String(), retrievedRecords.BlockNumber.String()) - assert.Equal(t, len(stateIDs), len(retrievedRecords.StateIDs)) - assert.Equal(t, stateIDs[0], retrievedRecords.StateIDs[0]) - }) - - t.Run("should handle duplicate block numbers with unique index", func(t *testing.T) { - // Create indexes first - err := storage.CreateIndexes(ctx) - require.NoError(t, err, "CreateIndexes should not return an error") - - // Create test data with same block number - blockNumber := api.NewBigInt(big.NewInt(666666)) - stateIDs1 := []api.StateID{ - api.RequireNewImprintV2("0000000000000000000000000000000000000000000000000000000000666661"), - } - stateIDs2 := []api.StateID{ - api.RequireNewImprintV2("0000000000000000000000000000000000000000000000000000000000666662"), - } - - records1 := createTestBlockRecords(blockNumber, stateIDs1) - records2 := createTestBlockRecords(blockNumber, stateIDs2) - - // Store the first record - err = storage.Store(ctx, records1) - require.NoError(t, err, "First store should not return an error") - - // Attempt to store the second record with the same block number - err = storage.Store(ctx, records2) - assert.Error(t, err, "Second store should return an error due to unique index") - assert.ErrorIs(t, err, interfaces.ErrDuplicateKey, "Error should wrap ErrDuplicateKey") - }) -} - -// TestBlockRecordsStorage_Store_Unit contains unit tests that don't require a real MongoDB connection -func TestBlockRecordsStorage_Store_Unit(t *testing.T) { - t.Run("should validate BlockRecords structure", func(t *testing.T) { - // Test that BlockRecords can be created properly - blockNumber := api.NewBigInt(big.NewInt(42)) - stateIDs := []api.StateID{ - api.RequireNewImprintV2("ea659cdc838619b3767c057fdf8e6d99fde2680c5d8517eb06761c0878d40c40"), - api.RequireNewImprintV2("1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"), - } - - records := createTestBlockRecords(blockNumber, stateIDs) - - // Validate the created records - assert.NotNil(t, records, "Created records should not be nil") - assert.Equal(t, blockNumber.String(), records.BlockNumber.String()) - assert.Equal(t, len(stateIDs), len(records.StateIDs)) - assert.NotNil(t, records.CreatedAt) - - for i, stateID := range stateIDs { - assert.Equal(t, stateID, records.StateIDs[i]) - } - }) - - t.Run("should handle empty state IDs list", func(t *testing.T) { - // Test that BlockRecords can be created with empty state IDs - blockNumber := api.NewBigInt(big.NewInt(100)) - stateIDs := []api.StateID{} - - records := createTestBlockRecords(blockNumber, stateIDs) - - // Validate the created records - assert.NotNil(t, records, "Created records should not be nil") - assert.Equal(t, blockNumber.String(), records.BlockNumber.String()) - assert.Equal(t, 0, len(records.StateIDs)) - assert.NotNil(t, records.CreatedAt) - }) - - t.Run("should handle large block numbers", func(t *testing.T) { - // Test that BlockRecords can be created with very large block numbers - largeNumber, ok := new(big.Int).SetString("999999999999999999999999999999999999999999", 10) - require.True(t, ok, "Should be able to create large big.Int") - - blockNumber := api.NewBigInt(largeNumber) - stateIDs := []api.StateID{ - api.RequireNewImprintV2("ffff123456789abcffff123456789abcffff123456789abcffff123456789abc"), - } - - records := createTestBlockRecords(blockNumber, stateIDs) - - // Validate the created records - assert.NotNil(t, records, "Created records should not be nil") - assert.Equal(t, largeNumber.String(), records.BlockNumber.String()) - assert.Equal(t, len(stateIDs), len(records.StateIDs)) - assert.NotNil(t, records.CreatedAt) - }) - - t.Run("should handle zero block number", func(t *testing.T) { - // Test that BlockRecords can be created with zero block number - blockNumber := api.NewBigInt(big.NewInt(0)) - stateIDs := []api.StateID{ - api.RequireNewImprintV2("0000000000000000000000000000000000000000000000000000000000000000"), - } - - records := createTestBlockRecords(blockNumber, stateIDs) - - // Validate the created records - assert.NotNil(t, records, "Created records should not be nil") - assert.Equal(t, "0", records.BlockNumber.String()) - assert.Equal(t, len(stateIDs), len(records.StateIDs)) - assert.NotNil(t, records.CreatedAt) - }) - - t.Run("should handle many state IDs", func(t *testing.T) { - // Test that BlockRecords can be created with many state IDs - blockNumber := api.NewBigInt(big.NewInt(1000)) - stateIDs := make([]api.StateID, 100) - for i := 0; i < 100; i++ { - stateIDs[i] = api.RequireNewImprintV2(fmt.Sprintf("0000%060d", i)) - } - - records := createTestBlockRecords(blockNumber, stateIDs) - - // Validate the created records - assert.NotNil(t, records, "Created records should not be nil") - assert.Equal(t, blockNumber.String(), records.BlockNumber.String()) - assert.Equal(t, 100, len(records.StateIDs)) - assert.NotNil(t, records.CreatedAt) - - // Verify a few random state IDs - assert.Equal(t, stateIDs[0], records.StateIDs[0]) - assert.Equal(t, stateIDs[50], records.StateIDs[50]) - assert.Equal(t, stateIDs[99], records.StateIDs[99]) - }) - - t.Run("should create storage instance", func(t *testing.T) { - // Test that NewBlockRecordsStorage creates a proper instance - // Note: We can't create a real database connection, but we can test the constructor - - // This would normally take a *mongo.Database, but for unit testing - // we can verify that the constructor works with a proper interface - storage := &BlockRecordsStorage{ - collection: nil, // In real usage, this would be a MongoDB collection - } - - assert.NotNil(t, storage, "Storage instance should not be nil") - // Note: collection will be nil in this test, but that's expected for unit testing - }) -} - -// TestBlockRecordsStorage_Store_BSON tests BSON marshaling/unmarshaling of BlockRecords -func TestBlockRecordsStorage_Store_BSON(t *testing.T) { - t.Run("should marshal and unmarshal BlockRecords to BSON", func(t *testing.T) { - // Create test data - blockNumber := api.NewBigInt(big.NewInt(12345)) - stateIDs := []api.StateID{ - api.RequireNewImprintV2("ea659cdc838619b3767c057fdf8e6d99fde2680c5d8517eb06761c0878d40c40"), - api.RequireNewImprintV2("1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"), - } - - originalRecords := createTestBlockRecords(blockNumber, stateIDs) - originalBSON, err := originalRecords.ToBSON() - require.NoError(t, err) - - // Marshal to BSON - bsonData, err := bson.Marshal(originalBSON) - require.NoError(t, err, "Should be able to marshal BlockRecords to BSON") - - // Unmarshal from BSON - var unmarshaledRecordsBSON models.BlockRecordsBSON - err = bson.Unmarshal(bsonData, &unmarshaledRecordsBSON) - require.NoError(t, err, "Should be able to unmarshal BlockRecords from BSON") - - unmarshaledRecords, err := unmarshaledRecordsBSON.FromBSON() - require.NoError(t, err) - - // Verify the data matches - assert.Equal(t, originalRecords.BlockNumber.String(), unmarshaledRecords.BlockNumber.String()) - assert.Equal(t, len(originalRecords.StateIDs), len(unmarshaledRecords.StateIDs)) - - for i, stateID := range originalRecords.StateIDs { - assert.Equal(t, stateID, unmarshaledRecords.StateIDs[i]) - } - - // Note: CreatedAt comparison would need special handling due to potential precision differences - assert.NotNil(t, unmarshaledRecords.CreatedAt) - }) - - t.Run("should marshal and unmarshal empty state IDs", func(t *testing.T) { - // Create test data with empty state IDs - blockNumber := api.NewBigInt(big.NewInt(54321)) - stateIDs := []api.StateID{} - - originalRecords := createTestBlockRecords(blockNumber, stateIDs) - originalBSON, err := originalRecords.ToBSON() - require.NoError(t, err) - - // Marshal to BSON - bsonData, err := bson.Marshal(originalBSON) - require.NoError(t, err, "Should be able to marshal BlockRecords with empty stateIDs to BSON") - - // Unmarshal from BSON - var unmarshaledRecordsBSON models.BlockRecordsBSON - err = bson.Unmarshal(bsonData, &unmarshaledRecordsBSON) - require.NoError(t, err, "Should be able to unmarshal BlockRecords with empty stateIDs from BSON") - unmarshaledRecords, err := unmarshaledRecordsBSON.FromBSON() - require.NoError(t, err) - - // Verify the data matches - assert.Equal(t, originalRecords.BlockNumber.String(), unmarshaledRecords.BlockNumber.String()) - assert.Equal(t, 0, len(unmarshaledRecords.StateIDs)) - assert.NotNil(t, unmarshaledRecords.CreatedAt) - }) - - t.Run("should marshal and unmarshal large block numbers", func(t *testing.T) { - // Create test data with large block number - largeNumber, ok := new(big.Int).SetString("999999999999999999999999999999", 10) - require.True(t, ok, "Should be able to create large big.Int") - - blockNumber := api.NewBigInt(largeNumber) - stateIDs := []api.StateID{ - api.RequireNewImprintV2("ffff123456789abcffff123456789abcffff123456789abcffff123456789abc"), - } - - originalRecords := createTestBlockRecords(blockNumber, stateIDs) - originalBSON, err := originalRecords.ToBSON() - require.NoError(t, err) - - // Marshal to BSON - bsonData, err := bson.Marshal(originalBSON) - require.NoError(t, err, "Should be able to marshal BlockRecords with large block number to BSON") - - // Unmarshal from BSON - var unmarshaledRecordsBSON models.BlockRecordsBSON - err = bson.Unmarshal(bsonData, &unmarshaledRecordsBSON) - require.NoError(t, err, "Should be able to unmarshal BlockRecords with large block number from BSON") - unmarshaledRecords, err := unmarshaledRecordsBSON.FromBSON() - require.NoError(t, err) - - // Verify the data matches - assert.Equal(t, originalRecords.BlockNumber.String(), unmarshaledRecords.BlockNumber.String()) - assert.Equal(t, len(originalRecords.StateIDs), len(unmarshaledRecords.StateIDs)) - assert.Equal(t, string(originalRecords.StateIDs[0]), string(unmarshaledRecords.StateIDs[0])) - assert.NotNil(t, unmarshaledRecords.CreatedAt) - }) -} - -// TestBlockRecordsStorage_Store_Comprehensive demonstrates complete functionality -func TestBlockRecordsStorage_Store_Comprehensive(t *testing.T) { - t.Run("should demonstrate complete BlockRecords functionality", func(t *testing.T) { - // This test demonstrates that all components work together: - // - BigInt BSON marshaling/unmarshaling - // - StateID (ImprintHexString) BSON marshaling/unmarshaling - // - Timestamp BSON marshaling/unmarshaling - // - BlockRecords structure and BSON serialization - - // Create test data with various data types - blockNumber := api.NewBigInt(big.NewInt(123456789)) - stateIDs := []api.StateID{ - api.RequireNewImprintV2("ea659cdc838619b3767c057fdf8e6d99fde2680c5d8517eb06761c0878d40c40"), - api.RequireNewImprintV2("1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"), - api.RequireNewImprintV2("ffff123456789abcffff123456789abcffff123456789abcffff123456789abc"), - } - - // Create BlockRecords - originalRecords := createTestBlockRecords(blockNumber, stateIDs) - originalRecordsBSON, err := originalRecords.ToBSON() - require.NoError(t, err) - - // Verify structure is correct - assert.NotNil(t, originalRecords) - assert.Equal(t, blockNumber.String(), originalRecords.BlockNumber.String()) - assert.Equal(t, len(stateIDs), len(originalRecords.StateIDs)) - assert.NotNil(t, originalRecords.CreatedAt) - - // Test BSON round-trip - bsonData, err := bson.Marshal(originalRecordsBSON) - require.NoError(t, err, "Should marshal BlockRecords to BSON") - - var unmarshaledRecordsBSON models.BlockRecordsBSON - err = bson.Unmarshal(bsonData, &unmarshaledRecordsBSON) - require.NoError(t, err, "Should unmarshal BlockRecords from BSON") - unmarshaledRecords, err := unmarshaledRecordsBSON.FromBSON() - require.NoError(t, err) - - // Verify all data is preserved through BSON round-trip - assert.Equal(t, originalRecords.BlockNumber.String(), unmarshaledRecords.BlockNumber.String()) - assert.Equal(t, len(originalRecords.StateIDs), len(unmarshaledRecords.StateIDs)) - - for i, originalID := range originalRecords.StateIDs { - assert.Equal(t, string(originalID), string(unmarshaledRecords.StateIDs[i])) - } - - // Verify that all custom types still function after unmarshaling - - // Test BigInt functionality - bigIntBytes := unmarshaledRecords.BlockNumber.Int.Bytes() - assert.True(t, len(bigIntBytes) > 0, "BigInt should have byte representation") - - // Test StateID functionality - for _, stateID := range unmarshaledRecords.StateIDs { - stateIDBytes := stateID.Bytes() - assert.NoError(t, err, "StateID should be able to expose bytes") - assert.True(t, len(stateIDBytes) > 0, "StateID should have bytes") - - assert.Len(t, stateID, api.StateTreeKeyLengthBytes, "StateID should be raw v2 bytes") - } - - // Test Timestamp functionality - assert.True(t, unmarshaledRecords.CreatedAt.UnixMilli() > 0, "Timestamp should have valid Unix time") - - t.Logf("✓ Complete round-trip successful:") - t.Logf(" Block Number: %s", unmarshaledRecords.BlockNumber.String()) - t.Logf(" State IDs: %d", len(unmarshaledRecords.StateIDs)) - t.Logf(" Created At: %v", unmarshaledRecords.CreatedAt.Time) - t.Logf(" BSON Size: %d bytes", len(bsonData)) - }) - - t.Run("should handle edge cases", func(t *testing.T) { - // Test various edge cases that Store method might encounter - - testCases := []struct { - name string - blockNumber *api.BigInt - stateIDs []api.StateID - }{ - { - name: "zero block number", - blockNumber: api.NewBigInt(big.NewInt(0)), - stateIDs: []api.StateID{api.RequireNewImprintV2("0000000000000000000000000000000000000000000000000000000000000000")}, - }, - { - name: "empty state IDs", - blockNumber: api.NewBigInt(big.NewInt(999)), - stateIDs: []api.StateID{}, - }, - { - name: "large block number", - blockNumber: func() *api.BigInt { - large, _ := new(big.Int).SetString("999999999999999999999999999999999", 10) - return api.NewBigInt(large) - }(), - stateIDs: []api.StateID{api.RequireNewImprintV2("ffff000000000000000000000000000000000000000000000000000000000000")}, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - records := createTestBlockRecords(tc.blockNumber, tc.stateIDs) - recordsBSON, err := records.ToBSON() - require.NoError(t, err) - - // Test BSON serialization - bsonData, err := bson.Marshal(recordsBSON) - require.NoError(t, err, "Should marshal edge case to BSON") - - var unmarshaledBSON models.BlockRecordsBSON - err = bson.Unmarshal(bsonData, &unmarshaledBSON) - require.NoError(t, err, "Should unmarshal edge case from BSON") - unmarshaled, err := unmarshaledBSON.FromBSON() - require.NoError(t, err) - - // Verify data integrity - assert.Equal(t, records.BlockNumber.String(), unmarshaled.BlockNumber.String()) - assert.Equal(t, len(records.StateIDs), len(unmarshaled.StateIDs)) - - t.Logf("✓ %s: Block %s with %d state IDs", - tc.name, unmarshaled.BlockNumber.String(), len(unmarshaled.StateIDs)) - }) - } - }) -} diff --git a/internal/storage/mongodb/block_test.go b/internal/storage/mongodb/block_test.go index be044e5..02aa96e 100644 --- a/internal/storage/mongodb/block_test.go +++ b/internal/storage/mongodb/block_test.go @@ -664,3 +664,36 @@ func TestBlockStorage_GetRange(t *testing.T) { assert.Equal(t, 0, largeIndexBigInt.Cmp(blocks[0].Index.Int), "Large index should match") }) } + +func TestBlockStorage_GetNextFinalizedAfter(t *testing.T) { + db, cleanup := setupBlockTestDB(t) + defer cleanup() + + storage := NewBlockStorage(db) + ctx := context.Background() + + err := storage.CreateIndexes(ctx) + require.NoError(t, err) + + blocks := createTestBlocksRange(1, 5) + blocks[1].Finalized = false + blocks[1].Status = models.FinalityStatusFinalizing + for _, block := range blocks { + err := storage.Store(ctx, block) + require.NoError(t, err) + } + + next, err := storage.GetNextFinalizedAfter(ctx, api.NewBigInt(big.NewInt(1)), api.NewBigInt(big.NewInt(5))) + require.NoError(t, err) + require.NotNil(t, next) + require.Equal(t, int64(3), next.Index.Int64()) + + next, err = storage.GetNextFinalizedAfter(ctx, api.NewBigInt(big.NewInt(3)), api.NewBigInt(big.NewInt(4))) + require.NoError(t, err) + require.NotNil(t, next) + require.Equal(t, int64(4), next.Index.Int64()) + + next, err = storage.GetNextFinalizedAfter(ctx, api.NewBigInt(big.NewInt(4)), api.NewBigInt(big.NewInt(4))) + require.NoError(t, err) + require.Nil(t, next) +} diff --git a/internal/storage/mongodb/connection.go b/internal/storage/mongodb/connection.go index 4a36c96..1e368f2 100644 --- a/internal/storage/mongodb/connection.go +++ b/internal/storage/mongodb/connection.go @@ -28,7 +28,6 @@ type Storage struct { aggregatorRecordStorage *AggregatorRecordStorage blockStorage *BlockStorage smtStorage *SmtStorage - blockRecordsStorage *BlockRecordsStorage leadershipStorage *LeadershipStorage cachedTrustBaseStorage *CachedTrustBaseStorage } @@ -90,7 +89,6 @@ func NewStorage(ctx context.Context, config config.Config) (*Storage, error) { storage.aggregatorRecordStorage = NewAggregatorRecordStorage(database, finalizationInsertOpts) storage.blockStorage = NewBlockStorage(database) storage.smtStorage = NewSmtStorage(database, finalizationInsertOpts) - storage.blockRecordsStorage = NewBlockRecordsStorage(database) storage.leadershipStorage = NewLeadershipStorage(database, config.HA.LockTTLSeconds) storage.cachedTrustBaseStorage = NewCachedTrustBaseStorage(NewTrustBaseStorage(database)) @@ -256,11 +254,6 @@ func (s *Storage) SmtStorage() interfaces.SmtStorage { return s.smtStorage } -// BlockRecordsStorage returns the block records storage implementation -func (s *Storage) BlockRecordsStorage() interfaces.BlockRecordsStorage { - return s.blockRecordsStorage -} - // LeadershipStorage returns the leadership storage implementation func (s *Storage) LeadershipStorage() interfaces.LeadershipStorage { return s.leadershipStorage @@ -400,10 +393,6 @@ func (s *Storage) createIndexes(ctx context.Context) error { return fmt.Errorf("failed to create SMT indexes: %w", err) } - if err := s.blockRecordsStorage.CreateIndexes(ctx); err != nil { - return fmt.Errorf("failed to create block records indexes: %w", err) - } - if err := s.leadershipStorage.CreateIndexes(ctx); err != nil { return fmt.Errorf("failed to create leadership indexes: %w", err) } diff --git a/internal/storage/mongodb/index_test.go b/internal/storage/mongodb/index_test.go index f5f8412..11bd6bc 100644 --- a/internal/storage/mongodb/index_test.go +++ b/internal/storage/mongodb/index_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/require" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" ) type mongoIndexSpec struct { @@ -63,24 +64,35 @@ func TestAggregatorRecordStorage_CreateIndexes_ProductionIndexSet(t *testing.T) require.NoError(t, storage.CreateIndexes(ctx)) indexes := listIndexSpecsByName(t, ctx, storage.collection) - requireIndexNames(t, indexes, "_id_", "stateId_1", "blockNumber_1") - requireIndexKey(t, indexes["stateId_1"], bson.D{{Key: "stateId", Value: 1}}) - requireIndexKey(t, indexes["blockNumber_1"], bson.D{{Key: "blockNumber", Value: 1}}) - assert.True(t, indexes["stateId_1"].Unique, "stateId must stay unique for proof lookup and duplicate protection") - assert.False(t, indexes["blockNumber_1"].Unique, "multiple records belong to the same block") + requireIndexNames(t, indexes, "_id_", "stateId_hashed", "blockNumber_1_proposalId_1") + requireIndexKey(t, indexes["stateId_hashed"], bson.D{{Key: "stateId", Value: "hashed"}}) + requireIndexKey(t, indexes["blockNumber_1_proposalId_1"], bson.D{{Key: "blockNumber", Value: 1}, {Key: "proposalId", Value: 1}}) + assert.False(t, indexes["stateId_hashed"].Unique, "duplicate filtering is handled by the SMT apply path") + assert.False(t, indexes["blockNumber_1_proposalId_1"].Unique, "multiple records belong to the same proposal") } -func TestBlockRecordsStorage_CreateIndexes_ProductionIndexSet(t *testing.T) { +func TestAggregatorRecordStorage_CreateIndexes_RejectsLegacyUniqueStateIDIndex(t *testing.T) { db := setupTestDB(t) - storage := NewBlockRecordsStorage(db) + storage := NewAggregatorRecordStorage(db) ctx := context.Background() - require.NoError(t, storage.CreateIndexes(ctx)) + // Simulate an upgraded-in-place database that still carries the pre-proposalId + // unique stateId index. The new model re-stages the same stateId across + // proposal attempts, so this index would silently drop records. + _, err := storage.collection.Indexes().CreateOne(ctx, mongo.IndexModel{ + Keys: bson.D{{Key: "stateId", Value: 1}}, + Options: options.Index().SetUnique(true), + }) + require.NoError(t, err) + + err = storage.CreateIndexes(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "legacy unique index") + // The check must bail before creating the new indexes, so a legacy DB is not + // left half-migrated. indexes := listIndexSpecsByName(t, ctx, storage.collection) - requireIndexNames(t, indexes, "_id_", "blockNumber_1") - requireIndexKey(t, indexes["blockNumber_1"], bson.D{{Key: "blockNumber", Value: 1}}) - assert.True(t, indexes["blockNumber_1"].Unique, "one block_records document is stored per block") + require.NotContains(t, indexes, "stateId_hashed") } func TestSmtStorage_CreateIndexes_ProductionIndexSet(t *testing.T) { diff --git a/internal/storage/mongodb/test_helpers_test.go b/internal/storage/mongodb/test_helpers_test.go new file mode 100644 index 0000000..1d98028 --- /dev/null +++ b/internal/storage/mongodb/test_helpers_test.go @@ -0,0 +1,69 @@ +package mongodb + +import ( + "context" + "testing" + "time" + + "github.com/testcontainers/testcontainers-go/modules/mongodb" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +const ( + testDBName = "test_aggregator_db" + testTimeout = 30 * time.Second +) + +func setupTestDB(t *testing.T) *mongo.Database { + t.Helper() + ctx := t.Context() + + mongoContainer, err := mongodb.Run(ctx, "mongo:7.0") + if err != nil { + t.Skipf("Skipping MongoDB tests - cannot start MongoDB container: %v", err) + } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + if err := mongoContainer.Terminate(ctx); err != nil { + t.Logf("Failed to terminate MongoDB container: %v", err) + } + }) + + mongoURI, err := mongoContainer.ConnectionString(ctx) + if err != nil { + t.Fatalf("Failed to get MongoDB connection string: %v", err) + } + + connectCtx, cancel := context.WithTimeout(ctx, testTimeout) + defer cancel() + + client, err := mongo.Connect(connectCtx, options.Client().ApplyURI(mongoURI)) + if err != nil { + t.Fatalf("Failed to connect to MongoDB: %v", err) + } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + if err := client.Disconnect(ctx); err != nil { + t.Logf("Failed to disconnect MongoDB client: %v", err) + } + }) + + if err := client.Ping(connectCtx, nil); err != nil { + t.Fatalf("Failed to ping MongoDB: %v", err) + } + + db := client.Database(testDBName) + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + + if err := db.Drop(ctx); err != nil { + t.Logf("Failed to drop test database: %v", err) + } + }) + + return db +} diff --git a/pkg/api/merkle_tree_path_verify_test.go b/pkg/api/merkle_tree_path_verify_test.go index 5198b80..f2daa1b 100644 --- a/pkg/api/merkle_tree_path_verify_test.go +++ b/pkg/api/merkle_tree_path_verify_test.go @@ -385,13 +385,11 @@ func TestMerkleTreePathVerifyAlternateAlgorithm(t *testing.T) { t.Run(fmt.Sprintf("Algorithm %d", algo), func(t *testing.T) { tree := smt.NewSparseMerkleTree(algo, 4) tree.AddLeaves(leaves) - root := tree.GetRootHashHex() - require.Equal(t, root[:4], fmt.Sprintf("%04x", algo)) for _, leaf := range leaves { path, err := tree.GetPath(leaf.Path) require.NoError(t, err) - require.Equal(t, root, path.Root) + require.Equal(t, fmt.Sprintf("%04x", algo), path.Root[:4]) res, err := path.Verify(leaf.Path) require.NoError(t, err) require.True(t, res.Result) diff --git a/sharding-compose.yml b/sharding-compose.yml index 122cc7d..0b67904 100644 --- a/sharding-compose.yml +++ b/sharding-compose.yml @@ -182,6 +182,7 @@ services: ENABLE_DOCS: "true" ENABLE_CORS: "true" ENABLE_TLS: "false" + MALLOC_ARENA_MAX: "${MALLOC_ARENA_MAX:-2}" TLS_CERT_FILE: "/app/certs/cert.pem" TLS_KEY_FILE: "/app/certs/key.pem" diff --git a/sharding-ha-compose.yml b/sharding-ha-compose.yml index 3e25ae2..b56c4eb 100644 --- a/sharding-ha-compose.yml +++ b/sharding-ha-compose.yml @@ -307,6 +307,7 @@ services: CONCURRENCY_LIMIT: "5000" ENABLE_DOCS: "true" ENABLE_CORS: "true" + MALLOC_ARENA_MAX: "${MALLOC_ARENA_MAX:-2}" MONGODB_DATABASE: "aggregator" MONGODB_CONNECT_TIMEOUT: "10s" MONGODB_SERVER_SELECTION_TIMEOUT: "5s"