From 1d2065ada30a02e55c507897e6d9f7b334882577 Mon Sep 17 00:00:00 2001 From: Jodson Graves Date: Sun, 12 Jul 2026 21:39:23 -0400 Subject: [PATCH] =?UTF-8?q?feat(record):=20Merkle=20proof=20shape=20?= =?UTF-8?q?=E2=80=94=20RFC-6962=20tree=20replaces=20the=20linear=20fold?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Executes the Merkle-now decision (Jodson, 2026-07-12), the stack plan's highest-rework item, inside the only cheap window it will ever have: no durable logs, no external consumers, no live witnesses. - Head is now the RFC-6962 Merkle tree head over leaf IDs (leaves keep drops/leaf/v0; interior nodes carry drops/node/v1 — the 0x00/0x01 discipline in canon-domain form, so leaves never replay as nodes). - Inclusion proofs are audit paths: ~log2(size) hashes however large the log grows. This is the open-problem-8 floor made real — an entry-level phone verifies its covenant in a million-entry log with ~20 hashes. - Consistency proofs are RFC-6962 subproofs; Log.ProveConsistency serves the witness's extension evidence, and Witness.Countersign consumes the proof instead of raw leaves. - LogID moves to its own drops/logid/v1 tag (the chain-fold arity is gone); checkpoint and witness payloads bump to v1. Entry/leaf/content stay v0 — byte-identical derivations keep their tags. - Exhaustive property suite: every leaf of every tree size to 33, every consistency pair to 33, tamper/truncation/padding/replay rejection, and the Log-level tie-in over a real appended log. The un-checkpointed-order residual is unchanged (exactly CT's), and the witness amnesia residual is unchanged; neither is papered over. Signed-off-by: Jodson Graves --- internal/record/checkpoint.go | 8 +- internal/record/doc.go | 46 +++++----- internal/record/entry.go | 12 ++- internal/record/log.go | 73 ++++++++------- internal/record/log_test.go | 79 ++++++++-------- internal/record/tree.go | 150 ++++++++++++++++++++++++++++++ internal/record/tree_test.go | 158 ++++++++++++++++++++++++++++++++ internal/record/verify.go | 62 +++++++------ internal/record/witness.go | 15 +-- internal/record/witness_test.go | 6 +- 10 files changed, 472 insertions(+), 137 deletions(-) create mode 100644 internal/record/tree.go create mode 100644 internal/record/tree_test.go diff --git a/internal/record/checkpoint.go b/internal/record/checkpoint.go index 94c9d0f..569b484 100644 --- a/internal/record/checkpoint.go +++ b/internal/record/checkpoint.go @@ -10,14 +10,16 @@ import ( // domainCheckpoint tags checkpoint signing payloads; an operator's // checkpoint signature is never valid under any other tag, so it can never // pose as a member seal or a witness countersignature. -const domainCheckpoint = "drops/checkpoint/v0" +const domainCheckpoint = "drops/checkpoint/v1" // Checkpoint is an operator's signed, monotonic commitment to its log head — -// the Certificate Transparency signed-tree-head analogue for a linear chain. +// the Certificate Transparency signed tree head. v1: Head is the RFC-6962 +// Merkle tree head over the leaf IDs (the v0 linear fold is gone, decided +// 2026-07-12 before any durable log existed). type Checkpoint struct { Log Hash // LogID of the operator's log Size uint64 // number of entries committed - Head Hash // chain hash after entry Size-1 (fold seeded with Log); the LogID seed when Size == 0 + Head Hash // Merkle tree head over leaves [0, Size); the LogID seed when Size == 0 IssuedAt time.Time // UTC Signature []byte // ed25519 by the operator; excluded from CanonicalBytes } diff --git a/internal/record/doc.go b/internal/record/doc.go index 7b11fd8..b144143 100644 --- a/internal/record/doc.go +++ b/internal/record/doc.go @@ -1,7 +1,7 @@ // Package record is Drops — Cloudy's dialog-sealed, append-only, witnessed // record: the durable account of what two members agreed to, kept as -// per-operator hash-chained logs whose integrity any member can verify -// offline. Drops is the product name (and the domain-tag namespace, drops/*); +// per-operator Merkle-tree logs (RFC 6962 shape) whose integrity any member +// can verify offline with logarithmic-size proofs. Drops is the product name (and the domain-tag namespace, drops/*); // record stays the Go package name per the 2026-07-09 naming decision. It is a // JFA member-economy layer the substrate coordination protocol does not // define, and the member-facing analogue of the protocol's anchor/ package: @@ -55,21 +55,21 @@ // is an ordinary new Entry whose Corrects field references the entry it // corrects while removing nothing. Runtime second: OpenLog replays and // cryptographically re-verifies the entire store (seals, log binding, -// corrections, duplicates, chain fold), rejecting invalid, foreign, -// duplicate, and dangling-correction entries — but the ORDER of independent -// un-checkpointed entries is NOT fixed by replay alone: a store with two -// such entries swapped replays cleanly to a different head. That order is -// protected by the next rung, and the gap is exactly CT's residual. Append -// rejects a nonzero Corrects that names no in-log entry. Detectable third: -// operator-signed monotonic Checkpoints plus +// corrections, duplicates) and rebuilds the Merkle tree, rejecting invalid, +// foreign, duplicate, and dangling-correction entries — but the ORDER of +// independent un-checkpointed entries is NOT fixed by replay alone: a store +// with two such entries swapped replays cleanly to a different head. That +// order is protected by the next rung, and the gap is exactly CT's residual. +// Append rejects a nonzero Corrects that names no in-log entry. Detectable +// third: operator-signed monotonic Checkpoints plus // VerifyConsistency make any rewrite of checkpointed history // cryptographically visible to anyone holding an older checkpoint; the // checkpoint pair is itself portable fork evidence. // // The record MUST NOT be a single global ledger over unrelated exchanges. // Structural: a Log is constructed from exactly one operator key, and its -// derived identity LogID(operator) seeds the chain fold, so every position -// is scoped to one log. Each Entry carries its LogID INSIDE the dual-signed +// derived identity LogID(operator) is the empty log's head and the scope of +// every checkpoint, so every position is scoped to one log. Each Entry carries its LogID INSIDE the dual-signed // bytes, so a sealed covenant can never migrate or be replayed into another // operator's log, and Checkpoint.Verify binds cp.Log == LogID(operator) so // commitments cannot be replayed either. The package exports no merge, no @@ -110,27 +110,29 @@ // agreements distinct and makes double-appends detectable. // // Signing bytes are canonical, domain-separated, and UTC. All signing and -// hashing payloads are built with the protocol's canon package under the six -// distinct tags below; canon.Time normalizes every signed instant to UTC -// nanoseconds and drops monotonic and location components. +// hashing payloads are built with the protocol's canon package under the +// seven distinct tags below; canon.Time normalizes every signed instant to +// UTC nanoseconds and drops monotonic and location components. // // # Domain tags // // Every hash and signature in the package is computed over canonical bytes -// beginning with one of six distinct, unexported domain tags: +// beginning with one of seven distinct, unexported domain tags: // // drops/content/v0 HashContent digests // drops/entry/v0 Entry signing payloads (both seals) // drops/leaf/v0 Entry.ID leaf hashes -// drops/chain/v0 the chain derivation: LogID seed and fold steps -// drops/checkpoint/v0 Checkpoint signing payloads -// drops/witness/v0 witness countersignature payloads +// drops/logid/v1 LogID derivation (the empty log's head) +// drops/node/v1 interior Merkle tree nodes +// drops/checkpoint/v1 Checkpoint signing payloads +// drops/witness/v1 witness countersignature payloads // // so no artifact is transferable between message types, between a hash role -// and a signature role, or back into sohocloud-protocol messages. The chain -// tag covers the two arities of the one chain derivation — the single-field -// seed (LogID over the operator key) and the two-field fold step — which -// canon's length prefixes keep disjoint. +// and a signature role, or back into sohocloud-protocol messages. Leaves and +// interior nodes carry different tags — the RFC 6962 0x00/0x01 discipline in +// canon form — so a leaf can never be replayed as a node or vice versa. The +// v1 tags mark the 2026-07-12 Merkle decision (proof shape settled before +// any durable log existed); the v0 chain tag is retired, not reused. // // # Named residuals // diff --git a/internal/record/entry.go b/internal/record/entry.go index 2bb4944..49ad5fd 100644 --- a/internal/record/entry.go +++ b/internal/record/entry.go @@ -39,11 +39,17 @@ func HashContent(content []byte) Hash { return Hash(sha256.Sum256(canon.New(domainContent).Bytes(content).Sum())) } +// domainLogID tags the LogID derivation — a log's identity, derived from +// its operator key, never chosen text; it is the empty log's head and scopes +// every entry and checkpoint. v1: the v0 chain-fold arity is gone (the tree +// replaced it), so the derivation carries its own single-purpose tag. +const domainLogID = "drops/logid/v1" + // LogID derives a log's identity from its operator's public key — derived, -// never chosen, so it cannot carry text and cannot be squatted; it seeds the -// log's hash chain and scopes every entry and checkpoint. +// never chosen, so it cannot carry text and cannot be squatted; it is the +// head of the empty log and scopes every entry and checkpoint. func LogID(operator ed25519.PublicKey) Hash { - return Hash(sha256.Sum256(canon.New(domainChain).Bytes(operator).Sum())) + return Hash(sha256.Sum256(canon.New(domainLogID).Bytes(operator).Sum())) } // Entry is one dialog-sealed covenant between two members: fixed-size diff --git a/internal/record/log.go b/internal/record/log.go index 482fd04..d2d1bc1 100644 --- a/internal/record/log.go +++ b/internal/record/log.go @@ -2,25 +2,11 @@ package record import ( "crypto/ed25519" - "crypto/sha256" "errors" "fmt" "time" - - "github.com/NTARI-RAND/sohocloud-protocol/canon" ) -// domainChain tags the one chain derivation in both its arities: the -// single-field seed (LogID over the operator key) and the two-field fold -// step (previous head, leaf). Canon's length prefixes keep the two shapes -// disjoint, and neither is a signature payload. -const domainChain = "drops/chain/v0" - -// chainStep folds one leaf hash into the running chain head. -func chainStep(prev, leaf Hash) Hash { - return Hash(sha256.Sum256(canon.New(domainChain).Bytes(prev[:]).Bytes(leaf[:]).Sum())) -} - // Store is the minimal append-only persistence surface: update and delete // are absent verbs, not forbidden ones. Log never trusts a Store — chains // are re-verified on open. @@ -73,16 +59,16 @@ func (s *MemStore) Len() (uint64, error) { return uint64(len(s.entries)), nil } -// Log is ONE operator's record: a LogID-seeded hash chain over dialog-sealed -// entries. No package-level log, no merge, no cross-log query or ordering -// exists anywhere in the surface. The operator's entire power is assigning -// sequence numbers: it holds no key that can produce a member seal, so it -// can order covenants but never author, amend, or remove one. +// Log is ONE operator's record: an RFC-6962-shaped Merkle tree over +// dialog-sealed entries, identified by LogID. No package-level log, no +// merge, no cross-log query or ordering exists anywhere in the surface. The +// operator's entire power is assigning sequence numbers: it holds no key +// that can produce a member seal, so it can order covenants but never +// author, amend, or remove one. type Log struct { id Hash store Store - heads []Hash // heads[i] is the chain head before entry i; heads[len(leaves)] is the current head - leafs []Hash // leaf IDs in sequence order + leafs []Hash // leaf IDs (Entry.ID) in sequence order — the tree's leaves index map[Hash]uint64 // leaf ID -> sequence, for dedup and Corrects resolution } @@ -99,11 +85,9 @@ func OpenLog(operator ed25519.PublicKey, s Store) (*Log, error) { if s == nil { return nil, errors.New("record: nil store") } - id := LogID(operator) l := &Log{ - id: id, + id: LogID(operator), store: s, - heads: []Hash{id}, index: make(map[Hash]uint64), } n, err := s.Len() @@ -145,12 +129,10 @@ func (l *Log) check(e Entry) (Hash, error) { return leaf, nil } -// admit folds an already-checked leaf into the in-memory chain state. +// admit adds an already-checked leaf to the in-memory tree state. func (l *Log) admit(leaf Hash) { - seq := uint64(len(l.leafs)) - l.index[leaf] = seq + l.index[leaf] = uint64(len(l.leafs)) l.leafs = append(l.leafs, leaf) - l.heads = append(l.heads, chainStep(l.heads[seq], leaf)) } // Append admits e only if e.Verify() passes, e.Log equals this log's LogID, @@ -170,33 +152,56 @@ func (l *Log) Append(e Entry) (uint64, error) { return seq, nil } -// Checkpoint returns the unsigned checkpoint over the current head at UTC -// instant at; the operator seals it with Checkpoint.Sign. Sizes are +// Checkpoint returns the unsigned checkpoint over the current tree head at +// UTC instant at; the operator seals it with Checkpoint.Sign. Sizes are // monotonic because the log only appends. The head of an empty log is the -// LogID seed. +// LogID seed; otherwise it is the Merkle tree head over the leaf IDs. func (l *Log) Checkpoint(at time.Time) Checkpoint { size := uint64(len(l.leafs)) + head := l.id + if size > 0 { + head = mth(l.leafs) + } return Checkpoint{ Log: l.id, Size: size, - Head: l.heads[size], + Head: head, IssuedAt: at, } } // Prove returns the inclusion proof for the entry at seq relative to the // current head; issue the matching Checkpoint in the same quiescent state. +// The proof is an RFC-6962 audit path: ~log2(size) sibling hashes, small +// enough to hold and verify on an entry-level device regardless of how +// large the log grows. func (l *Log) Prove(seq uint64) (Proof, error) { size := uint64(len(l.leafs)) if seq >= size { return Proof{}, fmt.Errorf("record: sequence %d out of range (size %d)", seq, size) } return Proof{ - Prior: l.heads[seq], - Links: append([]Hash(nil), l.leafs[seq+1:]...), + Seq: seq, + Path: provePath(seq, l.leafs), }, nil } +// ProveConsistency returns the RFC-6962 consistency proof showing the +// current tree extends the tree that had `size` entries — the extension +// evidence a witness needs before countersigning a newer checkpoint. A size +// of zero needs no proof (everything extends the empty log), and the +// current size yields the empty proof (a tree extends itself). +func (l *Log) ProveConsistency(size uint64) ([]Hash, error) { + current := uint64(len(l.leafs)) + if size > current { + return nil, fmt.Errorf("record: size %d exceeds log size %d", size, current) + } + if size == 0 || size == current { + return []Hash{}, nil + } + return proveConsistency(size, l.leafs), nil +} + // LeavesSince returns the leaf hashes of entries at positions // [size, current size) — the extension evidence a witness or member needs // for VerifyConsistency. diff --git a/internal/record/log_test.go b/internal/record/log_test.go index 062f7e1..168c0ca 100644 --- a/internal/record/log_test.go +++ b/internal/record/log_test.go @@ -302,8 +302,8 @@ func TestInclusionExhaustive(t *testing.T) { if !VerifyInclusion(entries[seq], p, tc.cp, op.pub) { t.Fatalf("inclusion of entry %d under size-%d checkpoint must verify", seq, tc.cp.Size) } - if got := tc.cp.Size - 1 - uint64(len(p.Links)); got != uint64(seq) { - t.Fatalf("proof position = %d, want %d", got, seq) + if p.Seq != uint64(seq) { + t.Fatalf("proof position = %d, want %d", p.Seq, seq) } } } @@ -317,18 +317,23 @@ func TestInclusionExhaustive(t *testing.T) { if err := half.Seal(a.priv); err != nil { t.Fatalf("Seal: %v", err) } + dishonestLeaves := make([]Hash, 0, 6) + for _, e := range entries { + dishonestLeaves = append(dishonestLeaves, e.ID()) + } + dishonestLeaves = append(dishonestLeaves, half.ID()) dishonest := Checkpoint{ Log: id, Size: cp5.Size + 1, - Head: chainStep(cp5.Head, half.ID()), + Head: mth(dishonestLeaves), IssuedAt: testInstant, } dishonest.Sign(op.priv) if !dishonest.Verify(op.pub) { t.Fatal("setup: dishonest checkpoint should carry a valid operator signature") } - if VerifyInclusion(half, Proof{Prior: cp5.Head}, dishonest, op.pub) { - t.Fatal("a half-sealed entry must never verify as included, even hand-chained by the operator") + if VerifyInclusion(half, Proof{Seq: 5, Path: provePath(5, dishonestLeaves)}, dishonest, op.pub) { + t.Fatal("a half-sealed entry must never verify as included, even hand-treed by the operator") } // A foreign entry (another operator's log) never verifies here. @@ -337,17 +342,23 @@ func TestInclusionExhaustive(t *testing.T) { t.Fatal("a foreign-log entry must not verify as included") } - // Truncated links. - trunc := Proof{Prior: proofs5[0].Prior, Links: proofs5[0].Links[:len(proofs5[0].Links)-1]} + // Truncated audit path. + trunc := Proof{Seq: proofs5[0].Seq, Path: proofs5[0].Path[:len(proofs5[0].Path)-1]} if VerifyInclusion(entries[0], trunc, cp5, op.pub) { - t.Fatal("a truncated Links slice must not verify") + t.Fatal("a truncated audit path must not verify") + } + + // Tampered sibling hash. + tampered := Proof{Seq: proofs5[1].Seq, Path: append([]Hash(nil), proofs5[1].Path...)} + tampered.Path[0][0] ^= 1 + if VerifyInclusion(entries[1], tampered, cp5, op.pub) { + t.Fatal("a tampered audit path must not verify") } - // Wrong prior. - wrongPrior := proofs5[1] - wrongPrior.Prior[0] ^= 1 - if VerifyInclusion(entries[1], wrongPrior, cp5, op.pub) { - t.Fatal("a wrong Prior must not verify") + // Wrong claimed position. + misplaced := Proof{Seq: proofs5[1].Seq + 1, Path: proofs5[1].Path} + if VerifyInclusion(entries[1], misplaced, cp5, op.pub) { + t.Fatal("a proof at the wrong position must not verify") } // Wrong operator key. @@ -408,25 +419,24 @@ func TestConsistency(t *testing.T) { } cp5 := sign(l.Checkpoint(testInstant)) - links, err := l.LeavesSince(2) + links, err := l.ProveConsistency(2) if err != nil { - t.Fatalf("LeavesSince(2): %v", err) + t.Fatalf("ProveConsistency(2): %v", err) } if !VerifyConsistency(cp2, cp5, links, op.pub) { t.Fatal("honest extension from size 2 to 5 must verify") } - all, err := l.LeavesSince(0) - if err != nil { - t.Fatalf("LeavesSince(0): %v", err) + if !VerifyConsistency(cp0, cp5, nil, op.pub) { + t.Fatal("consistency from the empty-log checkpoint needs no proof") } - if !VerifyConsistency(cp0, cp5, all, op.pub) { - t.Fatal("consistency from the empty-log checkpoint must accept the full leaf sequence") + if VerifyConsistency(cp0, cp5, links, op.pub) { + t.Fatal("a non-empty proof against the empty-log checkpoint must not verify") } if VerifyConsistency(cp5, cp2, nil, op.pub) { t.Fatal("newer.Size < older.Size must never verify") } if VerifyConsistency(cp5, cp2, links, op.pub) { - t.Fatal("newer.Size < older.Size must never verify regardless of links") + t.Fatal("newer.Size < older.Size must never verify regardless of proof") } // Fork: the operator rewrites entry 1 (inside cp2's history) and @@ -446,18 +456,22 @@ func TestConsistency(t *testing.T) { } } cpFork5 := sign(lf.Checkpoint(testInstant)) - forkSince2, err := lf.LeavesSince(2) + forkSince2, err := lf.ProveConsistency(2) if err != nil { - t.Fatalf("LeavesSince: %v", err) + t.Fatalf("ProveConsistency: %v", err) + } + garbage := make([]Hash, len(links)) + for i := range garbage { + garbage[i] = HashContent([]byte{byte(200 + i)}) } for name, candidate := range map[string][]Hash{ - "fork LeavesSince(2)": forkSince2, - "honest LeavesSince(2)": links, - "empty": nil, - "full fork leaves": mustLeaves(t, lf, 0), + "fork ProveConsistency(2)": forkSince2, + "honest ProveConsistency(2)": links, + "empty": nil, + "garbage hashes": garbage, } { if VerifyConsistency(cp2, cpFork5, candidate, op.pub) { - t.Fatalf("fork rewriting checkpointed history must fail VerifyConsistency (links: %s)", name) + t.Fatalf("fork rewriting checkpointed history must fail VerifyConsistency (proof: %s)", name) } } // Same-size fork against the honest size-5 checkpoint. @@ -465,12 +479,3 @@ func TestConsistency(t *testing.T) { t.Fatal("a same-size fork must fail VerifyConsistency") } } - -func mustLeaves(t *testing.T, l *Log, since uint64) []Hash { - t.Helper() - leaves, err := l.LeavesSince(since) - if err != nil { - t.Fatalf("LeavesSince(%d): %v", since, err) - } - return leaves -} diff --git a/internal/record/tree.go b/internal/record/tree.go new file mode 100644 index 0000000..d126a89 --- /dev/null +++ b/internal/record/tree.go @@ -0,0 +1,150 @@ +package record + +import ( + "crypto/sha256" + + "github.com/NTARI-RAND/sohocloud-protocol/canon" +) + +// domainNode tags interior Merkle nodes. Leaves keep their own domain +// (drops/leaf/v0, via Entry.ID), so a leaf hash can never be replayed as an +// interior node or vice versa — the RFC 6962 0x00/0x01 prefix discipline, +// expressed with canon domain tags like everything else in the package. +const domainNode = "drops/node/v1" + +// nodeHash combines two subtree hashes into their parent. +func nodeHash(left, right Hash) Hash { + return Hash(sha256.Sum256(canon.New(domainNode).Bytes(left[:]).Bytes(right[:]).Sum())) +} + +// largestPow2LT returns the largest power of two strictly less than n; +// n must be >= 2. +func largestPow2LT(n uint64) uint64 { + k := uint64(1) + for k<<1 < n { + k <<= 1 + } + return k +} + +// mth is the Merkle tree head over leaves (RFC 6962 shape): a single leaf is +// its own subtree hash, and a larger tree splits at the largest power of two +// strictly below its size. mth requires at least one leaf; the empty log's +// head is the LogID seed, handled by Log.Checkpoint. +func mth(leaves []Hash) Hash { + if len(leaves) == 1 { + return leaves[0] + } + k := largestPow2LT(uint64(len(leaves))) + return nodeHash(mth(leaves[:k]), mth(leaves[k:])) +} + +// provePath returns the audit path for the leaf at index m within leaves: +// the sibling subtree hashes from leaf-adjacent first to root-adjacent last. +func provePath(m uint64, leaves []Hash) []Hash { + if len(leaves) == 1 { + return nil + } + k := largestPow2LT(uint64(len(leaves))) + if m < k { + return append(provePath(m, leaves[:k]), mth(leaves[k:])) + } + return append(provePath(m-k, leaves[k:]), mth(leaves[:k])) +} + +// rootFromPath recomputes the tree head implied by placing leaf at index seq +// of a size-entry tree with the given audit path; ok is false when the path +// length does not match the tree shape. Nothing in the path is trusted — +// every hash is recomputed. +func rootFromPath(leaf Hash, seq, size uint64, path []Hash) (Hash, bool) { + if size == 0 || seq >= size { + return Hash{}, false + } + if size == 1 { + if len(path) != 0 { + return Hash{}, false + } + return leaf, true + } + if len(path) == 0 { + return Hash{}, false + } + k := largestPow2LT(size) + sibling := path[len(path)-1] + rest := path[:len(path)-1] + if seq < k { + sub, ok := rootFromPath(leaf, seq, k, rest) + if !ok { + return Hash{}, false + } + return nodeHash(sub, sibling), true + } + sub, ok := rootFromPath(leaf, seq-k, size-k, rest) + if !ok { + return Hash{}, false + } + return nodeHash(sibling, sub), true +} + +// proveConsistency returns the RFC 6962 consistency proof between the tree +// over leaves[:m] and the tree over all leaves (m entries -> n entries, +// 0 < m < n <= len(leaves)). +func proveConsistency(m uint64, leaves []Hash) []Hash { + return subProof(m, leaves, true) +} + +func subProof(m uint64, leaves []Hash, complete bool) []Hash { + n := uint64(len(leaves)) + if m == n { + if complete { + // The old tree is exactly this subtree; the verifier already + // holds its hash (the older checkpoint's head). + return nil + } + return []Hash{mth(leaves)} + } + k := largestPow2LT(n) + if m <= k { + return append(subProof(m, leaves[:k], complete), mth(leaves[k:])) + } + return append(subProof(m-k, leaves[k:], false), mth(leaves[:k])) +} + +// consRoots recomputes, from a consistency proof, the (older, newer) tree +// heads it implies. seed is the older checkpoint's head, standing in for the +// omitted subtree hash along the complete-prefix spine (RFC 6962's b=true). +// The proof is consumed root-adjacent end first; ok is false on any shape +// mismatch. +func consRoots(m, n uint64, complete bool, proof []Hash, seed Hash) (older, newer Hash, rest []Hash, ok bool) { + if m == n { + if complete { + return seed, seed, proof, true + } + if len(proof) == 0 { + return Hash{}, Hash{}, nil, false + } + h := proof[len(proof)-1] + return h, h, proof[:len(proof)-1], true + } + if m > n || len(proof) == 0 { + return Hash{}, Hash{}, nil, false + } + k := largestPow2LT(n) + last := proof[len(proof)-1] + rest = proof[:len(proof)-1] + if m <= k { + oldR, newR, remaining, subOK := consRoots(m, k, complete, rest, seed) + if !subOK { + return Hash{}, Hash{}, nil, false + } + // last is the hash of the new entries' subtree (leaves[k:n]); the + // old tree lies entirely in the left subtree. + return oldR, nodeHash(newR, last), remaining, true + } + oldR, newR, remaining, subOK := consRoots(m-k, n-k, false, rest, seed) + if !subOK { + return Hash{}, Hash{}, nil, false + } + // last is the full left subtree (leaves[:k]), shared by both trees. + return nodeHash(last, oldR), nodeHash(last, newR), remaining, true +} diff --git a/internal/record/tree_test.go b/internal/record/tree_test.go new file mode 100644 index 0000000..73d8d80 --- /dev/null +++ b/internal/record/tree_test.go @@ -0,0 +1,158 @@ +package record + +import ( + "fmt" + "testing" +) + +// syntheticLeaves returns n distinct deterministic leaf hashes. +func syntheticLeaves(n int) []Hash { + leaves := make([]Hash, n) + for i := range leaves { + leaves[i] = HashContent([]byte(fmt.Sprintf("leaf-%d", i))) + } + return leaves +} + +// TestMerkleInclusionExhaustive proves every leaf of every tree size up to 33 +// (crossing several power-of-two boundaries) against the tree head, and +// rejects every single-bit tamper of position, path, and leaf. +func TestMerkleInclusionExhaustive(t *testing.T) { + for n := 1; n <= 33; n++ { + leaves := syntheticLeaves(n) + root := mth(leaves) + for m := 0; m < n; m++ { + path := provePath(uint64(m), leaves) + got, ok := rootFromPath(leaves[m], uint64(m), uint64(n), path) + if !ok || got != root { + t.Fatalf("n=%d m=%d: honest audit path does not recompute the head", n, m) + } + // Wrong position never verifies (every other position). + for wrong := 0; wrong < n; wrong++ { + if wrong == m { + continue + } + if got, ok := rootFromPath(leaves[m], uint64(wrong), uint64(n), path); ok && got == root { + t.Fatalf("n=%d m=%d: path verified at wrong position %d", n, m, wrong) + } + } + // Tampered path element never verifies. + if len(path) > 0 { + bad := append([]Hash(nil), path...) + bad[0][0] ^= 1 + if got, ok := rootFromPath(leaves[m], uint64(m), uint64(n), bad); ok && got == root { + t.Fatalf("n=%d m=%d: tampered path verified", n, m) + } + } + // Wrong leaf never verifies. + wrongLeaf := HashContent([]byte("not a leaf")) + if got, ok := rootFromPath(wrongLeaf, uint64(m), uint64(n), path); ok && got == root { + t.Fatalf("n=%d m=%d: foreign leaf verified", n, m) + } + // Truncated and padded paths never verify. + if len(path) > 0 { + if got, ok := rootFromPath(leaves[m], uint64(m), uint64(n), path[:len(path)-1]); ok && got == root { + t.Fatalf("n=%d m=%d: truncated path verified", n, m) + } + } + padded := append(append([]Hash(nil), path...), Hash{}) + if got, ok := rootFromPath(leaves[m], uint64(m), uint64(n), padded); ok && got == root { + t.Fatalf("n=%d m=%d: padded path verified", n, m) + } + } + } +} + +// TestMerkleConsistencyExhaustive proves every (older, newer) size pair up to +// 33 and rejects tampers, prefix rewrites, and cross-pair replays. +func TestMerkleConsistencyExhaustive(t *testing.T) { + const max = 33 + leaves := syntheticLeaves(max) + roots := make([]Hash, max+1) // roots[k] = MTH(leaves[:k]), k >= 1 + for k := 1; k <= max; k++ { + roots[k] = mth(leaves[:k]) + } + verify := func(m, n uint64, oldRoot, newRoot Hash, proof []Hash) bool { + oldR, newR, rest, ok := consRoots(m, n, true, proof, oldRoot) + return ok && len(rest) == 0 && oldR == oldRoot && newR == newRoot + } + for n := 2; n <= max; n++ { + for m := 1; m < n; m++ { + proof := proveConsistency(uint64(m), leaves[:n]) + if !verify(uint64(m), uint64(n), roots[m], roots[n], proof) { + t.Fatalf("m=%d n=%d: honest consistency proof rejected", m, n) + } + // Tampered proof element. + if len(proof) > 0 { + bad := append([]Hash(nil), proof...) + bad[len(bad)/2][0] ^= 1 + if verify(uint64(m), uint64(n), roots[m], roots[n], bad) { + t.Fatalf("m=%d n=%d: tampered consistency proof accepted", m, n) + } + } + // A rewritten prefix (different old root) must not verify with any + // honest proof for this pair. + forged := roots[m] + forged[0] ^= 1 + if verify(uint64(m), uint64(n), forged, roots[n], proof) { + t.Fatalf("m=%d n=%d: rewritten prefix accepted", m, n) + } + // A different new root must not verify. + forgedNew := roots[n] + forgedNew[0] ^= 1 + if verify(uint64(m), uint64(n), roots[m], forgedNew, proof) { + t.Fatalf("m=%d n=%d: forged new head accepted", m, n) + } + } + } + // Cross-pair replay: a proof for (m1,n) never verifies as (m2,n), m1 != m2. + n := uint64(21) + for m1 := uint64(1); m1 < n; m1++ { + proof := proveConsistency(m1, leaves[:n]) + for m2 := uint64(1); m2 < n; m2++ { + if m1 == m2 { + continue + } + if verify(m2, n, roots[m2], roots[n], proof) { + t.Fatalf("proof for m=%d replayed as m=%d (n=%d)", m1, m2, n) + } + } + } +} + +// TestLogProveConsistencyMatchesTree ties Log.ProveConsistency to the tree +// primitives through the public API, over a real appended log. +func TestLogProveConsistencyMatchesTree(t *testing.T) { + op := newParty(t) + a, b := newParty(t), newParty(t) + id := LogID(op.pub) + l, err := OpenLog(op.pub, NewMemStore()) + if err != nil { + t.Fatalf("OpenLog: %v", err) + } + sign := func(cp Checkpoint) Checkpoint { + cp.Sign(op.priv) + return cp + } + var cps []Checkpoint + cps = append(cps, sign(l.Checkpoint(testInstant))) + for i := 0; i < 9; i++ { + e := sealedEntry(t, id, a, b, contentN(byte(40+i)), Hash{}, testInstant) + if _, err := l.Append(e); err != nil { + t.Fatalf("Append: %v", err) + } + cps = append(cps, sign(l.Checkpoint(testInstant))) + } + for older := 0; older <= 9; older++ { + proof, err := l.ProveConsistency(uint64(older)) + if err != nil { + t.Fatalf("ProveConsistency(%d): %v", older, err) + } + if !VerifyConsistency(cps[older], cps[9], proof, op.pub) { + t.Fatalf("consistency %d -> 9 rejected", older) + } + } + if _, err := l.ProveConsistency(10); err == nil { + t.Fatal("ProveConsistency beyond the log size must error") + } +} diff --git a/internal/record/verify.go b/internal/record/verify.go index 1008703..23ae15c 100644 --- a/internal/record/verify.go +++ b/internal/record/verify.go @@ -5,19 +5,22 @@ import ( ) // Proof is what a member keeps beside an entry to later prove its inclusion -// under a checkpoint, offline and without the operator. Nothing in it is -// trusted: verification recomputes every hash. +// under a checkpoint, offline and without the operator: the entry's sequence +// number and its RFC-6962 audit path — ~log2(size) hashes however large the +// log grows, which is what makes verification feasible on an entry-level +// device (the open-problem-8 floor). Nothing in it is trusted: verification +// recomputes every hash. type Proof struct { - Prior Hash // chain head immediately before the entry (the LogID seed for the first entry) - Links []Hash // leaf hashes (Entry.ID) of every later entry up to the checkpointed head + Seq uint64 // the entry's position in the log + Path []Hash // sibling subtree hashes, leaf-adjacent first, root-adjacent last } // VerifyInclusion reports whether e is a fully sealed covenant committed at -// position cp.Size-1-len(p.Links) of cp's log: it requires e.Verify(), -// cp.Verify(operator), e.Log == cp.Log, and that folding e.ID() over p.Prior -// and p.Links yields cp.Head. One call checks everything; there is no -// half-verification to forget. It does NOT prove non-inclusion or liveness: -// an operator that never appended the entry simply cannot produce a proof. +// position p.Seq of cp's log: it requires e.Verify(), cp.Verify(operator), +// e.Log == cp.Log, and that the audit path recomputes cp.Head from e.ID(). +// One call checks everything; there is no half-verification to forget. It +// does NOT prove non-inclusion or liveness: an operator that never appended +// the entry simply cannot produce a proof. func VerifyInclusion(e Entry, p Proof, cp Checkpoint, operator ed25519.PublicKey) bool { if !e.Verify() { return false @@ -28,36 +31,35 @@ func VerifyInclusion(e Entry, p Proof, cp Checkpoint, operator ed25519.PublicKey if e.Log != cp.Log { return false } - if cp.Size < uint64(len(p.Links))+1 { - return false - } - h := chainStep(p.Prior, e.ID()) - for _, link := range p.Links { - h = chainStep(h, link) - } - return h == cp.Head + root, ok := rootFromPath(e.ID(), p.Seq, cp.Size, p.Path) + return ok && root == cp.Head } // VerifyConsistency reports whether newer extends older without rewrite: -// both checkpoints verify under operator, same Log, -// newer.Size == older.Size+len(links), and folding links over older.Head -// yields newer.Head. Failure against two operator-signed checkpoints is -// portable fork evidence — this is how any member or witness detects a -// silent edit or deletion. It proves nothing about entries the older -// checkpoint never covered. -func VerifyConsistency(older, newer Checkpoint, links []Hash, operator ed25519.PublicKey) bool { +// both checkpoints verify under operator, same Log, and the RFC-6962 +// consistency proof recomputes BOTH heads — older.Head from its components +// and newer.Head from the same components plus the extension. Failure +// against two operator-signed checkpoints is portable fork evidence — this +// is how any member or witness detects a silent edit or deletion. It proves +// nothing about entries the older checkpoint never covered. +// +// The empty older log (Size 0) is consistent with anything on an empty +// proof; equal sizes are consistent only when the heads are equal. +func VerifyConsistency(older, newer Checkpoint, proof []Hash, operator ed25519.PublicKey) bool { if !older.Verify(operator) || !newer.Verify(operator) { return false } if older.Log != newer.Log { return false } - if newer.Size != older.Size+uint64(len(links)) { + switch { + case older.Size > newer.Size: return false + case older.Size == 0: + return len(proof) == 0 + case older.Size == newer.Size: + return len(proof) == 0 && older.Head == newer.Head } - h := older.Head - for _, link := range links { - h = chainStep(h, link) - } - return h == newer.Head + oldR, newR, rest, ok := consRoots(older.Size, newer.Size, true, proof, older.Head) + return ok && len(rest) == 0 && oldR == older.Head && newR == newer.Head } diff --git a/internal/record/witness.go b/internal/record/witness.go index a07783c..b7778c9 100644 --- a/internal/record/witness.go +++ b/internal/record/witness.go @@ -11,7 +11,7 @@ import ( // domainWitness tags witness countersignature payloads, distinct from the // checkpoint tag, so an operator signature can never pose as a witness // cosignature or vice versa. -const domainWitness = "drops/witness/v0" +const domainWitness = "drops/witness/v1" // witnessPayload is the canonical payload a witness countersigns: the // checkpoint's canonical bytes wrapped under the witness domain tag. @@ -78,11 +78,12 @@ func (w *Witness) Key() ed25519.PublicKey { // Countersign verifies cp under operator, refuses when operator equals the // witness's own key, refuses any rollback or fork against the last -// checkpoint it cosigned for cp.Log (using sinceLeaves as extension evidence -// via VerifyConsistency), then returns its cosignature and remembers cp. -// Countersigning a rewritten head is the one thing a witness exists to never -// do, and here it is enforced, not advised. -func (w *Witness) Countersign(cp Checkpoint, operator ed25519.PublicKey, sinceLeaves []Hash) (Countersignature, error) { +// checkpoint it cosigned for cp.Log (using the RFC-6962 consistency proof as +// extension evidence via VerifyConsistency; Log.ProveConsistency produces +// it), then returns its cosignature and remembers cp. Countersigning a +// rewritten head is the one thing a witness exists to never do, and here it +// is enforced, not advised. +func (w *Witness) Countersign(cp Checkpoint, operator ed25519.PublicKey, consistencyProof []Hash) (Countersignature, error) { if len(w.priv) != ed25519.PrivateKeySize { return Countersignature{}, errors.New("record: witness key is malformed") } @@ -93,7 +94,7 @@ func (w *Witness) Countersign(cp Checkpoint, operator ed25519.PublicKey, sinceLe return Countersignature{}, errors.New("record: checkpoint does not verify under operator") } if prev, seen := w.last[cp.Log]; seen { - if !VerifyConsistency(prev, cp, sinceLeaves, operator) { + if !VerifyConsistency(prev, cp, consistencyProof, operator) { return Countersignature{}, errors.New("record: checkpoint is not a consistent extension of the last cosigned checkpoint (rollback or fork refused)") } } diff --git a/internal/record/witness_test.go b/internal/record/witness_test.go index 5b9f5d9..0344a29 100644 --- a/internal/record/witness_test.go +++ b/internal/record/witness_test.go @@ -40,7 +40,11 @@ func newWitnessFixture(t *testing.T) witnessFixture { } } f.cp5 = sign(l.Checkpoint(testInstant)) - f.links2to5 = mustLeaves(t, l, 2) + proof, err := l.ProveConsistency(2) + if err != nil { + t.Fatalf("ProveConsistency(2): %v", err) + } + f.links2to5 = proof fork := f.cp2 fork.Head[0] ^= 1