From 2196e2939e22c47d195344839a5bb52ebd899c23 Mon Sep 17 00:00:00 2001 From: Giorgi Chomakhashvili <133794518+kaikisegfault@users.noreply.github.com> Date: Fri, 24 Jul 2026 22:24:13 +0200 Subject: [PATCH] feat(storage): persist and replay durable blocks Apply blocks to an independent ledger candidate, commit materialized state and canonical admitted history atomically, and publish only after durable commit. Reopen now validates trusted genesis, full-replays ordered history, compares every kernel output, and requires exact materialized-head agreement.\n\nStorage schema version 1 and consensus-visible kernel behavior are unchanged.\n\nRefs #11 --- CMakeLists.txt | 20 ++ docs/architecture/sqlite-ledger.md | 44 ++- docs/project/current-state.md | 40 ++- include/protocol/storage/sqlite_ledger.hpp | 10 +- src/storage/sqlite_block_store_v1.cpp | 204 ++++++++++++++ src/storage/sqlite_history_replay_v1.cpp | 217 +++++++++++++++ src/storage/sqlite_history_v1.hpp | 23 ++ src/storage/sqlite_ledger.cpp | 93 ++++++- src/storage/sqlite_schema_v1.cpp | 52 ++-- src/storage/sqlite_schema_v1.hpp | 11 +- tests/storage/sqlite_history_test.cpp | 301 +++++++++++++++++++++ tests/storage/sqlite_ledger_test.cpp | 2 +- 12 files changed, 965 insertions(+), 52 deletions(-) create mode 100644 src/storage/sqlite_block_store_v1.cpp create mode 100644 src/storage/sqlite_history_replay_v1.cpp create mode 100644 src/storage/sqlite_history_v1.hpp create mode 100644 tests/storage/sqlite_history_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index ff38803..431cc08 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -164,6 +164,8 @@ add_library( protocol_storage STATIC src/storage/sqlite_connection.cpp + src/storage/sqlite_block_store_v1.cpp + src/storage/sqlite_history_replay_v1.cpp src/storage/sqlite_ledger.cpp src/storage/sqlite_schema_v1.cpp ) @@ -233,12 +235,22 @@ add_executable( storage_sqlite_ledger_tests tests/storage/sqlite_ledger_test.cpp ) +add_executable( + storage_sqlite_history_tests + tests/storage/sqlite_history_test.cpp +) target_link_libraries( storage_sqlite_ledger_tests PRIVATE protocol_storage protocol_stack_sqlite ) +target_link_libraries( + storage_sqlite_history_tests + PRIVATE + protocol_storage + protocol_stack_sqlite +) set( PROTOCOL_STACK_TARGETS @@ -257,6 +269,7 @@ set( kernel_differential_runner sqlite_dependency_tests storage_sqlite_ledger_tests + storage_sqlite_history_tests ) if(PROTOCOL_STACK_ENABLE_FUZZING) add_library( @@ -443,6 +456,13 @@ add_test( "${PROJECT_SOURCE_DIR}/test-vectors/ledger-transition-v1.txt" "${CMAKE_CURRENT_BINARY_DIR}/storage-sqlite-ledger-test" ) +add_test( + NAME storage-sqlite-history + COMMAND + storage_sqlite_history_tests + "${PROJECT_SOURCE_DIR}/test-vectors/ledger-transition-v1.txt" + "${CMAKE_CURRENT_BINARY_DIR}/storage-sqlite-history-test" +) set_tests_properties( protocol-primitives-python ledger-transition-python diff --git a/docs/architecture/sqlite-ledger.md b/docs/architecture/sqlite-ledger.md index d237ea1..5345889 100644 --- a/docs/architecture/sqlite-ledger.md +++ b/docs/architecture/sqlite-ledger.md @@ -9,17 +9,29 @@ remain authoritative. ## Implemented outcome -`create_sqlite_ledger` now creates a brand-new database at height zero and -commits the caller-configured canonical genesis state. `open_sqlite_ledger` -reopens only that closed-world height-zero form. It rejects any database with -block, admitted-transaction, or snapshot rows until replay and recovery are -implemented. +`create_sqlite_ledger` creates a brand-new database at height zero and commits +the caller-configured canonical genesis state. `SQLiteLedger::apply_block` +copy-constructs an independent kernel candidate, applies the complete ordered +raw-input block, persists the resulting materialized state and canonical +history in one transaction, commits, and publishes the candidate through a +non-throwing pointer swap. A kernel block rejection never starts a storage +transaction. + +`open_sqlite_ledger` performs full genesis replay of every retained block +before publishing a live ledger. It admits only the stored 200-byte journal +rows, in explicit height and ordinal order, and compares every replayed +transaction ID, receipt, root, application header, and block ID with the +stored canonical output. The replay head must then exactly equal metadata, +materialized accounts, the fee pool, and the public ledger root. Snapshot rows +remain refused until independent snapshot recovery is implemented. The public header exposes no SQLite handle or SQL type. `SQLiteLedger` is move-constructible but not copyable or assignable. It owns the live `protocol::v1::Ledger`, serialized connection, normalized path, exact canonical genesis bytes, and cached state root. `read_head` returns owned state and root values while holding the adapter mutex; callers receive no borrowed view. +`apply_block` returns one of the exact kernel `BlockCommit`, the deterministic +kernel `BlockError`, or an operational `SQLiteLedgerError`. ## Trusted creation input @@ -76,10 +88,17 @@ rowid, and foreign-key metadata; unknown or modified objects are refused. Opening requires a single successful `integrity_check` result and an empty `foreign_key_check`. It exact-compares the persisted canonical genesis with -the independently trusted caller value, bounds account loading by the trusted -genesis account count, reconstructs an owned state, and calls +the independently trusted caller value, replays contiguous block and admitted +transaction rows from genesis, bounds materialized account loading by the +verified replay account count, reconstructs an owned state, and calls `restore_ledger` with caller-derived immutable parameters and the stored root. -The result must exactly equal the independently loaded genesis state and root. +The result must exactly equal the independently replayed state and root. + +Admission failures are absent from the journal. Empty and entirely unadmitted +blocks still have a block row and advance height. Duplicate admitted +transactions retain separate contiguous ordinals. Database projections never +replace canonical transaction, receipt, header, or identifier bytes as the +replay authority. ## Error and lifetime behavior @@ -96,8 +115,7 @@ before the completed adapter is returned. ## Remaining issue 11 work -The current boundary deliberately cannot apply or replay a block. The next -vertical result is one atomic durable block commit followed by validated clean -reopen and full genesis replay. Snapshot recovery, portable export/import, -fault injection around commit phases, long restart sequences, and final issue -closure follow that working block path. +The ordinary durable commit and full-genesis-replay path is implemented. +Snapshot recovery, portable export/import, automatic reopen after an ambiguous +commit result, fault injection around every commit phase, long seeded restart +sequences, and final issue closure remain. diff --git a/docs/project/current-state.md b/docs/project/current-state.md index 43c2079..3fd1860 100644 --- a/docs/project/current-state.md +++ b/docs/project/current-state.md @@ -6,8 +6,9 @@ Last updated: 2026-07-24 M1 — Sovereign Devnet Alpha. The deterministic in-memory ledger kernel is merged and verified. The owning storage adapter can now durably create and -validate a height-zero ledger. Atomic block persistence, replay, snapshots, -and recovery remain the active roadmap slice. +validate a height-zero ledger, atomically persist a complete block, and reopen +the identical head through full genesis replay. Multi-block restart coverage, +snapshots, fault injection, and recovery remain the active roadmap slice. ## Verified facts @@ -171,9 +172,17 @@ and recovery remain the active roadmap slice. Reopening never creates or changes journal mode and publishes no ledger unless integrity, foreign keys, exact schema, caller-trusted genesis, materialized state, and root all agree. -- The current adapter is deliberately closed at height zero. Any block, - admitted-transaction, or snapshot row is rejected until complete replay and - recovery validation are implemented. +- `SQLiteLedger::apply_block` applies ordered raw inputs to an independent + ledger candidate, writes changed and created accounts, exact admitted + transaction bytes and kernel outputs, the block row, and head metadata in + one SQLite transaction, durably commits, and publishes through a + non-throwing owning-pointer swap. Kernel block rejection never opens a + storage transaction. +- Opening validates caller-trusted genesis before history, then full-replays + contiguous block and admitted-transaction rows in explicit height and + ordinal order. Every replayed transaction ID, receipt, root, application + header, and block ID must equal storage before the replay head is compared + exactly with materialized state and metadata. Snapshot rows remain refused. - The owner prefers one active delivery branch, cleanup of obsolete branches/worktrees/build trees at phase boundaries, focused checks while iterating, one required completion matrix, and runnable vertical outcomes @@ -203,16 +212,27 @@ and recovery remain the active roadmap slice. wrong genesis, materialized-state and root corruption, immutable-fee projection corruption, foreign-key damage, truncated files, and refusal of unvalidated history. +- The frozen 15-input ledger block now passes through durable storage with the + exact kernel `BlockCommit`, 11 admitted journal rows, three omitted + admission failures, unchanged head on a rejected repeated height, clean + close, full genesis replay, and an identical owned head. +- Replay rejection coverage proves wrong-genesis precedence at nonzero height, + missing admitted ordinals, altered transaction and block identifiers, and + materialized state divergence are refused before publication. +- Clean completion verification passes 16/16 CTest tests in GCC debug, GCC + ASan+UBSan, and Clang debug, and 19/19 in Clang ASan+UBSan including the + three existing kernel fuzz smoke tests. Leak detection is disabled only for + sanitizer completion runs because the managed execution sandbox traces + processes and LeakSanitizer refuses to run under `ptrace`. ## Exact next action Continue issue #11: -> Implement one end-to-end durable block: apply ordered raw inputs to an -> independent ledger candidate, atomically persist materialized state, exact -> admitted bytes and kernel outputs, commit, publish with a non-throwing -> ownership transfer, then cleanly reopen by full genesis replay to the -> identical head. +> Add a deterministic multi-block and repeated-restart storage harness covering +> empty and entirely unadmitted blocks, duplicate admitted transactions, +> continued commits from reopened heads, exact replay after each restart, and +> materialized-state agreement throughout. ## Open autonomous decisions diff --git a/include/protocol/storage/sqlite_ledger.hpp b/include/protocol/storage/sqlite_ledger.hpp index 357d047..e55706c 100644 --- a/include/protocol/storage/sqlite_ledger.hpp +++ b/include/protocol/storage/sqlite_ledger.hpp @@ -1,6 +1,6 @@ #pragma once -#include "protocol/v1/types.hpp" +#include "protocol/v1/ledger.hpp" #include #include @@ -33,6 +33,11 @@ struct LedgerHead { struct SQLiteLedgerResult; +using SQLiteBlockResult = std::variant< + protocol::v1::BlockCommit, + protocol::v1::BlockError, + SQLiteLedgerError>; + class SQLiteLedger { public: ~SQLiteLedger() noexcept; @@ -43,6 +48,9 @@ class SQLiteLedger { SQLiteLedger& operator=(SQLiteLedger&&) = delete; LedgerHead read_head() const; + SQLiteBlockResult apply_block( + std::uint64_t height, + std::span raw_transactions); private: struct Impl; diff --git a/src/storage/sqlite_block_store_v1.cpp b/src/storage/sqlite_block_store_v1.cpp new file mode 100644 index 0000000..60dec292 --- /dev/null +++ b/src/storage/sqlite_block_store_v1.cpp @@ -0,0 +1,204 @@ +#include "sqlite_history_v1.hpp" + +#include + +#include +#include +#include +#include +#include + +namespace protocol::storage::internal { +namespace { + +namespace pv1 = protocol::v1; +constexpr std::size_t kMaximumBlockInputs = 65'535; + +[[noreturn]] void fail() { + throw Failure{SQLiteLedgerError::storage_failure}; +} + +void require_done(Statement& statement) { + if (statement.step() != SQLITE_DONE) fail(); +} + +void require_single_change(Connection& connection) { + if (sqlite3_changes(connection.get()) != 1) fail(); +} + +std::array encode_u64(std::uint64_t value) { + std::array encoded{}; + for (std::size_t index = 0; index < encoded.size(); ++index) { + const auto shift = static_cast( + (encoded.size() - index - 1) * 8); + encoded[index] = static_cast(value >> shift); + } + return encoded; +} + +std::array encode_u32(std::uint32_t value) { + std::array encoded{}; + for (std::size_t index = 0; index < encoded.size(); ++index) { + const auto shift = static_cast( + (encoded.size() - index - 1) * 8); + encoded[index] = static_cast(value >> shift); + } + return encoded; +} + +template +std::span tagged_bytes( + const Tagged& value) noexcept { + return {value.data(), value.size()}; +} + +std::size_t require_commit_shape( + const pv1::State& previous_state, + const pv1::State& resulting_state, + std::span raw_transactions, + const pv1::BlockCommit& commit) { + if (previous_state.height == std::numeric_limits::max() || + resulting_state.height != previous_state.height + 1 || + commit.height != resulting_state.height || + previous_state.parameters != resulting_state.parameters || + commit.admissions.size() != raw_transactions.size() || + commit.transaction_ids.size() != commit.receipts.size() || + commit.transaction_ids.size() != commit.encoded_receipts.size() || + commit.transaction_ids.size() > kMaximumBlockInputs || + commit.header.size() != 146) { + fail(); + } + + std::size_t admitted_count = 0; + for (const auto& admission : commit.admissions) { + if (!admission) ++admitted_count; + } + if (admitted_count != commit.transaction_ids.size()) fail(); + return admitted_count; +} + +void persist_accounts( + Connection& connection, + const pv1::State& previous_state, + const pv1::State& resulting_state) { + for (const auto& [account_id, account] : previous_state.accounts) { + const auto found = resulting_state.accounts.find(account_id); + if (found == resulting_state.accounts.end()) fail(); + (void)account; + } + + Statement upsert = connection.prepare( + "INSERT INTO accounts(account_id, balance, nonce) VALUES(?, ?, ?) " + "ON CONFLICT(account_id) DO UPDATE SET " + "balance=excluded.balance, nonce=excluded.nonce"); + for (const auto& [account_id, account] : resulting_state.accounts) { + const auto previous = previous_state.accounts.find(account_id); + if (previous != previous_state.accounts.end() && + previous->second == account) { + continue; + } + const auto balance = encode_u64(account.balance); + const auto nonce = encode_u64(account.nonce); + upsert.bind_blob(1, tagged_bytes(account_id)); + upsert.bind_blob(2, balance); + upsert.bind_blob(3, nonce); + require_done(upsert); + require_single_change(connection); + upsert.reset(); + } +} + +void persist_block_row( + Connection& connection, + const pv1::BlockCommit& commit, + std::size_t admitted_count) { + Statement insert = connection.prepare( + "INSERT INTO blocks(" + "height, previous_state_root, transaction_root, " + "resulting_state_root, admitted_count, header, block_id" + ") VALUES(?, ?, ?, ?, ?, ?, ?)"); + const auto height = encode_u64(commit.height); + const auto count = + encode_u32(static_cast(admitted_count)); + insert.bind_blob(1, height); + insert.bind_blob(2, tagged_bytes(commit.previous_state_root)); + insert.bind_blob(3, tagged_bytes(commit.transaction_root)); + insert.bind_blob(4, tagged_bytes(commit.resulting_state_root)); + insert.bind_blob(5, count); + insert.bind_blob(6, commit.header); + insert.bind_blob(7, tagged_bytes(commit.block_id)); + require_done(insert); + require_single_change(connection); +} + +void persist_admitted_transactions( + Connection& connection, + std::span raw_transactions, + const pv1::BlockCommit& commit) { + Statement insert = connection.prepare( + "INSERT INTO admitted_transactions(" + "height, ordinal, transaction_bytes, transaction_id, receipt_bytes" + ") VALUES(?, ?, ?, ?, ?)"); + const auto height = encode_u64(commit.height); + std::uint32_t ordinal = 0; + for (std::size_t raw_index = 0; + raw_index < raw_transactions.size(); ++raw_index) { + if (commit.admissions[raw_index]) continue; + const auto& raw = raw_transactions[raw_index]; + const auto& receipt = commit.encoded_receipts[ordinal]; + if (raw.size() != 200 || receipt.size() != 47) fail(); + const auto encoded_ordinal = encode_u32(ordinal); + insert.bind_blob(1, height); + insert.bind_blob(2, encoded_ordinal); + insert.bind_blob(3, raw); + insert.bind_blob( + 4, tagged_bytes(commit.transaction_ids[ordinal])); + insert.bind_blob(5, receipt); + require_done(insert); + require_single_change(connection); + insert.reset(); + ++ordinal; + } +} + +void update_metadata( + Connection& connection, + const pv1::State& previous_state, + const pv1::State& resulting_state, + const pv1::BlockCommit& commit) { + Statement update = connection.prepare( + "UPDATE ledger_meta SET " + "current_height=?, fee_pool=?, current_state_root=? " + "WHERE singleton=1 AND current_height=? " + "AND current_state_root=?"); + const auto next_height = encode_u64(resulting_state.height); + const auto next_fee_pool = encode_u64(resulting_state.fee_pool); + const auto previous_height = encode_u64(previous_state.height); + update.bind_blob(1, next_height); + update.bind_blob(2, next_fee_pool); + update.bind_blob(3, tagged_bytes(commit.resulting_state_root)); + update.bind_blob(4, previous_height); + update.bind_blob(5, tagged_bytes(commit.previous_state_root)); + require_done(update); + require_single_change(connection); +} + +} // namespace + +void persist_block_v1( + Connection& connection, + const pv1::State& previous_state, + const pv1::State& resulting_state, + std::span raw_transactions, + const pv1::BlockCommit& commit) { + if (connection.autocommit()) fail(); + const auto admitted_count = require_commit_shape( + previous_state, resulting_state, raw_transactions, commit); + persist_accounts(connection, previous_state, resulting_state); + persist_block_row(connection, commit, admitted_count); + persist_admitted_transactions(connection, raw_transactions, commit); + update_metadata( + connection, previous_state, resulting_state, commit); +} + +} // namespace protocol::storage::internal diff --git a/src/storage/sqlite_history_replay_v1.cpp b/src/storage/sqlite_history_replay_v1.cpp new file mode 100644 index 0000000..967c660 --- /dev/null +++ b/src/storage/sqlite_history_replay_v1.cpp @@ -0,0 +1,217 @@ +#include "sqlite_history_v1.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace protocol::storage::internal { +namespace { + +namespace pv1 = protocol::v1; + +constexpr std::uint32_t kMaximumBlockInputs = 65'535; + +struct StoredJournal { + std::vector transactions; + std::vector transaction_ids; + std::vector receipts; +}; + +[[noreturn]] void fail(SQLiteLedgerError error) { + throw Failure{error}; +} + +int checked_step(Statement& statement) { + const auto result = statement.step(); + if (result != SQLITE_ROW && result != SQLITE_DONE) { + fail(SQLiteLedgerError::storage_failure); + } + return result; +} + +std::array encode_u64(std::uint64_t value) { + std::array encoded{}; + for (std::size_t index = 0; index < encoded.size(); ++index) { + const auto shift = static_cast( + (encoded.size() - index - 1) * 8); + encoded[index] = static_cast(value >> shift); + } + return encoded; +} + +std::uint64_t decode_unsigned( + const Statement& statement, + int index, + std::size_t width) { + if (statement.column_type(index) != SQLITE_BLOB) { + fail(SQLiteLedgerError::state_mismatch); + } + const auto bytes = statement.column_blob(index); + if (bytes.size() != width) { + fail(SQLiteLedgerError::state_mismatch); + } + std::uint64_t value = 0; + for (const auto byte : bytes) value = (value << 8U) | byte; + return value; +} + +pv1::Bytes decode_bytes( + const Statement& statement, + int index, + std::size_t width) { + if (statement.column_type(index) != SQLITE_BLOB) { + fail(SQLiteLedgerError::state_mismatch); + } + const auto bytes = statement.column_blob(index); + if (bytes.size() != width) { + fail(SQLiteLedgerError::state_mismatch); + } + return pv1::Bytes(bytes.begin(), bytes.end()); +} + +template +Tagged decode_tagged_hash( + const Statement& statement, + int index) { + const auto bytes = decode_bytes(statement, index, pv1::Hash{}.size()); + pv1::Hash value{}; + std::copy(bytes.begin(), bytes.end(), value.begin()); + return Tagged{value}; +} + +StoredJournal load_journal( + Connection& connection, + std::uint64_t height, + std::uint32_t admitted_count) { + Statement rows = connection.prepare( + "SELECT ordinal, transaction_bytes, transaction_id, receipt_bytes " + "FROM admitted_transactions WHERE height=? ORDER BY ordinal"); + const auto encoded_height = encode_u64(height); + rows.bind_blob(1, encoded_height); + + StoredJournal stored; + stored.transactions.reserve(admitted_count); + stored.transaction_ids.reserve(admitted_count); + stored.receipts.reserve(admitted_count); + while (checked_step(rows) == SQLITE_ROW) { + if (rows.column_count() != 4 || + stored.transactions.size() >= admitted_count) { + fail(SQLiteLedgerError::state_mismatch); + } + const auto ordinal = decode_unsigned(rows, 0, 4); + if (ordinal != stored.transactions.size()) { + fail(SQLiteLedgerError::state_mismatch); + } + stored.transactions.push_back(decode_bytes(rows, 1, 200)); + stored.transaction_ids.push_back( + decode_tagged_hash(rows, 2)); + stored.receipts.push_back(decode_bytes(rows, 3, 47)); + } + if (stored.transactions.size() != admitted_count) { + fail(SQLiteLedgerError::state_mismatch); + } + return stored; +} + +bool all_admitted(const pv1::BlockCommit& commit) { + return std::all_of( + commit.admissions.begin(), commit.admissions.end(), + [](const auto& admission) { return !admission.has_value(); }); +} + +void require_replay_match( + const pv1::BlockCommit& replayed, + const StoredJournal& stored, + const pv1::StateRoot& previous_root, + const pv1::TransactionRoot& transaction_root, + const pv1::StateRoot& resulting_root, + const pv1::Bytes& header, + const pv1::BlockId& block_id) { + if (replayed.admissions.size() != stored.transactions.size() || + !all_admitted(replayed) || + replayed.transaction_ids != stored.transaction_ids || + replayed.encoded_receipts != stored.receipts || + replayed.receipts.size() != stored.receipts.size() || + replayed.previous_state_root != previous_root || + replayed.transaction_root != transaction_root || + replayed.resulting_state_root != resulting_root || + replayed.header != header || + replayed.block_id != block_id) { + fail(SQLiteLedgerError::state_mismatch); + } +} + +void replay_block_row( + Connection& connection, + Statement& blocks, + pv1::Ledger& ledger) { + if (blocks.column_count() != 7 || + ledger.state().height == std::numeric_limits::max()) { + fail(SQLiteLedgerError::state_mismatch); + } + const auto height = decode_unsigned(blocks, 0, 8); + if (height != ledger.state().height + 1) { + fail(SQLiteLedgerError::state_mismatch); + } + const auto previous_root = + decode_tagged_hash(blocks, 1); + const auto transaction_root = + decode_tagged_hash(blocks, 2); + const auto resulting_root = + decode_tagged_hash(blocks, 3); + const auto admitted_count_value = decode_unsigned(blocks, 4, 4); + if (admitted_count_value > kMaximumBlockInputs) { + fail(SQLiteLedgerError::state_mismatch); + } + const auto admitted_count = + static_cast(admitted_count_value); + const auto header = decode_bytes(blocks, 5, 146); + const auto block_id = + decode_tagged_hash(blocks, 6); + const auto stored = + load_journal(connection, height, admitted_count); + + auto applied = ledger.apply_block(height, stored.transactions); + if (!std::holds_alternative(applied)) { + fail(SQLiteLedgerError::state_mismatch); + } + const auto& replayed = std::get(applied); + require_replay_match( + replayed, stored, previous_root, transaction_root, + resulting_root, header, block_id); +} + +void require_no_snapshots(Connection& connection) { + Statement snapshots = + connection.prepare("SELECT 1 FROM snapshots LIMIT 1"); + if (checked_step(snapshots) != SQLITE_DONE) { + fail(SQLiteLedgerError::state_mismatch); + } +} + +} // namespace + +pv1::Ledger replay_history_v1( + Connection& connection, + const pv1::Ledger& trusted_genesis_ledger) { + pv1::Ledger ledger(trusted_genesis_ledger); + Statement blocks = connection.prepare( + "SELECT height, previous_state_root, transaction_root, " + "resulting_state_root, admitted_count, header, block_id " + "FROM blocks ORDER BY height"); + while (checked_step(blocks) == SQLITE_ROW) { + replay_block_row(connection, blocks, ledger); + } + require_no_snapshots(connection); + return ledger; +} + +} // namespace protocol::storage::internal diff --git a/src/storage/sqlite_history_v1.hpp b/src/storage/sqlite_history_v1.hpp new file mode 100644 index 0000000..f7e2c52 --- /dev/null +++ b/src/storage/sqlite_history_v1.hpp @@ -0,0 +1,23 @@ +#pragma once + +#include "sqlite_connection.hpp" + +#include "protocol/v1/ledger.hpp" + +#include +#include + +namespace protocol::storage::internal { + +void persist_block_v1( + Connection& connection, + const protocol::v1::State& previous_state, + const protocol::v1::State& resulting_state, + std::span raw_transactions, + const protocol::v1::BlockCommit& commit); + +protocol::v1::Ledger replay_history_v1( + Connection& connection, + const protocol::v1::Ledger& trusted_genesis_ledger); + +} // namespace protocol::storage::internal diff --git a/src/storage/sqlite_ledger.cpp b/src/storage/sqlite_ledger.cpp index a7374e4..9debad4 100644 --- a/src/storage/sqlite_ledger.cpp +++ b/src/storage/sqlite_ledger.cpp @@ -1,6 +1,7 @@ #include "protocol/storage/sqlite_ledger.hpp" #include "sqlite_connection.hpp" +#include "sqlite_history_v1.hpp" #include "sqlite_schema_v1.hpp" #include "protocol/v1/ledger.hpp" @@ -47,15 +48,32 @@ TrustedGenesis load_trusted_genesis( }; } -Ledger validate_height_zero( +StateRoot require_root( + const Ledger& ledger, + SQLiteLedgerError error) { + auto root = ledger.current_state_root(); + if (!std::holds_alternative(root)) { + throw internal::Failure{error}; + } + return std::get(std::move(root)); +} + +Ledger validate_durable_ledger( internal::Connection& connection, std::span canonical_genesis, const Ledger& trusted_genesis, const StateRoot& trusted_root) { internal::validate_integrity(connection); internal::validate_schema_v1(connection); - return internal::load_height_zero( + internal::validate_stored_genesis( connection, canonical_genesis, trusted_genesis, trusted_root); + auto replayed = + internal::replay_history_v1(connection, trusted_genesis); + const auto replayed_root = require_root( + replayed, SQLiteLedgerError::state_mismatch); + return internal::load_materialized_ledger( + connection, canonical_genesis, trusted_genesis, trusted_root, + replayed, replayed_root); } SQLiteLedgerResult error_result(SQLiteLedgerError error) { @@ -74,6 +92,7 @@ struct SQLiteLedger::Impl { mutable std::mutex mutex; std::unique_ptr ledger; StateRoot state_root; + bool poisoned; Impl( std::filesystem::path normalized_path, @@ -85,11 +104,15 @@ struct SQLiteLedger::Impl { canonical_genesis(std::move(exact_genesis)), resources(std::move(sqlite_resources)), ledger(std::move(live_ledger)), - state_root(std::move(verified_root)) {} + state_root(std::move(verified_root)), + poisoned(false) {} }; static_assert(std::is_nothrow_move_constructible_v); static_assert(std::is_nothrow_destructible_v); +static_assert(std::is_nothrow_move_constructible_v); +static_assert(std::is_nothrow_swappable_v>); +static_assert(std::is_nothrow_copy_assignable_v); SQLiteLedger::SQLiteLedger( std::unique_ptr implementation) noexcept @@ -106,6 +129,60 @@ LedgerHead SQLiteLedger::read_head() const { }; } +SQLiteBlockResult SQLiteLedger::apply_block( + std::uint64_t height, + std::span raw_transactions) { + const std::lock_guard lock(implementation_->mutex); + if (implementation_->poisoned) { + return SQLiteBlockResult( + std::in_place_type, + SQLiteLedgerError::storage_failure); + } + + auto candidate = + std::make_unique(*implementation_->ledger); + auto applied = candidate->apply_block(height, raw_transactions); + if (std::holds_alternative(applied)) { + return SQLiteBlockResult( + std::in_place_type, + std::get(applied)); + } + SQLiteBlockResult result( + std::in_place_type, + std::get(std::move(applied))); + const auto& commit = std::get(result); + + try { + auto& connection = implementation_->resources.connection; + internal::verify_stable_path( + implementation_->resources, implementation_->path); + internal::begin_exclusive(connection); + try { + internal::persist_block_v1( + connection, implementation_->ledger->state(), + candidate->state(), raw_transactions, commit); + internal::verify_stable_path( + implementation_->resources, implementation_->path); + } catch (...) { + internal::rollback_or_terminate(connection); + throw; + } + try { + internal::commit(connection); + } catch (...) { + implementation_->poisoned = true; + throw; + } + + implementation_->ledger.swap(candidate); + implementation_->state_root = commit.resulting_state_root; + return result; + } catch (const internal::Failure& failure) { + return SQLiteBlockResult( + std::in_place_type, failure.error); + } +} + SQLiteLedgerResult create_sqlite_ledger( const std::filesystem::path& path, std::span canonical_genesis) { @@ -133,7 +210,7 @@ SQLiteLedgerResult create_sqlite_ledger( internal::install_schema_v1( connection, stored_genesis, *implementation->ledger, implementation->state_root); - auto validation = validate_height_zero( + auto validation = validate_durable_ledger( connection, stored_genesis, *implementation->ledger, implementation->state_root); (void)validation; @@ -147,7 +224,7 @@ SQLiteLedgerResult create_sqlite_ledger( internal::verify_stable_path( implementation->resources, implementation->path); - auto durable_validation = validate_height_zero( + auto durable_validation = validate_durable_ledger( connection, stored_genesis, *implementation->ledger, implementation->state_root); (void)durable_validation; @@ -174,15 +251,17 @@ SQLiteLedgerResult open_sqlite_ledger( internal::require_existing_journal_mode(resources.connection); internal::verify_stable_path(resources, normalized); - auto live = validate_height_zero( + auto live = validate_durable_ledger( resources.connection, bytes_view(trusted.canonical_bytes), trusted.ledger, trusted.state_root); + const auto live_root = + require_root(live, SQLiteLedgerError::state_mismatch); auto implementation = std::make_unique( std::move(normalized), std::move(trusted.canonical_bytes), std::move(resources), std::make_unique(std::move(live)), - trusted.state_root); + live_root); return SQLiteLedgerResult{ std::variant( diff --git a/src/storage/sqlite_schema_v1.cpp b/src/storage/sqlite_schema_v1.cpp index 9311f52..7b6310d 100644 --- a/src/storage/sqlite_schema_v1.cpp +++ b/src/storage/sqlite_schema_v1.cpp @@ -501,11 +501,6 @@ void validate_foreign_keys(Connection& connection, const TableSpec& table) { require_done(keys, SQLiteLedgerError::schema_mismatch); } -void require_empty_table(Connection& connection, const char* sql) { - Statement rows = connection.prepare(sql); - require_done(rows, SQLiteLedgerError::state_mismatch); -} - std::map load_accounts( Connection& connection, std::size_t expected_count) { @@ -592,13 +587,42 @@ void validate_schema_v1(Connection& connection) { } } -pv1::Ledger load_height_zero( +void validate_stored_genesis( Connection& connection, std::span expected_genesis, const pv1::Ledger& trusted_genesis_ledger, const pv1::StateRoot& trusted_genesis_root) { require_trusted_genesis( expected_genesis, trusted_genesis_ledger, trusted_genesis_root); + Statement metadata = connection.prepare( + "SELECT singleton, canonical_genesis " + "FROM ledger_meta ORDER BY singleton"); + require_row(metadata, SQLiteLedgerError::state_mismatch); + if (metadata.column_count() != 2 || + !integer_column(metadata, 0, 1) || + metadata.column_type(1) != SQLITE_BLOB) { + fail(SQLiteLedgerError::state_mismatch); + } + const auto stored_genesis = metadata.column_blob(1); + if (stored_genesis.size() != expected_genesis.size() || + !std::equal( + stored_genesis.begin(), stored_genesis.end(), + expected_genesis.begin())) { + fail(SQLiteLedgerError::genesis_mismatch); + } + require_done(metadata, SQLiteLedgerError::state_mismatch); +} + +pv1::Ledger load_materialized_ledger( + Connection& connection, + std::span expected_genesis, + const pv1::Ledger& trusted_genesis_ledger, + const pv1::StateRoot& trusted_genesis_root, + const pv1::Ledger& expected_ledger, + const pv1::StateRoot& expected_root) { + validate_stored_genesis( + connection, + expected_genesis, trusted_genesis_ledger, trusted_genesis_root); Statement metadata = connection.prepare( "SELECT singleton, canonical_genesis, chain_id, supply_limit, " @@ -634,14 +658,6 @@ pv1::Ledger load_height_zero( const auto stored_root = decode_tagged_hash( metadata, 8, SQLiteLedgerError::state_mismatch); require_done(metadata, SQLiteLedgerError::state_mismatch); - if (height != 0) fail(SQLiteLedgerError::state_mismatch); - - require_empty_table( - connection, "SELECT 1 FROM blocks LIMIT 1"); - require_empty_table( - connection, "SELECT 1 FROM admitted_transactions LIMIT 1"); - require_empty_table( - connection, "SELECT 1 FROM snapshots LIMIT 1"); pv1::State persisted_state{ pv1::Parameters{ @@ -653,7 +669,7 @@ pv1::Ledger load_height_zero( height, fee_pool, load_accounts( - connection, trusted_genesis_ledger.state().accounts.size()), + connection, expected_ledger.state().accounts.size()), }; auto restored = pv1::restore_ledger( std::move(persisted_state), @@ -663,11 +679,11 @@ pv1::Ledger load_height_zero( fail(SQLiteLedgerError::state_mismatch); } auto ledger = std::get(std::move(restored.result)); - if (stored_root != trusted_genesis_root || - ledger.state() != trusted_genesis_ledger.state() || + if (stored_root != expected_root || + ledger.state() != expected_ledger.state() || require_current_root( ledger, SQLiteLedgerError::state_mismatch) != - trusted_genesis_root) { + expected_root) { fail(SQLiteLedgerError::state_mismatch); } return ledger; diff --git a/src/storage/sqlite_schema_v1.hpp b/src/storage/sqlite_schema_v1.hpp index 776c3c8..5021980 100644 --- a/src/storage/sqlite_schema_v1.hpp +++ b/src/storage/sqlite_schema_v1.hpp @@ -17,11 +17,18 @@ void install_schema_v1( void validate_integrity(Connection& connection); void validate_schema_v1(Connection& connection); - -protocol::v1::Ledger load_height_zero( +void validate_stored_genesis( Connection& connection, std::span expected_genesis, const protocol::v1::Ledger& trusted_genesis_ledger, const protocol::v1::StateRoot& trusted_genesis_root); +protocol::v1::Ledger load_materialized_ledger( + Connection& connection, + std::span expected_genesis, + const protocol::v1::Ledger& trusted_genesis_ledger, + const protocol::v1::StateRoot& trusted_genesis_root, + const protocol::v1::Ledger& expected_ledger, + const protocol::v1::StateRoot& expected_root); + } // namespace protocol::storage::internal diff --git a/tests/storage/sqlite_history_test.cpp b/tests/storage/sqlite_history_test.cpp new file mode 100644 index 0000000..5341409 --- /dev/null +++ b/tests/storage/sqlite_history_test.cpp @@ -0,0 +1,301 @@ +#include "protocol/storage/sqlite_ledger.hpp" +#include "protocol/v1/ledger.hpp" + +#include "../../tools/protocol-vectors/vector_common.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pv = protocol_vectors; +namespace ps = protocol::storage; +namespace p = protocol::v1; + +namespace { + +class DatabaseFiles { + public: + explicit DatabaseFiles(std::filesystem::path path) + : path_(std::move(path)) { + remove(); + } + + ~DatabaseFiles() { remove(); } + + const std::filesystem::path& path() const noexcept { return path_; } + + private: + void remove() noexcept { + std::error_code ignored; + std::filesystem::remove(path_, ignored); + std::filesystem::remove(path_.string() + "-journal", ignored); + std::filesystem::remove(path_.string() + "-wal", ignored); + std::filesystem::remove(path_.string() + "-shm", ignored); + } + + std::filesystem::path path_; +}; + +p::Bytes genesis_bytes(const pv::Values& values) { + return pv::hex_decode(values.at("genesis")); +} + +std::vector block_transactions(const pv::Values& values) { + const auto count = static_cast( + std::stoull(values.at("raw_count"))); + std::vector transactions; + transactions.reserve(count); + for (std::size_t index = 0; index < count; ++index) { + transactions.push_back( + pv::hex_decode(values.at("raw" + std::to_string(index)))); + } + return transactions; +} + +ps::SQLiteLedger take_ledger( + ps::SQLiteLedgerResult result, + std::string_view message) { + pv::require(std::holds_alternative(result.result), + message); + return std::get(std::move(result.result)); +} + +p::BlockCommit take_commit( + ps::SQLiteBlockResult result, + std::string_view message) { + pv::require(std::holds_alternative(result), message); + return std::get(std::move(result)); +} + +void require_open_error( + ps::SQLiteLedgerResult result, + ps::SQLiteLedgerError expected, + std::string_view message) { + pv::require( + std::holds_alternative(result.result) && + std::get(result.result) == expected, + message); +} + +void require_block_error( + ps::SQLiteBlockResult result, + p::BlockError expected, + std::string_view message) { + pv::require( + std::holds_alternative(result) && + std::get(result) == expected, + message); +} + +bool same_commit( + const p::BlockCommit& left, + const p::BlockCommit& right) { + return left.height == right.height && + left.admissions == right.admissions && + left.transaction_ids == right.transaction_ids && + left.receipts == right.receipts && + left.encoded_receipts == right.encoded_receipts && + left.previous_state_root == right.previous_state_root && + left.transaction_root == right.transaction_root && + left.resulting_state_root == right.resulting_state_root && + left.header == right.header && + left.block_id == right.block_id; +} + +void raw_execute( + const std::filesystem::path& path, + const std::string& sql) { + sqlite3* database = nullptr; + const auto opened = sqlite3_open_v2( + path.c_str(), &database, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_PRIVATECACHE | + SQLITE_OPEN_NOFOLLOW | SQLITE_OPEN_EXRESCODE, + nullptr); + if (opened != SQLITE_OK) { + if (database != nullptr) sqlite3_close(database); + throw std::runtime_error("raw SQLite open failed"); + } + const auto executed = + sqlite3_exec(database, sql.c_str(), nullptr, nullptr, nullptr); + const auto closed = sqlite3_close(database); + pv::require(executed == SQLITE_OK, "raw SQLite execution failed"); + pv::require(closed == SQLITE_OK, "raw SQLite close failed"); +} + +std::string raw_scalar_text( + const std::filesystem::path& path, + const char* sql) { + sqlite3* database = nullptr; + pv::require( + sqlite3_open_v2( + path.c_str(), &database, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_PRIVATECACHE | + SQLITE_OPEN_NOFOLLOW | SQLITE_OPEN_EXRESCODE, + nullptr) == SQLITE_OK, + "raw scalar open failed"); + sqlite3_stmt* statement = nullptr; + pv::require( + sqlite3_prepare_v2(database, sql, -1, &statement, nullptr) == + SQLITE_OK, + "raw scalar prepare failed"); + pv::require(sqlite3_step(statement) == SQLITE_ROW, + "raw scalar row missing"); + const auto* raw = sqlite3_column_text(statement, 0); + pv::require(raw != nullptr, "raw scalar text missing"); + const std::string value(reinterpret_cast(raw)); + pv::require(sqlite3_step(statement) == SQLITE_DONE, + "raw scalar extra row"); + pv::require(sqlite3_finalize(statement) == SQLITE_OK, + "raw scalar finalize failed"); + pv::require(sqlite3_close(database) == SQLITE_OK, + "raw scalar close failed"); + return value; +} + +void create_durable_block( + const std::filesystem::path& path, + const p::Bytes& genesis, + const std::vector& transactions) { + auto ledger = take_ledger( + ps::create_sqlite_ledger(path, genesis), + "corruption baseline create failed"); + (void)take_commit( + ledger.apply_block(1, transactions), + "corruption baseline block failed"); +} + +void verify_durable_block( + const pv::Values& values, + const std::filesystem::path& prefix) { + DatabaseFiles files(prefix.string() + "-block.db"); + const auto genesis = genesis_bytes(values); + const auto transactions = block_transactions(values); + + auto loaded = p::load_genesis(genesis); + pv::require(std::holds_alternative(loaded.result), + "fixture genesis rejected"); + auto expected_ledger = + std::get(std::move(loaded.result)); + auto expected_result = + expected_ledger.apply_block(1, transactions); + pv::require( + std::holds_alternative(expected_result), + "fixture block rejected"); + const auto expected_commit = + std::get(std::move(expected_result)); + const ps::LedgerHead expected{ + expected_ledger.state(), expected_commit.resulting_state_root}; + + { + auto stored = take_ledger( + ps::create_sqlite_ledger(files.path(), genesis), + "durable database create failed"); + const auto commit = take_commit( + stored.apply_block(1, transactions), "durable block rejected"); + pv::require(same_commit(commit, expected_commit), + "durable output changed"); + pv::require(stored.read_head() == expected, + "published head mismatch"); + require_block_error( + stored.apply_block(1, transactions), + p::BlockError::invalid_height, + "rejected block returned wrong error"); + pv::require(stored.read_head() == expected, + "rejected block changed head"); + } + + pv::require( + raw_scalar_text(files.path(), "SELECT count(*) FROM blocks") == "1", + "durable block row missing"); + pv::require( + raw_scalar_text( + files.path(), + "SELECT count(*) FROM admitted_transactions") == + values.at("admitted_count"), + "journal retained wrong input count"); + + auto reopened = take_ledger( + ps::open_sqlite_ledger(files.path(), genesis), + "full genesis replay rejected"); + pv::require(reopened.read_head() == expected, + "replayed head mismatch"); +} + +void require_corruption_rejected( + const std::filesystem::path& path, + const p::Bytes& genesis, + const std::vector& transactions, + const std::string& sql, + std::string_view message) { + DatabaseFiles files(path); + create_durable_block(files.path(), genesis, transactions); + raw_execute(files.path(), sql); + require_open_error( + ps::open_sqlite_ledger(files.path(), genesis), + ps::SQLiteLedgerError::state_mismatch, message); +} + +void verify_replay_rejection( + const pv::Values& values, + const std::filesystem::path& prefix) { + const auto genesis = genesis_bytes(values); + const auto transactions = block_transactions(values); + + DatabaseFiles wrong(prefix.string() + "-wrong-genesis.db"); + create_durable_block(wrong.path(), genesis, transactions); + auto alternate = genesis; + alternate[33] ^= 1U; + require_open_error( + ps::open_sqlite_ledger(wrong.path(), alternate), + ps::SQLiteLedgerError::genesis_mismatch, + "wrong genesis lost precedence before replay"); + + require_corruption_rejected( + prefix.string() + "-transaction-id.db", genesis, transactions, + "UPDATE admitted_transactions SET transaction_id=zeroblob(32) " + "WHERE ordinal=X'00000000';", + "transaction-ID corruption accepted"); + require_corruption_rejected( + prefix.string() + "-ordinal-gap.db", genesis, transactions, + "DELETE FROM admitted_transactions WHERE ordinal=X'00000005';", + "missing journal ordinal accepted"); + require_corruption_rejected( + prefix.string() + "-block-id.db", genesis, transactions, + "UPDATE blocks SET block_id=zeroblob(32);", + "block-ID corruption accepted"); + require_corruption_rejected( + prefix.string() + "-materialized.db", genesis, transactions, + "UPDATE accounts SET balance=X'0000000000000000' " + "WHERE account_id=(SELECT account_id FROM accounts " + "ORDER BY account_id LIMIT 1);", + "materialized state diverging from replay accepted"); +} + +} // namespace + +int main(int argc, char** argv) { + try { + pv::require( + argc == 3, + "usage: storage_sqlite_history_tests VECTOR_FILE PATH_PREFIX"); + pv::require(sodium_init() >= 0, "libsodium initialization"); + const auto values = pv::load_values(argv[1]); + const std::filesystem::path prefix(argv[2]); + verify_durable_block(values, prefix); + verify_replay_rejection(values, prefix); + std::cout << "SQLite history tests: passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << "SQLite history tests: failed: " + << error.what() << '\n'; + return 1; + } +} diff --git a/tests/storage/sqlite_ledger_test.cpp b/tests/storage/sqlite_ledger_test.cpp index d1227f8..feea068 100644 --- a/tests/storage/sqlite_ledger_test.cpp +++ b/tests/storage/sqlite_ledger_test.cpp @@ -424,7 +424,7 @@ void verify_state_and_integrity_rejection( require_error( ps::open_sqlite_ledger(history.path(), genesis), ps::SQLiteLedgerError::state_mismatch, - "nonzero height history accepted by height-zero adapter"); + "incomplete nonzero-height history accepted"); DatabaseFiles foreign(prefix.string() + "-foreign.db"); create_and_close(foreign.path(), genesis);