diff --git a/CMakeLists.txt b/CMakeLists.txt index ff38803..431cc08 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -164,6 +164,8 @@ add_library( protocol_storage STATIC src/storage/sqlite_connection.cpp + src/storage/sqlite_block_store_v1.cpp + src/storage/sqlite_history_replay_v1.cpp src/storage/sqlite_ledger.cpp src/storage/sqlite_schema_v1.cpp ) @@ -233,12 +235,22 @@ add_executable( storage_sqlite_ledger_tests tests/storage/sqlite_ledger_test.cpp ) +add_executable( + storage_sqlite_history_tests + tests/storage/sqlite_history_test.cpp +) target_link_libraries( storage_sqlite_ledger_tests PRIVATE protocol_storage protocol_stack_sqlite ) +target_link_libraries( + storage_sqlite_history_tests + PRIVATE + protocol_storage + protocol_stack_sqlite +) set( PROTOCOL_STACK_TARGETS @@ -257,6 +269,7 @@ set( kernel_differential_runner sqlite_dependency_tests storage_sqlite_ledger_tests + storage_sqlite_history_tests ) if(PROTOCOL_STACK_ENABLE_FUZZING) add_library( @@ -443,6 +456,13 @@ add_test( "${PROJECT_SOURCE_DIR}/test-vectors/ledger-transition-v1.txt" "${CMAKE_CURRENT_BINARY_DIR}/storage-sqlite-ledger-test" ) +add_test( + NAME storage-sqlite-history + COMMAND + storage_sqlite_history_tests + "${PROJECT_SOURCE_DIR}/test-vectors/ledger-transition-v1.txt" + "${CMAKE_CURRENT_BINARY_DIR}/storage-sqlite-history-test" +) set_tests_properties( protocol-primitives-python ledger-transition-python diff --git a/docs/architecture/sqlite-ledger.md b/docs/architecture/sqlite-ledger.md index d237ea1..5345889 100644 --- a/docs/architecture/sqlite-ledger.md +++ b/docs/architecture/sqlite-ledger.md @@ -9,17 +9,29 @@ remain authoritative. ## Implemented outcome -`create_sqlite_ledger` now creates a brand-new database at height zero and -commits the caller-configured canonical genesis state. `open_sqlite_ledger` -reopens only that closed-world height-zero form. It rejects any database with -block, admitted-transaction, or snapshot rows until replay and recovery are -implemented. +`create_sqlite_ledger` creates a brand-new database at height zero and commits +the caller-configured canonical genesis state. `SQLiteLedger::apply_block` +copy-constructs an independent kernel candidate, applies the complete ordered +raw-input block, persists the resulting materialized state and canonical +history in one transaction, commits, and publishes the candidate through a +non-throwing pointer swap. A kernel block rejection never starts a storage +transaction. + +`open_sqlite_ledger` performs full genesis replay of every retained block +before publishing a live ledger. It admits only the stored 200-byte journal +rows, in explicit height and ordinal order, and compares every replayed +transaction ID, receipt, root, application header, and block ID with the +stored canonical output. The replay head must then exactly equal metadata, +materialized accounts, the fee pool, and the public ledger root. Snapshot rows +remain refused until independent snapshot recovery is implemented. The public header exposes no SQLite handle or SQL type. `SQLiteLedger` is move-constructible but not copyable or assignable. It owns the live `protocol::v1::Ledger`, serialized connection, normalized path, exact canonical genesis bytes, and cached state root. `read_head` returns owned state and root values while holding the adapter mutex; callers receive no borrowed view. +`apply_block` returns one of the exact kernel `BlockCommit`, the deterministic +kernel `BlockError`, or an operational `SQLiteLedgerError`. ## Trusted creation input @@ -76,10 +88,17 @@ rowid, and foreign-key metadata; unknown or modified objects are refused. Opening requires a single successful `integrity_check` result and an empty `foreign_key_check`. It exact-compares the persisted canonical genesis with -the independently trusted caller value, bounds account loading by the trusted -genesis account count, reconstructs an owned state, and calls +the independently trusted caller value, replays contiguous block and admitted +transaction rows from genesis, bounds materialized account loading by the +verified replay account count, reconstructs an owned state, and calls `restore_ledger` with caller-derived immutable parameters and the stored root. -The result must exactly equal the independently loaded genesis state and root. +The result must exactly equal the independently replayed state and root. + +Admission failures are absent from the journal. Empty and entirely unadmitted +blocks still have a block row and advance height. Duplicate admitted +transactions retain separate contiguous ordinals. Database projections never +replace canonical transaction, receipt, header, or identifier bytes as the +replay authority. ## Error and lifetime behavior @@ -96,8 +115,7 @@ before the completed adapter is returned. ## Remaining issue 11 work -The current boundary deliberately cannot apply or replay a block. The next -vertical result is one atomic durable block commit followed by validated clean -reopen and full genesis replay. Snapshot recovery, portable export/import, -fault injection around commit phases, long restart sequences, and final issue -closure follow that working block path. +The ordinary durable commit and full-genesis-replay path is implemented. +Snapshot recovery, portable export/import, automatic reopen after an ambiguous +commit result, fault injection around every commit phase, long seeded restart +sequences, and final issue closure remain. diff --git a/docs/project/current-state.md b/docs/project/current-state.md index 43c2079..3fd1860 100644 --- a/docs/project/current-state.md +++ b/docs/project/current-state.md @@ -6,8 +6,9 @@ Last updated: 2026-07-24 M1 — Sovereign Devnet Alpha. The deterministic in-memory ledger kernel is merged and verified. The owning storage adapter can now durably create and -validate a height-zero ledger. Atomic block persistence, replay, snapshots, -and recovery remain the active roadmap slice. +validate a height-zero ledger, atomically persist a complete block, and reopen +the identical head through full genesis replay. Multi-block restart coverage, +snapshots, fault injection, and recovery remain the active roadmap slice. ## Verified facts @@ -171,9 +172,17 @@ and recovery remain the active roadmap slice. Reopening never creates or changes journal mode and publishes no ledger unless integrity, foreign keys, exact schema, caller-trusted genesis, materialized state, and root all agree. -- The current adapter is deliberately closed at height zero. Any block, - admitted-transaction, or snapshot row is rejected until complete replay and - recovery validation are implemented. +- `SQLiteLedger::apply_block` applies ordered raw inputs to an independent + ledger candidate, writes changed and created accounts, exact admitted + transaction bytes and kernel outputs, the block row, and head metadata in + one SQLite transaction, durably commits, and publishes through a + non-throwing owning-pointer swap. Kernel block rejection never opens a + storage transaction. +- Opening validates caller-trusted genesis before history, then full-replays + contiguous block and admitted-transaction rows in explicit height and + ordinal order. Every replayed transaction ID, receipt, root, application + header, and block ID must equal storage before the replay head is compared + exactly with materialized state and metadata. Snapshot rows remain refused. - The owner prefers one active delivery branch, cleanup of obsolete branches/worktrees/build trees at phase boundaries, focused checks while iterating, one required completion matrix, and runnable vertical outcomes @@ -203,16 +212,27 @@ and recovery remain the active roadmap slice. wrong genesis, materialized-state and root corruption, immutable-fee projection corruption, foreign-key damage, truncated files, and refusal of unvalidated history. +- The frozen 15-input ledger block now passes through durable storage with the + exact kernel `BlockCommit`, 11 admitted journal rows, three omitted + admission failures, unchanged head on a rejected repeated height, clean + close, full genesis replay, and an identical owned head. +- Replay rejection coverage proves wrong-genesis precedence at nonzero height, + missing admitted ordinals, altered transaction and block identifiers, and + materialized state divergence are refused before publication. +- Clean completion verification passes 16/16 CTest tests in GCC debug, GCC + ASan+UBSan, and Clang debug, and 19/19 in Clang ASan+UBSan including the + three existing kernel fuzz smoke tests. Leak detection is disabled only for + sanitizer completion runs because the managed execution sandbox traces + processes and LeakSanitizer refuses to run under `ptrace`. ## Exact next action Continue issue #11: -> Implement one end-to-end durable block: apply ordered raw inputs to an -> independent ledger candidate, atomically persist materialized state, exact -> admitted bytes and kernel outputs, commit, publish with a non-throwing -> ownership transfer, then cleanly reopen by full genesis replay to the -> identical head. +> Add a deterministic multi-block and repeated-restart storage harness covering +> empty and entirely unadmitted blocks, duplicate admitted transactions, +> continued commits from reopened heads, exact replay after each restart, and +> materialized-state agreement throughout. ## Open autonomous decisions diff --git a/include/protocol/storage/sqlite_ledger.hpp b/include/protocol/storage/sqlite_ledger.hpp index 357d047..e55706c 100644 --- a/include/protocol/storage/sqlite_ledger.hpp +++ b/include/protocol/storage/sqlite_ledger.hpp @@ -1,6 +1,6 @@ #pragma once -#include "protocol/v1/types.hpp" +#include "protocol/v1/ledger.hpp" #include #include @@ -33,6 +33,11 @@ struct LedgerHead { struct SQLiteLedgerResult; +using SQLiteBlockResult = std::variant< + protocol::v1::BlockCommit, + protocol::v1::BlockError, + SQLiteLedgerError>; + class SQLiteLedger { public: ~SQLiteLedger() noexcept; @@ -43,6 +48,9 @@ class SQLiteLedger { SQLiteLedger& operator=(SQLiteLedger&&) = delete; LedgerHead read_head() const; + SQLiteBlockResult apply_block( + std::uint64_t height, + std::span raw_transactions); private: struct Impl; diff --git a/src/storage/sqlite_block_store_v1.cpp b/src/storage/sqlite_block_store_v1.cpp new file mode 100644 index 0000000..60dec292 --- /dev/null +++ b/src/storage/sqlite_block_store_v1.cpp @@ -0,0 +1,204 @@ +#include "sqlite_history_v1.hpp" + +#include + +#include +#include +#include +#include +#include + +namespace protocol::storage::internal { +namespace { + +namespace pv1 = protocol::v1; +constexpr std::size_t kMaximumBlockInputs = 65'535; + +[[noreturn]] void fail() { + throw Failure{SQLiteLedgerError::storage_failure}; +} + +void require_done(Statement& statement) { + if (statement.step() != SQLITE_DONE) fail(); +} + +void require_single_change(Connection& connection) { + if (sqlite3_changes(connection.get()) != 1) fail(); +} + +std::array encode_u64(std::uint64_t value) { + std::array encoded{}; + for (std::size_t index = 0; index < encoded.size(); ++index) { + const auto shift = static_cast( + (encoded.size() - index - 1) * 8); + encoded[index] = static_cast(value >> shift); + } + return encoded; +} + +std::array encode_u32(std::uint32_t value) { + std::array encoded{}; + for (std::size_t index = 0; index < encoded.size(); ++index) { + const auto shift = static_cast( + (encoded.size() - index - 1) * 8); + encoded[index] = static_cast(value >> shift); + } + return encoded; +} + +template +std::span tagged_bytes( + const Tagged& value) noexcept { + return {value.data(), value.size()}; +} + +std::size_t require_commit_shape( + const pv1::State& previous_state, + const pv1::State& resulting_state, + std::span raw_transactions, + const pv1::BlockCommit& commit) { + if (previous_state.height == std::numeric_limits::max() || + resulting_state.height != previous_state.height + 1 || + commit.height != resulting_state.height || + previous_state.parameters != resulting_state.parameters || + commit.admissions.size() != raw_transactions.size() || + commit.transaction_ids.size() != commit.receipts.size() || + commit.transaction_ids.size() != commit.encoded_receipts.size() || + commit.transaction_ids.size() > kMaximumBlockInputs || + commit.header.size() != 146) { + fail(); + } + + std::size_t admitted_count = 0; + for (const auto& admission : commit.admissions) { + if (!admission) ++admitted_count; + } + if (admitted_count != commit.transaction_ids.size()) fail(); + return admitted_count; +} + +void persist_accounts( + Connection& connection, + const pv1::State& previous_state, + const pv1::State& resulting_state) { + for (const auto& [account_id, account] : previous_state.accounts) { + const auto found = resulting_state.accounts.find(account_id); + if (found == resulting_state.accounts.end()) fail(); + (void)account; + } + + Statement upsert = connection.prepare( + "INSERT INTO accounts(account_id, balance, nonce) VALUES(?, ?, ?) " + "ON CONFLICT(account_id) DO UPDATE SET " + "balance=excluded.balance, nonce=excluded.nonce"); + for (const auto& [account_id, account] : resulting_state.accounts) { + const auto previous = previous_state.accounts.find(account_id); + if (previous != previous_state.accounts.end() && + previous->second == account) { + continue; + } + const auto balance = encode_u64(account.balance); + const auto nonce = encode_u64(account.nonce); + upsert.bind_blob(1, tagged_bytes(account_id)); + upsert.bind_blob(2, balance); + upsert.bind_blob(3, nonce); + require_done(upsert); + require_single_change(connection); + upsert.reset(); + } +} + +void persist_block_row( + Connection& connection, + const pv1::BlockCommit& commit, + std::size_t admitted_count) { + Statement insert = connection.prepare( + "INSERT INTO blocks(" + "height, previous_state_root, transaction_root, " + "resulting_state_root, admitted_count, header, block_id" + ") VALUES(?, ?, ?, ?, ?, ?, ?)"); + const auto height = encode_u64(commit.height); + const auto count = + encode_u32(static_cast(admitted_count)); + insert.bind_blob(1, height); + insert.bind_blob(2, tagged_bytes(commit.previous_state_root)); + insert.bind_blob(3, tagged_bytes(commit.transaction_root)); + insert.bind_blob(4, tagged_bytes(commit.resulting_state_root)); + insert.bind_blob(5, count); + insert.bind_blob(6, commit.header); + insert.bind_blob(7, tagged_bytes(commit.block_id)); + require_done(insert); + require_single_change(connection); +} + +void persist_admitted_transactions( + Connection& connection, + std::span raw_transactions, + const pv1::BlockCommit& commit) { + Statement insert = connection.prepare( + "INSERT INTO admitted_transactions(" + "height, ordinal, transaction_bytes, transaction_id, receipt_bytes" + ") VALUES(?, ?, ?, ?, ?)"); + const auto height = encode_u64(commit.height); + std::uint32_t ordinal = 0; + for (std::size_t raw_index = 0; + raw_index < raw_transactions.size(); ++raw_index) { + if (commit.admissions[raw_index]) continue; + const auto& raw = raw_transactions[raw_index]; + const auto& receipt = commit.encoded_receipts[ordinal]; + if (raw.size() != 200 || receipt.size() != 47) fail(); + const auto encoded_ordinal = encode_u32(ordinal); + insert.bind_blob(1, height); + insert.bind_blob(2, encoded_ordinal); + insert.bind_blob(3, raw); + insert.bind_blob( + 4, tagged_bytes(commit.transaction_ids[ordinal])); + insert.bind_blob(5, receipt); + require_done(insert); + require_single_change(connection); + insert.reset(); + ++ordinal; + } +} + +void update_metadata( + Connection& connection, + const pv1::State& previous_state, + const pv1::State& resulting_state, + const pv1::BlockCommit& commit) { + Statement update = connection.prepare( + "UPDATE ledger_meta SET " + "current_height=?, fee_pool=?, current_state_root=? " + "WHERE singleton=1 AND current_height=? " + "AND current_state_root=?"); + const auto next_height = encode_u64(resulting_state.height); + const auto next_fee_pool = encode_u64(resulting_state.fee_pool); + const auto previous_height = encode_u64(previous_state.height); + update.bind_blob(1, next_height); + update.bind_blob(2, next_fee_pool); + update.bind_blob(3, tagged_bytes(commit.resulting_state_root)); + update.bind_blob(4, previous_height); + update.bind_blob(5, tagged_bytes(commit.previous_state_root)); + require_done(update); + require_single_change(connection); +} + +} // namespace + +void persist_block_v1( + Connection& connection, + const pv1::State& previous_state, + const pv1::State& resulting_state, + std::span raw_transactions, + const pv1::BlockCommit& commit) { + if (connection.autocommit()) fail(); + const auto admitted_count = require_commit_shape( + previous_state, resulting_state, raw_transactions, commit); + persist_accounts(connection, previous_state, resulting_state); + persist_block_row(connection, commit, admitted_count); + persist_admitted_transactions(connection, raw_transactions, commit); + update_metadata( + connection, previous_state, resulting_state, commit); +} + +} // namespace protocol::storage::internal diff --git a/src/storage/sqlite_history_replay_v1.cpp b/src/storage/sqlite_history_replay_v1.cpp new file mode 100644 index 0000000..967c660 --- /dev/null +++ b/src/storage/sqlite_history_replay_v1.cpp @@ -0,0 +1,217 @@ +#include "sqlite_history_v1.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace protocol::storage::internal { +namespace { + +namespace pv1 = protocol::v1; + +constexpr std::uint32_t kMaximumBlockInputs = 65'535; + +struct StoredJournal { + std::vector transactions; + std::vector transaction_ids; + std::vector receipts; +}; + +[[noreturn]] void fail(SQLiteLedgerError error) { + throw Failure{error}; +} + +int checked_step(Statement& statement) { + const auto result = statement.step(); + if (result != SQLITE_ROW && result != SQLITE_DONE) { + fail(SQLiteLedgerError::storage_failure); + } + return result; +} + +std::array encode_u64(std::uint64_t value) { + std::array encoded{}; + for (std::size_t index = 0; index < encoded.size(); ++index) { + const auto shift = static_cast( + (encoded.size() - index - 1) * 8); + encoded[index] = static_cast(value >> shift); + } + return encoded; +} + +std::uint64_t decode_unsigned( + const Statement& statement, + int index, + std::size_t width) { + if (statement.column_type(index) != SQLITE_BLOB) { + fail(SQLiteLedgerError::state_mismatch); + } + const auto bytes = statement.column_blob(index); + if (bytes.size() != width) { + fail(SQLiteLedgerError::state_mismatch); + } + std::uint64_t value = 0; + for (const auto byte : bytes) value = (value << 8U) | byte; + return value; +} + +pv1::Bytes decode_bytes( + const Statement& statement, + int index, + std::size_t width) { + if (statement.column_type(index) != SQLITE_BLOB) { + fail(SQLiteLedgerError::state_mismatch); + } + const auto bytes = statement.column_blob(index); + if (bytes.size() != width) { + fail(SQLiteLedgerError::state_mismatch); + } + return pv1::Bytes(bytes.begin(), bytes.end()); +} + +template +Tagged decode_tagged_hash( + const Statement& statement, + int index) { + const auto bytes = decode_bytes(statement, index, pv1::Hash{}.size()); + pv1::Hash value{}; + std::copy(bytes.begin(), bytes.end(), value.begin()); + return Tagged{value}; +} + +StoredJournal load_journal( + Connection& connection, + std::uint64_t height, + std::uint32_t admitted_count) { + Statement rows = connection.prepare( + "SELECT ordinal, transaction_bytes, transaction_id, receipt_bytes " + "FROM admitted_transactions WHERE height=? ORDER BY ordinal"); + const auto encoded_height = encode_u64(height); + rows.bind_blob(1, encoded_height); + + StoredJournal stored; + stored.transactions.reserve(admitted_count); + stored.transaction_ids.reserve(admitted_count); + stored.receipts.reserve(admitted_count); + while (checked_step(rows) == SQLITE_ROW) { + if (rows.column_count() != 4 || + stored.transactions.size() >= admitted_count) { + fail(SQLiteLedgerError::state_mismatch); + } + const auto ordinal = decode_unsigned(rows, 0, 4); + if (ordinal != stored.transactions.size()) { + fail(SQLiteLedgerError::state_mismatch); + } + stored.transactions.push_back(decode_bytes(rows, 1, 200)); + stored.transaction_ids.push_back( + decode_tagged_hash(rows, 2)); + stored.receipts.push_back(decode_bytes(rows, 3, 47)); + } + if (stored.transactions.size() != admitted_count) { + fail(SQLiteLedgerError::state_mismatch); + } + return stored; +} + +bool all_admitted(const pv1::BlockCommit& commit) { + return std::all_of( + commit.admissions.begin(), commit.admissions.end(), + [](const auto& admission) { return !admission.has_value(); }); +} + +void require_replay_match( + const pv1::BlockCommit& replayed, + const StoredJournal& stored, + const pv1::StateRoot& previous_root, + const pv1::TransactionRoot& transaction_root, + const pv1::StateRoot& resulting_root, + const pv1::Bytes& header, + const pv1::BlockId& block_id) { + if (replayed.admissions.size() != stored.transactions.size() || + !all_admitted(replayed) || + replayed.transaction_ids != stored.transaction_ids || + replayed.encoded_receipts != stored.receipts || + replayed.receipts.size() != stored.receipts.size() || + replayed.previous_state_root != previous_root || + replayed.transaction_root != transaction_root || + replayed.resulting_state_root != resulting_root || + replayed.header != header || + replayed.block_id != block_id) { + fail(SQLiteLedgerError::state_mismatch); + } +} + +void replay_block_row( + Connection& connection, + Statement& blocks, + pv1::Ledger& ledger) { + if (blocks.column_count() != 7 || + ledger.state().height == std::numeric_limits::max()) { + fail(SQLiteLedgerError::state_mismatch); + } + const auto height = decode_unsigned(blocks, 0, 8); + if (height != ledger.state().height + 1) { + fail(SQLiteLedgerError::state_mismatch); + } + const auto previous_root = + decode_tagged_hash(blocks, 1); + const auto transaction_root = + decode_tagged_hash(blocks, 2); + const auto resulting_root = + decode_tagged_hash(blocks, 3); + const auto admitted_count_value = decode_unsigned(blocks, 4, 4); + if (admitted_count_value > kMaximumBlockInputs) { + fail(SQLiteLedgerError::state_mismatch); + } + const auto admitted_count = + static_cast(admitted_count_value); + const auto header = decode_bytes(blocks, 5, 146); + const auto block_id = + decode_tagged_hash(blocks, 6); + const auto stored = + load_journal(connection, height, admitted_count); + + auto applied = ledger.apply_block(height, stored.transactions); + if (!std::holds_alternative(applied)) { + fail(SQLiteLedgerError::state_mismatch); + } + const auto& replayed = std::get(applied); + require_replay_match( + replayed, stored, previous_root, transaction_root, + resulting_root, header, block_id); +} + +void require_no_snapshots(Connection& connection) { + Statement snapshots = + connection.prepare("SELECT 1 FROM snapshots LIMIT 1"); + if (checked_step(snapshots) != SQLITE_DONE) { + fail(SQLiteLedgerError::state_mismatch); + } +} + +} // namespace + +pv1::Ledger replay_history_v1( + Connection& connection, + const pv1::Ledger& trusted_genesis_ledger) { + pv1::Ledger ledger(trusted_genesis_ledger); + Statement blocks = connection.prepare( + "SELECT height, previous_state_root, transaction_root, " + "resulting_state_root, admitted_count, header, block_id " + "FROM blocks ORDER BY height"); + while (checked_step(blocks) == SQLITE_ROW) { + replay_block_row(connection, blocks, ledger); + } + require_no_snapshots(connection); + return ledger; +} + +} // namespace protocol::storage::internal diff --git a/src/storage/sqlite_history_v1.hpp b/src/storage/sqlite_history_v1.hpp new file mode 100644 index 0000000..f7e2c52 --- /dev/null +++ b/src/storage/sqlite_history_v1.hpp @@ -0,0 +1,23 @@ +#pragma once + +#include "sqlite_connection.hpp" + +#include "protocol/v1/ledger.hpp" + +#include +#include + +namespace protocol::storage::internal { + +void persist_block_v1( + Connection& connection, + const protocol::v1::State& previous_state, + const protocol::v1::State& resulting_state, + std::span raw_transactions, + const protocol::v1::BlockCommit& commit); + +protocol::v1::Ledger replay_history_v1( + Connection& connection, + const protocol::v1::Ledger& trusted_genesis_ledger); + +} // namespace protocol::storage::internal diff --git a/src/storage/sqlite_ledger.cpp b/src/storage/sqlite_ledger.cpp index a7374e4..9debad4 100644 --- a/src/storage/sqlite_ledger.cpp +++ b/src/storage/sqlite_ledger.cpp @@ -1,6 +1,7 @@ #include "protocol/storage/sqlite_ledger.hpp" #include "sqlite_connection.hpp" +#include "sqlite_history_v1.hpp" #include "sqlite_schema_v1.hpp" #include "protocol/v1/ledger.hpp" @@ -47,15 +48,32 @@ TrustedGenesis load_trusted_genesis( }; } -Ledger validate_height_zero( +StateRoot require_root( + const Ledger& ledger, + SQLiteLedgerError error) { + auto root = ledger.current_state_root(); + if (!std::holds_alternative(root)) { + throw internal::Failure{error}; + } + return std::get(std::move(root)); +} + +Ledger validate_durable_ledger( internal::Connection& connection, std::span canonical_genesis, const Ledger& trusted_genesis, const StateRoot& trusted_root) { internal::validate_integrity(connection); internal::validate_schema_v1(connection); - return internal::load_height_zero( + internal::validate_stored_genesis( connection, canonical_genesis, trusted_genesis, trusted_root); + auto replayed = + internal::replay_history_v1(connection, trusted_genesis); + const auto replayed_root = require_root( + replayed, SQLiteLedgerError::state_mismatch); + return internal::load_materialized_ledger( + connection, canonical_genesis, trusted_genesis, trusted_root, + replayed, replayed_root); } SQLiteLedgerResult error_result(SQLiteLedgerError error) { @@ -74,6 +92,7 @@ struct SQLiteLedger::Impl { mutable std::mutex mutex; std::unique_ptr ledger; StateRoot state_root; + bool poisoned; Impl( std::filesystem::path normalized_path, @@ -85,11 +104,15 @@ struct SQLiteLedger::Impl { canonical_genesis(std::move(exact_genesis)), resources(std::move(sqlite_resources)), ledger(std::move(live_ledger)), - state_root(std::move(verified_root)) {} + state_root(std::move(verified_root)), + poisoned(false) {} }; static_assert(std::is_nothrow_move_constructible_v); static_assert(std::is_nothrow_destructible_v); +static_assert(std::is_nothrow_move_constructible_v); +static_assert(std::is_nothrow_swappable_v>); +static_assert(std::is_nothrow_copy_assignable_v); SQLiteLedger::SQLiteLedger( std::unique_ptr implementation) noexcept @@ -106,6 +129,60 @@ LedgerHead SQLiteLedger::read_head() const { }; } +SQLiteBlockResult SQLiteLedger::apply_block( + std::uint64_t height, + std::span raw_transactions) { + const std::lock_guard lock(implementation_->mutex); + if (implementation_->poisoned) { + return SQLiteBlockResult( + std::in_place_type, + SQLiteLedgerError::storage_failure); + } + + auto candidate = + std::make_unique(*implementation_->ledger); + auto applied = candidate->apply_block(height, raw_transactions); + if (std::holds_alternative(applied)) { + return SQLiteBlockResult( + std::in_place_type, + std::get(applied)); + } + SQLiteBlockResult result( + std::in_place_type, + std::get(std::move(applied))); + const auto& commit = std::get(result); + + try { + auto& connection = implementation_->resources.connection; + internal::verify_stable_path( + implementation_->resources, implementation_->path); + internal::begin_exclusive(connection); + try { + internal::persist_block_v1( + connection, implementation_->ledger->state(), + candidate->state(), raw_transactions, commit); + internal::verify_stable_path( + implementation_->resources, implementation_->path); + } catch (...) { + internal::rollback_or_terminate(connection); + throw; + } + try { + internal::commit(connection); + } catch (...) { + implementation_->poisoned = true; + throw; + } + + implementation_->ledger.swap(candidate); + implementation_->state_root = commit.resulting_state_root; + return result; + } catch (const internal::Failure& failure) { + return SQLiteBlockResult( + std::in_place_type, failure.error); + } +} + SQLiteLedgerResult create_sqlite_ledger( const std::filesystem::path& path, std::span canonical_genesis) { @@ -133,7 +210,7 @@ SQLiteLedgerResult create_sqlite_ledger( internal::install_schema_v1( connection, stored_genesis, *implementation->ledger, implementation->state_root); - auto validation = validate_height_zero( + auto validation = validate_durable_ledger( connection, stored_genesis, *implementation->ledger, implementation->state_root); (void)validation; @@ -147,7 +224,7 @@ SQLiteLedgerResult create_sqlite_ledger( internal::verify_stable_path( implementation->resources, implementation->path); - auto durable_validation = validate_height_zero( + auto durable_validation = validate_durable_ledger( connection, stored_genesis, *implementation->ledger, implementation->state_root); (void)durable_validation; @@ -174,15 +251,17 @@ SQLiteLedgerResult open_sqlite_ledger( internal::require_existing_journal_mode(resources.connection); internal::verify_stable_path(resources, normalized); - auto live = validate_height_zero( + auto live = validate_durable_ledger( resources.connection, bytes_view(trusted.canonical_bytes), trusted.ledger, trusted.state_root); + const auto live_root = + require_root(live, SQLiteLedgerError::state_mismatch); auto implementation = std::make_unique( std::move(normalized), std::move(trusted.canonical_bytes), std::move(resources), std::make_unique(std::move(live)), - trusted.state_root); + live_root); return SQLiteLedgerResult{ std::variant( diff --git a/src/storage/sqlite_schema_v1.cpp b/src/storage/sqlite_schema_v1.cpp index 9311f52..7b6310d 100644 --- a/src/storage/sqlite_schema_v1.cpp +++ b/src/storage/sqlite_schema_v1.cpp @@ -501,11 +501,6 @@ void validate_foreign_keys(Connection& connection, const TableSpec& table) { require_done(keys, SQLiteLedgerError::schema_mismatch); } -void require_empty_table(Connection& connection, const char* sql) { - Statement rows = connection.prepare(sql); - require_done(rows, SQLiteLedgerError::state_mismatch); -} - std::map load_accounts( Connection& connection, std::size_t expected_count) { @@ -592,13 +587,42 @@ void validate_schema_v1(Connection& connection) { } } -pv1::Ledger load_height_zero( +void validate_stored_genesis( Connection& connection, std::span expected_genesis, const pv1::Ledger& trusted_genesis_ledger, const pv1::StateRoot& trusted_genesis_root) { require_trusted_genesis( expected_genesis, trusted_genesis_ledger, trusted_genesis_root); + Statement metadata = connection.prepare( + "SELECT singleton, canonical_genesis " + "FROM ledger_meta ORDER BY singleton"); + require_row(metadata, SQLiteLedgerError::state_mismatch); + if (metadata.column_count() != 2 || + !integer_column(metadata, 0, 1) || + metadata.column_type(1) != SQLITE_BLOB) { + fail(SQLiteLedgerError::state_mismatch); + } + const auto stored_genesis = metadata.column_blob(1); + if (stored_genesis.size() != expected_genesis.size() || + !std::equal( + stored_genesis.begin(), stored_genesis.end(), + expected_genesis.begin())) { + fail(SQLiteLedgerError::genesis_mismatch); + } + require_done(metadata, SQLiteLedgerError::state_mismatch); +} + +pv1::Ledger load_materialized_ledger( + Connection& connection, + std::span expected_genesis, + const pv1::Ledger& trusted_genesis_ledger, + const pv1::StateRoot& trusted_genesis_root, + const pv1::Ledger& expected_ledger, + const pv1::StateRoot& expected_root) { + validate_stored_genesis( + connection, + expected_genesis, trusted_genesis_ledger, trusted_genesis_root); Statement metadata = connection.prepare( "SELECT singleton, canonical_genesis, chain_id, supply_limit, " @@ -634,14 +658,6 @@ pv1::Ledger load_height_zero( const auto stored_root = decode_tagged_hash( metadata, 8, SQLiteLedgerError::state_mismatch); require_done(metadata, SQLiteLedgerError::state_mismatch); - if (height != 0) fail(SQLiteLedgerError::state_mismatch); - - require_empty_table( - connection, "SELECT 1 FROM blocks LIMIT 1"); - require_empty_table( - connection, "SELECT 1 FROM admitted_transactions LIMIT 1"); - require_empty_table( - connection, "SELECT 1 FROM snapshots LIMIT 1"); pv1::State persisted_state{ pv1::Parameters{ @@ -653,7 +669,7 @@ pv1::Ledger load_height_zero( height, fee_pool, load_accounts( - connection, trusted_genesis_ledger.state().accounts.size()), + connection, expected_ledger.state().accounts.size()), }; auto restored = pv1::restore_ledger( std::move(persisted_state), @@ -663,11 +679,11 @@ pv1::Ledger load_height_zero( fail(SQLiteLedgerError::state_mismatch); } auto ledger = std::get(std::move(restored.result)); - if (stored_root != trusted_genesis_root || - ledger.state() != trusted_genesis_ledger.state() || + if (stored_root != expected_root || + ledger.state() != expected_ledger.state() || require_current_root( ledger, SQLiteLedgerError::state_mismatch) != - trusted_genesis_root) { + expected_root) { fail(SQLiteLedgerError::state_mismatch); } return ledger; diff --git a/src/storage/sqlite_schema_v1.hpp b/src/storage/sqlite_schema_v1.hpp index 776c3c8..5021980 100644 --- a/src/storage/sqlite_schema_v1.hpp +++ b/src/storage/sqlite_schema_v1.hpp @@ -17,11 +17,18 @@ void install_schema_v1( void validate_integrity(Connection& connection); void validate_schema_v1(Connection& connection); - -protocol::v1::Ledger load_height_zero( +void validate_stored_genesis( Connection& connection, std::span expected_genesis, const protocol::v1::Ledger& trusted_genesis_ledger, const protocol::v1::StateRoot& trusted_genesis_root); +protocol::v1::Ledger load_materialized_ledger( + Connection& connection, + std::span expected_genesis, + const protocol::v1::Ledger& trusted_genesis_ledger, + const protocol::v1::StateRoot& trusted_genesis_root, + const protocol::v1::Ledger& expected_ledger, + const protocol::v1::StateRoot& expected_root); + } // namespace protocol::storage::internal diff --git a/tests/storage/sqlite_history_test.cpp b/tests/storage/sqlite_history_test.cpp new file mode 100644 index 0000000..5341409 --- /dev/null +++ b/tests/storage/sqlite_history_test.cpp @@ -0,0 +1,301 @@ +#include "protocol/storage/sqlite_ledger.hpp" +#include "protocol/v1/ledger.hpp" + +#include "../../tools/protocol-vectors/vector_common.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pv = protocol_vectors; +namespace ps = protocol::storage; +namespace p = protocol::v1; + +namespace { + +class DatabaseFiles { + public: + explicit DatabaseFiles(std::filesystem::path path) + : path_(std::move(path)) { + remove(); + } + + ~DatabaseFiles() { remove(); } + + const std::filesystem::path& path() const noexcept { return path_; } + + private: + void remove() noexcept { + std::error_code ignored; + std::filesystem::remove(path_, ignored); + std::filesystem::remove(path_.string() + "-journal", ignored); + std::filesystem::remove(path_.string() + "-wal", ignored); + std::filesystem::remove(path_.string() + "-shm", ignored); + } + + std::filesystem::path path_; +}; + +p::Bytes genesis_bytes(const pv::Values& values) { + return pv::hex_decode(values.at("genesis")); +} + +std::vector block_transactions(const pv::Values& values) { + const auto count = static_cast( + std::stoull(values.at("raw_count"))); + std::vector transactions; + transactions.reserve(count); + for (std::size_t index = 0; index < count; ++index) { + transactions.push_back( + pv::hex_decode(values.at("raw" + std::to_string(index)))); + } + return transactions; +} + +ps::SQLiteLedger take_ledger( + ps::SQLiteLedgerResult result, + std::string_view message) { + pv::require(std::holds_alternative(result.result), + message); + return std::get(std::move(result.result)); +} + +p::BlockCommit take_commit( + ps::SQLiteBlockResult result, + std::string_view message) { + pv::require(std::holds_alternative(result), message); + return std::get(std::move(result)); +} + +void require_open_error( + ps::SQLiteLedgerResult result, + ps::SQLiteLedgerError expected, + std::string_view message) { + pv::require( + std::holds_alternative(result.result) && + std::get(result.result) == expected, + message); +} + +void require_block_error( + ps::SQLiteBlockResult result, + p::BlockError expected, + std::string_view message) { + pv::require( + std::holds_alternative(result) && + std::get(result) == expected, + message); +} + +bool same_commit( + const p::BlockCommit& left, + const p::BlockCommit& right) { + return left.height == right.height && + left.admissions == right.admissions && + left.transaction_ids == right.transaction_ids && + left.receipts == right.receipts && + left.encoded_receipts == right.encoded_receipts && + left.previous_state_root == right.previous_state_root && + left.transaction_root == right.transaction_root && + left.resulting_state_root == right.resulting_state_root && + left.header == right.header && + left.block_id == right.block_id; +} + +void raw_execute( + const std::filesystem::path& path, + const std::string& sql) { + sqlite3* database = nullptr; + const auto opened = sqlite3_open_v2( + path.c_str(), &database, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_PRIVATECACHE | + SQLITE_OPEN_NOFOLLOW | SQLITE_OPEN_EXRESCODE, + nullptr); + if (opened != SQLITE_OK) { + if (database != nullptr) sqlite3_close(database); + throw std::runtime_error("raw SQLite open failed"); + } + const auto executed = + sqlite3_exec(database, sql.c_str(), nullptr, nullptr, nullptr); + const auto closed = sqlite3_close(database); + pv::require(executed == SQLITE_OK, "raw SQLite execution failed"); + pv::require(closed == SQLITE_OK, "raw SQLite close failed"); +} + +std::string raw_scalar_text( + const std::filesystem::path& path, + const char* sql) { + sqlite3* database = nullptr; + pv::require( + sqlite3_open_v2( + path.c_str(), &database, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_PRIVATECACHE | + SQLITE_OPEN_NOFOLLOW | SQLITE_OPEN_EXRESCODE, + nullptr) == SQLITE_OK, + "raw scalar open failed"); + sqlite3_stmt* statement = nullptr; + pv::require( + sqlite3_prepare_v2(database, sql, -1, &statement, nullptr) == + SQLITE_OK, + "raw scalar prepare failed"); + pv::require(sqlite3_step(statement) == SQLITE_ROW, + "raw scalar row missing"); + const auto* raw = sqlite3_column_text(statement, 0); + pv::require(raw != nullptr, "raw scalar text missing"); + const std::string value(reinterpret_cast(raw)); + pv::require(sqlite3_step(statement) == SQLITE_DONE, + "raw scalar extra row"); + pv::require(sqlite3_finalize(statement) == SQLITE_OK, + "raw scalar finalize failed"); + pv::require(sqlite3_close(database) == SQLITE_OK, + "raw scalar close failed"); + return value; +} + +void create_durable_block( + const std::filesystem::path& path, + const p::Bytes& genesis, + const std::vector& transactions) { + auto ledger = take_ledger( + ps::create_sqlite_ledger(path, genesis), + "corruption baseline create failed"); + (void)take_commit( + ledger.apply_block(1, transactions), + "corruption baseline block failed"); +} + +void verify_durable_block( + const pv::Values& values, + const std::filesystem::path& prefix) { + DatabaseFiles files(prefix.string() + "-block.db"); + const auto genesis = genesis_bytes(values); + const auto transactions = block_transactions(values); + + auto loaded = p::load_genesis(genesis); + pv::require(std::holds_alternative(loaded.result), + "fixture genesis rejected"); + auto expected_ledger = + std::get(std::move(loaded.result)); + auto expected_result = + expected_ledger.apply_block(1, transactions); + pv::require( + std::holds_alternative(expected_result), + "fixture block rejected"); + const auto expected_commit = + std::get(std::move(expected_result)); + const ps::LedgerHead expected{ + expected_ledger.state(), expected_commit.resulting_state_root}; + + { + auto stored = take_ledger( + ps::create_sqlite_ledger(files.path(), genesis), + "durable database create failed"); + const auto commit = take_commit( + stored.apply_block(1, transactions), "durable block rejected"); + pv::require(same_commit(commit, expected_commit), + "durable output changed"); + pv::require(stored.read_head() == expected, + "published head mismatch"); + require_block_error( + stored.apply_block(1, transactions), + p::BlockError::invalid_height, + "rejected block returned wrong error"); + pv::require(stored.read_head() == expected, + "rejected block changed head"); + } + + pv::require( + raw_scalar_text(files.path(), "SELECT count(*) FROM blocks") == "1", + "durable block row missing"); + pv::require( + raw_scalar_text( + files.path(), + "SELECT count(*) FROM admitted_transactions") == + values.at("admitted_count"), + "journal retained wrong input count"); + + auto reopened = take_ledger( + ps::open_sqlite_ledger(files.path(), genesis), + "full genesis replay rejected"); + pv::require(reopened.read_head() == expected, + "replayed head mismatch"); +} + +void require_corruption_rejected( + const std::filesystem::path& path, + const p::Bytes& genesis, + const std::vector& transactions, + const std::string& sql, + std::string_view message) { + DatabaseFiles files(path); + create_durable_block(files.path(), genesis, transactions); + raw_execute(files.path(), sql); + require_open_error( + ps::open_sqlite_ledger(files.path(), genesis), + ps::SQLiteLedgerError::state_mismatch, message); +} + +void verify_replay_rejection( + const pv::Values& values, + const std::filesystem::path& prefix) { + const auto genesis = genesis_bytes(values); + const auto transactions = block_transactions(values); + + DatabaseFiles wrong(prefix.string() + "-wrong-genesis.db"); + create_durable_block(wrong.path(), genesis, transactions); + auto alternate = genesis; + alternate[33] ^= 1U; + require_open_error( + ps::open_sqlite_ledger(wrong.path(), alternate), + ps::SQLiteLedgerError::genesis_mismatch, + "wrong genesis lost precedence before replay"); + + require_corruption_rejected( + prefix.string() + "-transaction-id.db", genesis, transactions, + "UPDATE admitted_transactions SET transaction_id=zeroblob(32) " + "WHERE ordinal=X'00000000';", + "transaction-ID corruption accepted"); + require_corruption_rejected( + prefix.string() + "-ordinal-gap.db", genesis, transactions, + "DELETE FROM admitted_transactions WHERE ordinal=X'00000005';", + "missing journal ordinal accepted"); + require_corruption_rejected( + prefix.string() + "-block-id.db", genesis, transactions, + "UPDATE blocks SET block_id=zeroblob(32);", + "block-ID corruption accepted"); + require_corruption_rejected( + prefix.string() + "-materialized.db", genesis, transactions, + "UPDATE accounts SET balance=X'0000000000000000' " + "WHERE account_id=(SELECT account_id FROM accounts " + "ORDER BY account_id LIMIT 1);", + "materialized state diverging from replay accepted"); +} + +} // namespace + +int main(int argc, char** argv) { + try { + pv::require( + argc == 3, + "usage: storage_sqlite_history_tests VECTOR_FILE PATH_PREFIX"); + pv::require(sodium_init() >= 0, "libsodium initialization"); + const auto values = pv::load_values(argv[1]); + const std::filesystem::path prefix(argv[2]); + verify_durable_block(values, prefix); + verify_replay_rejection(values, prefix); + std::cout << "SQLite history tests: passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << "SQLite history tests: failed: " + << error.what() << '\n'; + return 1; + } +} diff --git a/tests/storage/sqlite_ledger_test.cpp b/tests/storage/sqlite_ledger_test.cpp index d1227f8..feea068 100644 --- a/tests/storage/sqlite_ledger_test.cpp +++ b/tests/storage/sqlite_ledger_test.cpp @@ -424,7 +424,7 @@ void verify_state_and_integrity_rejection( require_error( ps::open_sqlite_ledger(history.path(), genesis), ps::SQLiteLedgerError::state_mismatch, - "nonzero height history accepted by height-zero adapter"); + "incomplete nonzero-height history accepted"); DatabaseFiles foreign(prefix.string() + "-foreign.db"); create_and_close(foreign.path(), genesis);