diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index f83fe5446d..90eaa02759 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -1,12 +1,23 @@ file(GLOB BENCHMARK CONFIGURE_DEPENDS "*.cpp") add_executable( benchmark ${BENCHMARK} ) -target_link_libraries( benchmark sysio_testing fc Boost::program_options bn256::bn256) +# trace_api_plugin brings the process_block / process_block_to_json implementations +# used by the trace_api_json benchmark; its include path pulls in the data_log_entry +# variant type needed by the synthetic trace builder. +target_link_libraries( benchmark sysio_testing fc Boost::program_options bn256::bn256 trace_api_plugin) target_include_directories( benchmark PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/../unittests/include" ) +# ABI fixture paths consumed by abi_serializer.cpp. These point at the system-contract +# sources, not the built artifacts, so the benchmark works regardless of build state. +target_compile_definitions( benchmark + PRIVATE + TOKEN_ABI_PATH="${CMAKE_SOURCE_DIR}/contracts/sysio.token/sysio.token.abi" + SYSTEM_ABI_PATH="${CMAKE_SOURCE_DIR}/contracts/sysio.system/sysio.system.abi" +) + # Standalone dedup head-to-head benchmark. Links only sysio_chain + chainbase (no testing # harness / contracts), so it builds without the system-contract WASMs. Not built by default. add_executable( dedup_bench EXCLUDE_FROM_ALL dedup/dedup_bench.cpp ) diff --git a/benchmark/abi_serializer.cpp b/benchmark/abi_serializer.cpp new file mode 100644 index 0000000000..675ffd80f8 --- /dev/null +++ b/benchmark/abi_serializer.cpp @@ -0,0 +1,344 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +// Microbenchmark for sysio::chain::abi_serializer. +// +// Measures construction cost and steady-state decode throughput so that +// successive optimization commits can report a verifiable before/after +// delta. NOT a correctness test -- abi_tests.cpp covers that. +// +// Run via the unified benchmark harness: +// +// ninja -C cmake-build-release -j8 benchmark +// ./cmake-build-release/benchmark/benchmark abi_serializer --num-runs 10 +// +// Release build (`-DCMAKE_BUILD_TYPE=Release`, `-O3`) is required for +// comparable numbers; debug / RelWithDebInfo timings are dominated by +// instrumentation. Each scenario does its own inner loop sized to keep +// per-call overhead amortised; the harness's `--num-runs` controls how +// many times the outer scenario is repeated for variance reporting. + +namespace sysio::benchmark { + +using namespace sysio::chain; +using bytes = sysio::benchmark::bytes; + +namespace { + +constexpr fc::microseconds max_serialization_time = fc::microseconds{1'000'000}; + +abi_serializer::yield_function_t yield_fn() { + return abi_serializer::create_yield_function(max_serialization_time); +} + +std::string slurp(const std::filesystem::path& p) { + std::ifstream f(p); + if (!f) { + std::cerr << "abi_serializer benchmark: failed to open " << p << "\n"; + std::exit(1); + } + std::stringstream ss; + ss << f.rdbuf(); + return ss.str(); +} + +} // namespace + +void abi_serializer_benchmarking() { + const std::filesystem::path token_path = TOKEN_ABI_PATH; + const std::filesystem::path system_path = SYSTEM_ABI_PATH; + + const auto token_abi = fc::json::from_string(slurp(token_path)).as(); + const auto system_abi = fc::json::from_string(slurp(system_path)).as(); + + // Prebuild serializers + packed payloads for the steady-state decode + // scenarios. Cost of these setup steps is NOT in the measured region. + abi_serializer token_abis{abi_def(token_abi), yield_fn()}; + abi_serializer system_abis{abi_def(system_abi), yield_fn()}; + + const auto transfer_var = fc::json::from_string( + R"({"from":"alice","to":"bob","quantity":"1.0000 SYS","memo":"benchmark payment"})"); + chain::bytes transfer_bytes; + try { + transfer_bytes = token_abis.variant_to_binary("transfer", transfer_var, yield_fn()); + } catch (const std::exception& e) { + std::cerr << "abi_serializer benchmark: transfer pack failed: " << e.what() << "\n"; + return; + } + + // newaccount exercises a nested struct path (authority sub-struct + array + // of key_weights). An authority with one key is enough to cover the + // nested+array code paths without bloating the fixture. + const auto newaccount_var = fc::json::from_string( + R"({"creator":"sysio","name":"alice","owner":)" + R"({"threshold":1,"keys":[{"key":"SYS1111111111111111111111111111111114T1Anm","weight":1}],)" + R"("accounts":[],"waits":[]},"active":)" + R"({"threshold":1,"keys":[{"key":"SYS1111111111111111111111111111111114T1Anm","weight":1}],)" + R"("accounts":[],"waits":[]}})"); + chain::bytes newaccount_bytes; + try { + newaccount_bytes = system_abis.variant_to_binary("newaccount", newaccount_var, yield_fn()); + } catch (const std::exception& e) { + std::cerr << "abi_serializer benchmark: newaccount pack failed: " << e.what() << "\n"; + return; + } + + // regproducer -- flat-struct baseline comparable to transfer but in a + // larger ABI context. Measures per-field dispatch amortised over the + // whole-action cost in the big-ABI case. + chain::bytes regproducer_bytes; + bool regproducer_ok = false; + try { + const auto regproducer_var = fc::json::from_string( + R"({"producer":"alice","producer_key":"SYS1111111111111111111111111111111114T1Anm",)" + R"("url":"https://example.com","location":0})"); + regproducer_bytes = system_abis.variant_to_binary("regproducer", regproducer_var, yield_fn()); + regproducer_ok = true; + } catch (const std::exception& e) { + std::cerr << "abi_serializer benchmark warning: regproducer prepack failed (" << e.what() + << "); skipping regproducer scenarios.\n"; + } + + // Construction scenarios. Each construction is slow enough that one outer + // run is meaningful; no inner loop needed. + benchmarking("abi_serializer construct token (small ~4KB)", [&] { + abi_serializer a{abi_def(token_abi), yield_fn()}; + (void)a; + }); + benchmarking("abi_serializer construct system (~37KB)", [&] { + abi_serializer a{abi_def(system_abi), yield_fn()}; + (void)a; + }); + + // Steady-state decode (variant path). Inner loop sized to keep per-op + // overhead well above timer resolution. + constexpr size_t decode_iters = 10000; + constexpr size_t big_decode_iters = 5000; + + benchmarking("decode transfer x10000 (variant)", [&] { + for (size_t i = 0; i < decode_iters; ++i) { + auto v = token_abis.binary_to_variant("transfer", transfer_bytes, yield_fn()); + (void)v; + } + }); + benchmarking("decode newaccount x5000 (variant, nested)", [&] { + for (size_t i = 0; i < big_decode_iters; ++i) { + auto v = system_abis.binary_to_variant("newaccount", newaccount_bytes, yield_fn()); + (void)v; + } + }); + if (regproducer_ok) { + benchmarking("decode regproducer x5000 (variant)", [&] { + for (size_t i = 0; i < big_decode_iters; ++i) { + auto v = system_abis.binary_to_variant("regproducer", regproducer_bytes, yield_fn()); + (void)v; + } + }); + } + + // Round-trip pack+unpack of transfer -- covers the pack half, not just unpack. + benchmarking("roundtrip transfer x5000 (pack+unpack)", [&] { + for (size_t i = 0; i < big_decode_iters; ++i) { + auto bin = token_abis.variant_to_binary("transfer", transfer_var, yield_fn()); + auto var = token_abis.binary_to_variant("transfer", bin, yield_fn()); + (void)var; + } + }); + + // resolve_type on a typedef ("account_name" -> "name") -- typedef chain walk. + benchmarking("resolve_type account_name x100000", [&] { + const std::string probe = "account_name"; + for (size_t i = 0; i < 100000; ++i) { + auto r = system_abis.resolve_type(probe); + (void)r; + } + }); + + // ------------------------------------------------------------------------- + // Streaming JSON comparisons. Each pair measures the same payload twice: + // - "(variant) json": binary_to_variant + fc::json::to_string + // - "(stream) json" : binary_to_json_stream directly into a json_writer + // + // The streaming path skips the per-field fc::variant tree and the post-pass + // string serialiser, so the gap should grow with field count. + // ------------------------------------------------------------------------- + + benchmarking("transfer x10000 (variant) json", [&] { + for (size_t i = 0; i < decode_iters; ++i) { + auto v = token_abis.binary_to_variant("transfer", transfer_bytes, yield_fn()); + auto s = fc::json::to_string(v, fc::time_point::maximum()); + (void)s; + } + }); + benchmarking("transfer x10000 (stream) json", [&] { + for (size_t i = 0; i < decode_iters; ++i) { + std::string out; + out.reserve(256); + fc::json_writer w(out); + token_abis.binary_to_json_stream("transfer", transfer_bytes, w, yield_fn()); + } + }); + + benchmarking("newaccount x5000 (variant) json", [&] { + for (size_t i = 0; i < big_decode_iters; ++i) { + auto v = system_abis.binary_to_variant("newaccount", newaccount_bytes, yield_fn()); + auto s = fc::json::to_string(v, fc::time_point::maximum()); + (void)s; + } + }); + benchmarking("newaccount x5000 (stream) json", [&] { + for (size_t i = 0; i < big_decode_iters; ++i) { + std::string out; + out.reserve(512); + fc::json_writer w(out); + system_abis.binary_to_json_stream("newaccount", newaccount_bytes, w, yield_fn()); + } + }); + + if (regproducer_ok) { + benchmarking("regproducer x5000 (variant) json", [&] { + for (size_t i = 0; i < big_decode_iters; ++i) { + auto v = system_abis.binary_to_variant("regproducer", regproducer_bytes, yield_fn()); + auto s = fc::json::to_string(v, fc::time_point::maximum()); + (void)s; + } + }); + benchmarking("regproducer x5000 (stream) json", [&] { + for (size_t i = 0; i < big_decode_iters; ++i) { + std::string out; + out.reserve(256); + fc::json_writer w(out); + system_abis.binary_to_json_stream("regproducer", regproducer_bytes, w, yield_fn()); + } + }); + } + + // ------------------------------------------------------------------------- + // get_block-shape comparison. Builds a synthetic signed_block holding a + // single transaction with N transfer actions, then times two paths: + // - variant: abi_serializer::to_variant + json::to_string + // (mirrors chain_plugin::convert_block) + // - stream: abi_serializer::to_json_stream straight into + // a json_writer (mirrors convert_block_stream) + // The actions count surfaces the per-action ABI decode cost, which is the + // variant path's main allocation hot spot. + // ------------------------------------------------------------------------- + + constexpr size_t actions_per_trx = 4; + constexpr size_t block_iters = 2000; + + transaction inner_trx; + inner_trx.expiration = fc::time_point_sec{ fc::time_point::now() + fc::seconds(60) }; + inner_trx.ref_block_num = 1; + inner_trx.ref_block_prefix = 1; + for (size_t i = 0; i < actions_per_trx; ++i) { + action act; + act.account = "sysio.token"_n; + act.name = "transfer"_n; + act.authorization = { { "alice"_n, "active"_n } }; + act.data = transfer_bytes; + inner_trx.actions.push_back(std::move(act)); + } + + signed_transaction signed_trx{ std::move(inner_trx), {}, {} }; + packed_transaction ptrx(std::move(signed_trx), packed_transaction::compression_type::none); + transaction_receipt rcpt(std::move(ptrx)); + + signed_block block; + block.producer = "alice"_n; + block.timestamp = block_timestamp_type{}; + block.transactions.push_back(std::move(rcpt)); + + // Resolver: sysio.token -> the prebuilt token_abis; other accounts -> nullopt + // (matches production chain_plugin behaviour for unknown accounts). + auto block_resolver = [&token_abis](const name& acct) -> std::optional { + if (acct == "sysio.token"_n) return token_abis; + return {}; + }; + + benchmarking("block (4 transfers) x2000 (variant) json", [&] { + for (size_t i = 0; i < block_iters; ++i) { + fc::variant pretty; + abi_serializer::to_variant(block, pretty, block_resolver, max_serialization_time); + auto s = fc::json::to_string(pretty, fc::time_point::maximum()); + (void)s; + } + }); + benchmarking("block (4 transfers) x2000 (stream) json", [&] { + for (size_t i = 0; i < block_iters; ++i) { + std::string out; + out.reserve(2048); + fc::json_writer w(out); + abi_serializer::to_json_stream(block, w, block_resolver, max_serialization_time); + } + }); + + // ------------------------------------------------------------------------- + // get_block-shape comparison INCLUDING per-request resolver build. The + // scenarios above hold the abi_serializer constant across iterations -- that + // measures only the walk + emit cost. In production each /v1/chain/get_block + // request rebuilds the resolver from the captured abi bytes (Phase 2 of + // read_only::get_block_stream_async), so the per-request cost includes + // abi_serializer::to_abi(bytes, abi_def) + abi_serializer ctor for every + // unique account referenced in the block. These scenarios capture that. + // + // Setup: pack the parsed token ABI to raw bytes once (matches what + // capture_abis would produce), then in each iteration parse + construct + + // walk + emit, simulating the full Phase 2 + Phase 3 work for one request. + // ------------------------------------------------------------------------- + + const chain::bytes token_abi_bytes = fc::raw::pack(token_abi); + + constexpr size_t block_full_iters = 500; + + benchmarking("get_block-shape x500 (variant + abi build) json", [&] { + for (size_t i = 0; i < block_full_iters; ++i) { + abi_def abi; + abi_serializer::to_abi(token_abi_bytes, abi); + abi_serializer fresh{std::move(abi), yield_fn()}; + auto resolver = [&fresh](const name& acct) -> std::optional { + if (acct == "sysio.token"_n) return fresh; + return {}; + }; + fc::variant pretty; + abi_serializer::to_variant(block, pretty, resolver, max_serialization_time); + auto s = fc::json::to_string(pretty, fc::time_point::maximum()); + (void)s; + } + }); + benchmarking("get_block-shape x500 (stream + abi build) json", [&] { + for (size_t i = 0; i < block_full_iters; ++i) { + abi_def abi; + abi_serializer::to_abi(token_abi_bytes, abi); + abi_serializer fresh{std::move(abi), yield_fn()}; + auto resolver = [&fresh](const name& acct) -> std::optional { + if (acct == "sysio.token"_n) return fresh; + return {}; + }; + std::string out; + out.reserve(2048); + fc::json_writer w(out); + abi_serializer::to_json_stream(block, w, resolver, max_serialization_time); + } + }); +} + +} // namespace sysio::benchmark diff --git a/benchmark/benchmark.cpp b/benchmark/benchmark.cpp index f2887b8f85..914c511919 100644 --- a/benchmark/benchmark.cpp +++ b/benchmark/benchmark.cpp @@ -17,6 +17,8 @@ std::map> features { { "blake2", blake2_benchmarking }, { "bls", bls_benchmarking }, { "merkle", merkle_benchmarking }, + { "trace_api_json", trace_api_json_benchmarking }, + { "abi_serializer", abi_serializer_benchmarking }, { "auth", auth_benchmarking } }; @@ -78,7 +80,18 @@ void benchmarking(const std::string& name, const std::function& func, for (auto i = 0U; i < runs; ++i) { auto start_time = std::chrono::high_resolution_clock::now(); - func(); + // A throwing scenario (e.g. an abi-serialization deadline tripped by a transient + // machine stall during a long -r run) reports and skips the scenario instead of + // taking the whole benchmark process down with an uncaught exception. + try { + func(); + } catch (const std::exception& e) { + std::cout << name << ": SKIPPED after run " << i << " -- " << e.what() << "\n"; + return; + } catch (...) { + std::cout << name << ": SKIPPED after run " << i << " -- unknown exception\n"; + return; + } auto end_time = std::chrono::high_resolution_clock::now(); uint64_t duration = std::chrono::duration_cast(end_time - start_time).count(); diff --git a/benchmark/benchmark.hpp b/benchmark/benchmark.hpp index 59629e4ce6..384b9d1885 100644 --- a/benchmark/benchmark.hpp +++ b/benchmark/benchmark.hpp @@ -24,6 +24,8 @@ void hash_benchmarking(); void blake2_benchmarking(); void bls_benchmarking(); void merkle_benchmarking(); +void trace_api_json_benchmarking(); +void abi_serializer_benchmarking(); void auth_benchmarking(); void benchmarking(const std::string& name, const std::function& func, std::optional num_runs = {}); diff --git a/benchmark/trace_api_json.cpp b/benchmark/trace_api_json.cpp new file mode 100644 index 0000000000..5ed9c1c2ba --- /dev/null +++ b/benchmark/trace_api_json.cpp @@ -0,0 +1,306 @@ +#include + +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +// Compares the two trace_api block serialization paths: +// 1) fc::mutable_variant_object -> fc::variant -> fc::json::to_string (legacy) +// 2) fc::json_writer streaming directly into std::string (this PR) +// +// Each run serializes a fresh synthetic block_trace_v0 with a configurable number of +// transactions and actions-per-transaction. The shapes and field types match what +// real producers emit, so the comparison reflects realistic HTTP workloads. + +namespace sysio::benchmark { + +namespace { + +using namespace sysio::trace_api; + +// Deterministic filler for reproducible timings across runs. Uses a stable PRNG so +// the synthetic trace's byte payloads don't change shape between "variant" and +// "stream" timings - otherwise any compiler/cache effect would dominate the signal. +std::mt19937_64 make_rng() { return std::mt19937_64{0xDEADBEEFCAFEF00Dull}; } + +chain::bytes random_bytes(std::mt19937_64& rng, size_t n) { + chain::bytes b(n); + std::uniform_int_distribution byte_dist{0, 255}; + for (auto& c : b) c = static_cast(byte_dist(rng)); + return b; +} + +const chain::signature_type& fake_signature() { + // One real K1 signature generated once at first call, reused across all synthetic + // trxs. We're measuring serialization cost of the signature type, not the + // number of distinct signatures, so reuse is fine. + static const chain::signature_type s = []{ + auto priv = fc::crypto::private_key::generate(); + return priv.sign(fc::sha256::hash(std::string{"benchmark-digest"})); + }(); + return s; +} + +block_trace_v0 make_synthetic_block_trace(size_t num_trxs, size_t actions_per_trx, size_t action_data_bytes) { + auto rng = make_rng(); + + block_trace_v0 bt; + bt.id = fc::sha256::hash(std::string{"block-id"}); + bt.number = 123456; + bt.previous_id = fc::sha256::hash(std::string{"prev-id"}); + bt.timestamp = chain::block_timestamp_type(1'700'000'000u); + bt.producer = chain::name{"defproducera"}; + bt.transaction_mroot = fc::sha256::hash(std::string{"trx-mroot"}); + bt.finality_mroot = fc::sha256::hash(std::string{"fin-mroot"}); + + bt.transactions.reserve(num_trxs); + for (size_t i = 0; i < num_trxs; ++i) { + transaction_trace_v0 trx; + trx.id = fc::sha256::hash(std::string{"trx-"} + std::to_string(i)); + trx.actions.reserve(actions_per_trx); + uint64_t seq = i * actions_per_trx; + for (size_t a = 0; a < actions_per_trx; ++a) { + action_trace_v0 act; + act.global_sequence = seq++; + act.receiver = chain::name{"sysio.token"}; + act.account = chain::name{"sysio.token"}; + act.action = chain::name{"transfer"}; + act.authorization.push_back({chain::name{"alice"}, chain::name{"active"}}); + act.data = random_bytes(rng, action_data_bytes); + act.return_value = {}; + trx.actions.push_back(std::move(act)); + } + trx.cpu_usage_us = 150; + trx.net_usage_words = fc::unsigned_int{16}; + trx.signatures.push_back(fake_signature()); + trx.trx_header.expiration = fc::time_point_sec{1'700'000'100u}; + trx.trx_header.ref_block_num = static_cast(i); + trx.trx_header.ref_block_prefix = 0xcafebabe; + trx.block_num = bt.number; + trx.block_time = bt.timestamp; + trx.producer_block_id = bt.id; + bt.transactions.push_back(std::move(trx)); + } + return bt; +} + +// A data_handler that does no ABI decode (returns a null params variant). The two +// paths hit the same no-op branch, so this isolates the envelope serialization cost +// from the ABI decode cost - which is shared unchanged between paths. +data_handler_function make_noop_data_handler() { + return [](const std::variant&) -> std::tuple> { + return {fc::variant(), std::optional{}}; + }; +} + +// Streaming counterpart: emits no params/return_data keys, mirroring the variant noop above so +// both measured paths skip the same ABI-decode work. +stream_data_handler_function make_noop_stream_data_handler() { + return [](const std::variant&, fc::json_writer&) {}; +} + +} // namespace + +void trace_api_json_benchmarking() { + using namespace sysio::trace_api; + + // Real produced blocks from the perf harness peak at ~25k trxs with 1 action each. + // Benchmark several shapes so the scaling (per-trx, per-action, payload bytes) is + // visible. + struct shape { size_t trxs; size_t actions_per_trx; size_t data_bytes; }; + const shape shapes[] = { + { 100, 1, 32 }, // small block, common action + { 5'000, 1, 32 }, // medium block + { 25'000, 1, 32 }, // high-TPS block size + { 1'000, 4, 128 }, // complex trxs, modest block + }; + + const auto noop_handler = make_noop_data_handler(); + const auto noop_stream_handler = make_noop_stream_data_handler(); + const bool irreversible = true; + + for (const auto& s : shapes) { + const auto bt = make_synthetic_block_trace(s.trxs, s.actions_per_trx, s.data_bytes); + const auto entry = data_log_entry{bt}; + + const std::string label_prefix = std::to_string(s.trxs) + " trxs x " + + std::to_string(s.actions_per_trx) + " acts (" + + std::to_string(s.data_bytes) + "B data)"; + + // Legacy total path: build variant tree + json::to_string. Times the full + // chain the HTTP handler would execute (variant build on main thread, then + // json::to_string on HTTP thread). Useful as a total-CPU baseline. + benchmarking("variant+json: " + label_prefix, [&]() { + auto v = detail::response_formatter::process_block(entry, irreversible, noop_handler); + std::string out = fc::json::to_string(v, fc::json::yield_function_t()); + // Defeat the optimizer. + asm volatile("" : : "g"(out.size()) : "memory"); + }); + + // Variant-only: build the variant tree without serializing to JSON. This is + // the actual current main-thread cost in production - json::to_string runs on + // an HTTP-pool thread after cb() hands the variant off. Compare against + // `stream:` below to decide whether moving JSON emission onto the main thread + // is a net main-thread win or regression. + benchmarking("variant-only: " + label_prefix, [&]() { + auto v = detail::response_formatter::process_block(entry, irreversible, noop_handler); + asm volatile("" : : "g"(v.is_object()) : "memory"); + }); + + // Streaming path: process_block_to_json writes JSON directly into std::string. + // This is the proposed new main-thread cost (no variant intermediate, no + // separate HTTP-thread serialization step). + benchmarking("stream: " + label_prefix, [&]() { + std::string out = detail::response_formatter::process_block_to_json(entry, irreversible, noop_stream_handler); + asm volatile("" : : "g"(out.size()) : "memory"); + }); + + // Struct-pass (HTTP-thread streaming pivot floor): main-thread cost when + // the API lambda hands the result struct (not a variant, not a string) to + // the HTTP thread for serialization. API lambda captures the struct into + // a std::function closure and hands it to cb(); the + // HTTP thread runs the closure later (off-main, not measured here). The + // capture is move-only so the per-trx vectors transfer pointers, not data. + // std::function does heap-allocate because the captured block_trace_v0 + // exceeds the small-buffer optimisation threshold. + // + // Pre-builds a pool of data_log_entry copies outside the timing loop so + // each iter consumes a fresh one. Setup cost is excluded from the bench + // (matches variant-only / stream which also start from a pre-built entry). + const size_t pool_size = 64; // > any reasonable --runs + std::vector fresh_a(pool_size, entry); + std::vector fresh_b(pool_size, entry); + size_t fresh_idx_a = 0; + size_t fresh_idx_b = 0; + benchmarking("struct-pass (fn): " + label_prefix, [&]() { + data_log_entry e = std::move(fresh_a[fresh_idx_a++ % pool_size]); + std::function body = + [e = std::move(e)](fc::json_writer& w) mutable { + asm volatile("" : : "g"(&e) : "memory"); + }; + auto holder = std::move(body); // mimic cb() posting onto the HTTP pool + asm volatile("" : : "g"(&holder) : "memory"); + }); + benchmarking("struct-pass (mof): " + label_prefix, [&]() { + data_log_entry e = std::move(fresh_b[fresh_idx_b++ % pool_size]); + fc::move_only_function body = + [e = std::move(e)](fc::json_writer& w) mutable { + asm volatile("" : : "g"(&e) : "memory"); + }; + auto holder = std::move(body); + asm volatile("" : : "g"(&holder) : "memory"); + }); + } + + // ---- Isolated hex-emission micro-bench ------------------------------------ + // + // Probes the hex-encoding cost in isolation, the suspected dominant cause of + // the streaming-vs-variant-only main-thread regression on data-heavy actions. + // Two paths per byte size: + // to_hex+value_string : fc::to_hex(...) -> std::string -> w.value_string(sv) + // value_hex : json_writer::value_hex writes hex straight into the buffer + // + // Each iteration emits a single JSON string token of N hex chars, so the delta + // is the per-byte hex-encoding overhead (plus one std::string alloc/dealloc on + // the to_hex path). Higher N amortises the prefix/quote bookkeeping and + // surfaces the loop cost. + + { + auto rng = make_rng(); + const size_t hex_iter_bytes_total = 64 * 1024; // ~constant work per scenario + const size_t sizes[] = { 32, 128, 512, 2048 }; + for (size_t bytes_per_call : sizes) { + auto buf = random_bytes(rng, bytes_per_call); + const size_t calls = std::max(1, hex_iter_bytes_total / bytes_per_call); + const std::string label = std::to_string(bytes_per_call) + "B per call x " + std::to_string(calls); + + benchmarking("hex variant: " + label, [&]() { + std::string out; + fc::json_writer w(out); + w.begin_array(); + for (size_t i = 0; i < calls; ++i) { + w.value_string(fc::to_hex(buf.data(), buf.size())); + } + w.end_array(); + asm volatile("" : : "g"(out.size()) : "memory"); + }); + + benchmarking("hex direct: " + label, [&]() { + std::string out; + fc::json_writer w(out); + w.begin_array(); + for (size_t i = 0; i < calls; ++i) { + w.value_hex(buf.data(), buf.size()); + } + w.end_array(); + asm volatile("" : : "g"(out.size()) : "memory"); + }); + } + } + + // ---- Isolated integer-emission micro-bench -------------------------------- + // + // Probes the integer-to-decimal cost in isolation, comparing snprintf (the + // pre-change implementation of json_writer::value_uint64) against std::to_chars + // (the post-change implementation). Each iteration emits 1024 uint64 values + // covering a realistic mix of magnitudes: small (status / counters), medium + // (block numbers, cpu_usage_us), large (global_sequence on a long-running + // chain). Values are pre-shuffled so branch prediction can't dominate. + + { + auto rng = make_rng(); + std::vector mix; + mix.reserve(1024); + for (size_t i = 0; i < 1024; ++i) { + std::uniform_int_distribution bucket{0, 2}; + switch (bucket(rng)) { + case 0: mix.push_back(rng() % 256); break; // small + case 1: mix.push_back(rng() % 100'000'000ull); break; // medium + case 2: mix.push_back(rng()); break; // large (full uint64) + } + } + + auto append_snprintf = [](std::string& out, uint64_t n) { + char buf[24]; + int k = std::snprintf(buf, sizeof(buf), "%llu", static_cast(n)); + if (k > 0) out.append(buf, static_cast(k)); + }; + + auto append_to_chars = [](std::string& out, uint64_t n) { + char buf[24]; + auto r = std::to_chars(buf, buf + sizeof(buf), n); + out.append(buf, static_cast(r.ptr - buf)); + }; + + benchmarking("uint64 snprintf: 1024 mixed", [&]() { + std::string out; + out.reserve(16 * 1024); + for (uint64_t n : mix) { + append_snprintf(out, n); + out.push_back(','); + } + asm volatile("" : : "g"(out.size()) : "memory"); + }); + + benchmarking("uint64 to_chars: 1024 mixed", [&]() { + std::string out; + out.reserve(16 * 1024); + for (uint64_t n : mix) { + append_to_chars(out, n); + out.push_back(','); + } + asm volatile("" : : "g"(out.size()) : "memory"); + }); + } +} + +} // namespace sysio::benchmark diff --git a/libraries/chain/abi_serializer.cpp b/libraries/chain/abi_serializer.cpp index 83de928b4e..d7c2e776c3 100644 --- a/libraries/chain/abi_serializer.cpp +++ b/libraries/chain/abi_serializer.cpp @@ -1,5 +1,7 @@ #include +#include #include +#include #include #include #include @@ -10,6 +12,10 @@ #include #include +#include +#include +#include + namespace sysio::chain { const size_t abi_serializer::max_recursion_depth; @@ -73,6 +79,141 @@ namespace sysio::chain { ); } + // Stream-side counterpart to `pack_unpack`. The lambda reads a single value of type + // `T` from the datastream and forwards to the per-type `emit(value, json_writer&)` + // callback, which is responsible for matching the variant path's exact JSON shape + // (quote conventions for large 64-bit ints, decimal strings for 128-bit, etc). + template + auto pack_unpack_stream(Emit emit) { + return [emit]( fc::datastream& stream, bool is_array, bool is_optional, + const abi_serializer::yield_function_t& yield, fc::json_writer& w ) { + if( is_array ) { + fc::unsigned_int sz; + fc::raw::unpack(stream, sz); + w.begin_array(); + for( fc::unsigned_int::base_uint i = 0; i < sz.value; ++i ) { + T val; + fc::raw::unpack(stream, val); + emit(val, w); + } + w.end_array(); + } else if( is_optional ) { + char flag; + fc::raw::unpack(stream, flag); + if( flag ) { + T val; + fc::raw::unpack(stream, val); + emit(val, w); + } else { + w.value_null(); + } + } else { + T val; + fc::raw::unpack(stream, val); + emit(val, w); + } + }; + } + + // Same as `pack_unpack_stream`, but invokes `yield(0)` on each value read so a + // long signature/public_key decode can bail out at the configured deadline. + template + auto pack_unpack_stream_deadline(Emit emit) { + return [emit]( fc::datastream& stream, bool is_array, bool is_optional, + const abi_serializer::yield_function_t& yield, fc::json_writer& w ) { + if( is_array ) { + fc::unsigned_int sz; + fc::raw::unpack(stream, sz); + w.begin_array(); + for( fc::unsigned_int::base_uint i = 0; i < sz.value; ++i ) { + T val; + fc::raw::unpack(stream, val); + yield(0); + emit(val, w); + } + w.end_array(); + } else if( is_optional ) { + char flag; + fc::raw::unpack(stream, flag); + if( flag ) { + T val; + fc::raw::unpack(stream, val); + yield(0); + emit(val, w); + } else { + w.value_null(); + } + } else { + T val; + fc::raw::unpack(stream, val); + yield(0); + emit(val, w); + } + }; + } + + using built_in_stream_unpack_fn = std::function&, bool, bool, + const abi_serializer::yield_function_t&, + fc::json_writer&)>; + + // Static dispatch table for the streaming side of `unpack_built_in`. Built once on + // first access and then read-only. Variant-side overrides via + // `add_specialized_unpack_pack` are not reflected here; the streaming consumers + // (chain_api endpoints) do not register overrides today. + const std::map>& get_built_in_stream_unpacks() { + static const auto map = []() { + std::map> m; + + // fc::to_json_stream provides the variant-path shapes for every scalar here: + // int64/uint64 quote past fc::json_integer_quote_magnitude, float32/float64 + // quote with the digits10+2 fixed-precision to_chars form. + auto generic = [](const auto& v, fc::json_writer& w) { fc::to_json_stream(v, w); }; + + m.emplace("bool", pack_unpack_stream(generic)); + m.emplace("int8", pack_unpack_stream(generic)); + m.emplace("uint8", pack_unpack_stream(generic)); + m.emplace("int16", pack_unpack_stream(generic)); + m.emplace("uint16", pack_unpack_stream(generic)); + m.emplace("int32", pack_unpack_stream(generic)); + m.emplace("uint32", pack_unpack_stream(generic)); + m.emplace("int64", pack_unpack_stream(generic)); + m.emplace("uint64", pack_unpack_stream(generic)); + m.emplace("int128", pack_unpack_stream(generic)); + m.emplace("uint128", pack_unpack_stream(generic)); + m.emplace("varint32", pack_unpack_stream(generic)); + m.emplace("varuint32", pack_unpack_stream(generic)); + + m.emplace("float32", pack_unpack_stream(generic)); + m.emplace("float64", pack_unpack_stream(generic)); + m.emplace("float128", pack_unpack_stream(generic)); + + m.emplace("time_point", pack_unpack_stream(generic)); + m.emplace("time_point_sec", pack_unpack_stream(generic)); + m.emplace("block_timestamp_type", pack_unpack_stream(generic)); + + m.emplace("name", pack_unpack_stream(generic)); + + m.emplace("bytes", pack_unpack_stream(generic)); + m.emplace("string", pack_unpack_stream(generic)); + + m.emplace("checksum160", pack_unpack_stream(generic)); + m.emplace("checksum256", pack_unpack_stream(generic)); + m.emplace("checksum512", pack_unpack_stream(generic)); + + m.emplace("public_key", pack_unpack_stream_deadline(generic)); + m.emplace("signature", pack_unpack_stream_deadline(generic)); + + m.emplace("symbol", pack_unpack_stream(generic)); + m.emplace("symbol_code", pack_unpack_stream(generic)); + m.emplace("asset", pack_unpack_stream(generic)); + m.emplace("extended_asset", pack_unpack_stream(generic)); + m.emplace("bitset", pack_unpack_stream(generic)); + + return m; + }(); + return map; + } + abi_serializer::abi_serializer( abi_def abi, const yield_function_t& yield ) { configure_built_in_types(); set_abi(std::move(abi), yield); @@ -88,6 +229,12 @@ namespace sysio::chain { built_in_types[name] = std::move( unpack_pack ); } + const std::pair* + abi_serializer::find_built_in(std::string_view type) const noexcept { + auto it = built_in_types.find(type); + return it == built_in_types.end() ? nullptr : &it->second; + } + void abi_serializer::configure_built_in_types() { built_in_types.emplace("bool", pack_unpack()); @@ -262,38 +409,63 @@ namespace sysio::chain { SYS_ASSERT( status.ok(), pack_exception, "Failed to convert JSON to protobuf message: {}", std::string(status.message()) ); } + namespace { + // Shared length-prefix decode used by both pb_binary_to_variant and + // pb_binary_to_json_string. Returns the decoded protobuf message, having + // already advanced `stream` past the consumed bytes. + std::unique_ptr pb_decode_message( + const std::string_view& type, fc::datastream& stream, + const google::protobuf::Descriptor* desc, + google::protobuf::DynamicMessageFactory& factory ) { + SYS_ASSERT( desc, unpack_exception, "Unknown protobuf type '{}'", impl::limit_size(type) ); + + // Outer varuint32 length prefix covers inner length prefix + protobuf bytes. + fc::unsigned_int outer_len; + fc::raw::unpack(stream, outer_len); + SYS_ASSERT( stream.remaining() >= outer_len.value, unpack_exception, + "Not enough data for protobuf message '{}': need {} bytes, have {}", + impl::limit_size(type), outer_len.value, stream.remaining() ); + + // Inner varuint32 length prefix (zpp_bits size_varint format). + fc::datastream inner_stream(stream.pos(), outer_len.value); + fc::unsigned_int pb_len; + fc::raw::unpack(inner_stream, pb_len); + + SYS_ASSERT( pb_len.value <= inner_stream.remaining(), unpack_exception, + "Protobuf message '{}': inner length {} exceeds available data {}", + impl::limit_size(type), pb_len.value, inner_stream.remaining() ); + SYS_ASSERT( pb_len.value == inner_stream.remaining(), unpack_exception, + "Protobuf message '{}': trailing data detected (inner length {} but {} bytes remain)", + impl::limit_size(type), pb_len.value, inner_stream.remaining() ); + + auto prototype = factory.GetPrototype(desc); + std::unique_ptr msg(prototype->New()); + SYS_ASSERT( msg->ParseFromArray(inner_stream.pos(), pb_len.value), unpack_exception, + "Failed to parse protobuf message '{}'", impl::limit_size(type) ); + stream.skip(outer_len.value); + return msg; + } + } + fc::variant abi_serializer::pb_binary_to_variant(const std::string_view& type, fc::datastream& stream) const { auto desc = get_pb_descriptor(type); - SYS_ASSERT( desc, unpack_exception, "Unknown protobuf type '{}'", impl::limit_size(type) ); - - // Read outer varuint32 length prefix (covers inner length prefix + protobuf bytes) - fc::unsigned_int outer_len; - fc::raw::unpack(stream, outer_len); - SYS_ASSERT( stream.remaining() >= outer_len.value, unpack_exception, - "Not enough data for protobuf message '{}': need {} bytes, have {}", - impl::limit_size(type), outer_len.value, stream.remaining() ); - - // Read inner varuint32 length prefix (zpp_bits size_varint format) - fc::datastream inner_stream(stream.pos(), outer_len.value); - fc::unsigned_int pb_len; - fc::raw::unpack(inner_stream, pb_len); - - SYS_ASSERT( pb_len.value <= inner_stream.remaining(), unpack_exception, - "Protobuf message '{}': inner length {} exceeds available data {}", - impl::limit_size(type), pb_len.value, inner_stream.remaining() ); - SYS_ASSERT( pb_len.value == inner_stream.remaining(), unpack_exception, - "Protobuf message '{}': trailing data detected (inner length {} but {} bytes remain)", - impl::limit_size(type), pb_len.value, inner_stream.remaining() ); - - auto prototype = pb_factory->GetPrototype(desc); - std::unique_ptr msg(prototype->New()); - SYS_ASSERT( msg->ParseFromArray(inner_stream.pos(), pb_len.value), unpack_exception, - "Failed to parse protobuf message '{}'", impl::limit_size(type) ); - stream.skip(outer_len.value); - + auto msg = pb_decode_message(type, stream, desc, *pb_factory); return pb_message_to_variant(*msg); } + std::string abi_serializer::pb_binary_to_json_string(const std::string_view& type, fc::datastream& stream) const { + namespace gpb = google::protobuf; + auto desc = get_pb_descriptor(type); + auto msg = pb_decode_message(type, stream, desc, *pb_factory); + std::string json; + gpb::util::JsonPrintOptions opts; + opts.preserve_proto_field_names = true; + auto status = gpb::util::MessageToJsonString(*msg, &json, opts); + SYS_ASSERT( status.ok(), unpack_exception, "Failed to convert protobuf message to JSON: {}", + std::string(status.message()) ); + return json; + } + void abi_serializer::pb_variant_to_binary(const std::string_view& type, const fc::variant& var, fc::datastream& ds) const { auto desc = get_pb_descriptor(type); SYS_ASSERT( desc, pack_exception, "Unknown protobuf type '{}'", impl::limit_size(type) ); @@ -511,136 +683,12 @@ namespace sysio::chain { return type; } - void abi_serializer::_binary_to_variant( const std::string_view& type, fc::datastream& stream, - fc::mutable_variant_object& obj, impl::binary_to_variant_context& ctx )const - { - auto h = ctx.enter_scope(); - auto s_itr = structs.find(type); - SYS_ASSERT( s_itr != structs.end(), invalid_type_inside_abi, "Unknown type {}", ctx.maybe_shorten(type) ); - ctx.hint_struct_type_if_in_array( s_itr ); - const auto& st = s_itr->second; - if( st.base != type_name() ) { - _binary_to_variant(resolve_type(st.base), stream, obj, ctx); - } - bool encountered_extension = false; - for( uint32_t i = 0; i < st.fields.size(); ++i ) { - const auto& field = st.fields[i]; - bool extension = field.type.ends_with("$"); - encountered_extension |= extension; - if( !stream.remaining() ) { - if( extension ) { - continue; - } - if( encountered_extension ) { - SYS_THROW( abi_exception, "Encountered field '{}' without binary extension designation while processing struct '{}'", - ctx.maybe_shorten(field.name), ctx.get_path_string() ); - } - SYS_THROW( unpack_exception, "Stream unexpectedly ended; unable to unpack field '{}' of struct '{}'", - ctx.maybe_shorten(field.name), ctx.get_path_string() ); - - } - auto h1 = ctx.push_to_path( impl::field_path_item{ .parent_struct_itr = s_itr, .field_ordinal = i } ); - auto field_type = resolve_type( extension ? _remove_bin_extension(field.type) : field.type ); - auto v = _binary_to_variant(field_type, stream, ctx); - if( ctx.is_logging() && v.is_string() && field_type == "bytes" ) { - fc::mutable_variant_object sub_obj; - auto size = v.get_string().size() / 2; // half because it is in hex - sub_obj( "size", size ); - if( size > impl::hex_log_max_size ) { - sub_obj( "trimmed_hex", v.get_string().substr( 0, impl::hex_log_max_size*2 ) ); - } else { - sub_obj( "hex", std::move( v ) ); - } - obj( field.name, std::move(sub_obj) ); - } else { - obj( field.name, std::move(v) ); - } - } - } - fc::variant abi_serializer::_binary_to_variant( const std::string_view& type, fc::datastream& stream, impl::binary_to_variant_context& ctx )const { - auto h = ctx.enter_scope(); - auto rtype = resolve_type(type); - auto ftype = fundamental_type(rtype); - auto fixed_array_sz = is_szarray(rtype); - - auto read_array = [&](fc::unsigned_int::base_uint sz) { - ctx.hint_array_type_if_in_array(); - fc::variants vars; - vars.reserve(std::min(sz, 1024u)); // limit the maximum size that can be reserved before data is read - auto h1 = ctx.push_to_path( impl::array_index_path_item{} ); - for( fc::unsigned_int::base_uint i = 0; i < sz; ++i ) { - ctx.set_array_index_of_path_back(i); - auto v = _binary_to_variant(ftype, stream, ctx); - // The exception below is commented out to allow array of optional as input data - //SYS_ASSERT( !v.is_null(), unpack_exception, "Invalid packed array '${p}'", ("p", ctx.get_path_string()) ); - vars.emplace_back(std::move(v)); - } - return fc::variant(std::move(vars)); - }; - - if (fixed_array_sz) { - return read_array(*fixed_array_sz); - } else if( auto btype = built_in_types.find(ftype ); btype != built_in_types.end() ) { - try { - return btype->second.first(stream, is_array(rtype), is_optional(rtype), ctx.get_yield_function()); - } SYS_RETHROW_EXCEPTIONS( unpack_exception, "Unable to unpack {} type '{}' while processing '{}'", - is_array(rtype) ? "array of built-in" : is_optional(rtype) ? "optional of built-in" : "built-in", - impl::limit_size(ftype), ctx.get_path_string() ) - } else if( is_protobuf_type(ftype) ) { - try { - return pb_binary_to_variant(ftype, stream); - } SYS_RETHROW_EXCEPTIONS( unpack_exception, "Unable to unpack protobuf type '{}' while processing '{}'", - impl::limit_size(ftype), ctx.get_path_string() ) - } - if ( is_array(rtype) ) { - fc::unsigned_int size; - try { - fc::raw::unpack(stream, size); - } SYS_RETHROW_EXCEPTIONS( unpack_exception, "Unable to unpack size of array '{}'", ctx.get_path_string() ) - return read_array(size.value); - } else if ( is_optional(rtype) ) { - char flag; - try { - fc::raw::unpack(stream, flag); - } SYS_RETHROW_EXCEPTIONS( unpack_exception, "Unable to unpack presence flag of optional '{}'", ctx.get_path_string() ) - return flag ? _binary_to_variant(ftype, stream, ctx) : fc::variant(); - } else if( auto e_itr = enums.find(rtype); e_itr != enums.end() ) { - // Enum type: read as underlying integer, return member name string. - // Falls back to integer for values not in the enum definition. - auto btype = built_in_types.find(e_itr->second.type); - SYS_ASSERT( btype != built_in_types.end(), invalid_type_inside_abi, - "Enum '{}' has unknown underlying type '{}'", impl::limit_size(rtype), impl::limit_size(e_itr->second.type) ); - auto int_var = btype->second.first(stream, false, false, ctx.get_yield_function()); - auto int_val = int_var.as_int64(); - for( const auto& ev : e_itr->second.values ) { - if( ev.value == int_val ) { - return fc::variant(ev.name); - } - } - return int_var; // Unknown value — return as integer - } else { - auto v_itr = variants.find(rtype); - if( v_itr != variants.end() ) { - ctx.hint_variant_type_if_in_array( v_itr ); - fc::unsigned_int select; - try { - fc::raw::unpack(stream, select); - } SYS_RETHROW_EXCEPTIONS( unpack_exception, "Unable to unpack tag of variant '{}'", ctx.get_path_string() ) - SYS_ASSERT( (size_t)select < v_itr->second.types.size(), unpack_exception, - "Unpacked invalid tag ({}) for variant '{}'", select.value, ctx.get_path_string() ); - auto h1 = ctx.push_to_path( impl::variant_path_item{ .variant_itr = v_itr, .variant_ordinal = static_cast(select) } ); - return fc::variants{v_itr->second.types[select], _binary_to_variant(v_itr->second.types[select], stream, ctx)}; - } - } - - fc::mutable_variant_object mvo; - _binary_to_variant(rtype, stream, mvo, ctx); - // QUESTION: Is this assert actually desired? It disallows unpacking empty structs from datastream. - SYS_ASSERT( mvo.size() > 0, unpack_exception, "Unable to unpack '{}' from stream", ctx.get_path_string() ); - return fc::variant( std::move(mvo) ); + impl::variant_sink sink(*this); + _binary_walk(type, stream, sink, ctx); + return std::move(sink).take_result(); } fc::variant abi_serializer::_binary_to_variant( const std::string_view& type, const bytes& binary, impl::binary_to_variant_context& ctx )const @@ -674,6 +722,34 @@ namespace sysio::chain { return _binary_to_variant(type, binary, ctx); } + void abi_serializer::binary_to_json_stream( const std::string_view& type, fc::datastream& binary, fc::json_writer& w, + const yield_function_t& yield, bool short_path )const { + impl::binary_to_variant_context ctx(*this, yield, fc::microseconds{}, type); + ctx.short_path = short_path; + impl::stream_sink sink(*this, w); + _binary_walk(type, binary, sink, ctx); + } + + void abi_serializer::binary_to_json_stream( const std::string_view& type, fc::datastream& binary, fc::json_writer& w, + const fc::microseconds& max_action_data_serialization_time, bool short_path )const { + impl::binary_to_variant_context ctx(*this, create_depth_yield_function(), max_action_data_serialization_time, type); + ctx.short_path = short_path; + impl::stream_sink sink(*this, w); + _binary_walk(type, binary, sink, ctx); + } + + void abi_serializer::binary_to_json_stream( const std::string_view& type, const bytes& binary, fc::json_writer& w, + const yield_function_t& yield, bool short_path )const { + fc::datastream ds(binary.data(), binary.size()); + binary_to_json_stream(type, ds, w, yield, short_path); + } + + void abi_serializer::binary_to_json_stream( const std::string_view& type, const bytes& binary, fc::json_writer& w, + const fc::microseconds& max_action_data_serialization_time, bool short_path )const { + fc::datastream ds(binary.data(), binary.size()); + binary_to_json_stream(type, ds, w, max_action_data_serialization_time, short_path); + } + void abi_serializer::_variant_to_binary( const std::string_view& type, const fc::variant& var, fc::datastream& ds, impl::variant_to_binary_context& ctx )const { try { auto h = ctx.enter_scope(); @@ -1207,3 +1283,453 @@ void from_variant(const fc::variant& v, sysio::chain::abi_def& abi) { } } // namespace fc + +namespace sysio::chain::impl { + +// -- variant_sink ------------------------------------------------------------------------ + +void variant_sink::begin_object() { + stack_.emplace_back(); + stack_.back().kind = frame_kind::object; +} + +void variant_sink::end_object() { + FC_ASSERT(!stack_.empty() && stack_.back().kind == frame_kind::object, + "variant_sink: end_object without matching begin_object"); + auto popped = std::move(stack_.back()); + stack_.pop_back(); + emit_value(fc::variant(std::move(popped.mvo))); +} + +void variant_sink::begin_array() { + stack_.emplace_back(); + stack_.back().kind = frame_kind::array; +} + +void variant_sink::end_array() { + FC_ASSERT(!stack_.empty() && stack_.back().kind == frame_kind::array, + "variant_sink: end_array without matching begin_array"); + auto popped = std::move(stack_.back()); + stack_.pop_back(); + emit_value(fc::variant(std::move(popped.arr))); +} + +void variant_sink::key(std::string_view k) { + FC_ASSERT(!stack_.empty() && stack_.back().kind == frame_kind::object, + "variant_sink: key emitted outside an object frame"); + auto& f = stack_.back(); + f.pending_key.assign(k.data(), k.size()); + f.has_pending_key = true; +} + +void variant_sink::value_string(std::string_view s) { emit_value(fc::variant(s)); } +void variant_sink::value_int64(int64_t n) { emit_value(fc::variant(n)); } +void variant_sink::value_uint64(uint64_t n) { emit_value(fc::variant(n)); } +void variant_sink::value_bool(bool b) { emit_value(fc::variant(b)); } +void variant_sink::value_null() { emit_value(fc::variant()); } + +void variant_sink::value_hex(const char* data, size_t size) { + if (size == 0) { + emit_value(fc::variant(std::string{})); + return; + } + emit_value(fc::variant(fc::to_hex(data, size))); +} + +void variant_sink::raw_value(std::string_view raw_json) { + emit_value(fc::json::from_string(std::string(raw_json))); +} + +bool variant_sink::frame_has_items() const noexcept { + return !stack_.empty() && stack_.back().item_count > 0; +} + +void variant_sink::emit_value(fc::variant v) { + if (stack_.empty()) { + result_ = std::move(v); + return; + } + auto& f = stack_.back(); + if (f.kind == frame_kind::object) { + FC_ASSERT(f.has_pending_key, "variant_sink: value without preceding key in object frame"); + f.mvo(std::move(f.pending_key), std::move(v)); + f.pending_key.clear(); + f.has_pending_key = false; + } else { + f.arr.emplace_back(std::move(v)); + } + ++f.item_count; +} + +void variant_sink::unpack_built_in(std::string_view ftype, fc::datastream& stream, + bool is_array, bool is_optional, abi_traverse_context& ctx) { + FC_ASSERT(abi_, "variant_sink::unpack_built_in requires the abi_serializer-bound constructor"); + auto bv = abi_->find_built_in(ftype); + FC_ASSERT(bv, "variant_sink::unpack_built_in: '{}' is not a built-in", std::string(ftype)); + emit_value(bv->first(stream, is_array, is_optional, ctx.get_yield_function())); +} + +void variant_sink::unpack_protobuf(std::string_view ftype, fc::datastream& stream) { + FC_ASSERT(abi_, "variant_sink::unpack_protobuf requires the abi_serializer-bound constructor"); + emit_value(abi_->pb_binary_to_variant(ftype, stream)); +} + +void variant_sink::unpack_action_data(const abi_serializer& abi, std::string_view type, const bytes& data, + abi_traverse_context& ctx, bool short_path) { + action_data_to_variant_context inner(abi, ctx, type); + inner.short_path = short_path; + emit_value(abi._binary_to_variant(type, data, inner)); +} + +bool variant_sink::unpack_action_data_field(std::string_view key_name, const abi_serializer& abi, + std::string_view type, const bytes& data, + abi_traverse_context& ctx, bool short_path) { + // _binary_to_variant builds the variant before we touch the sink; if it throws + // pending_key was set by key() but never committed by emit_value, so we just + // clear it and the surviving frame is left as if neither call happened. + try { + key(key_name); + action_data_to_variant_context inner(abi, ctx, type); + inner.short_path = short_path; + emit_value(abi._binary_to_variant(type, data, inner)); + return true; + } catch(...) { + if (!stack_.empty()) { + auto& f = stack_.back(); + f.pending_key.clear(); + f.has_pending_key = false; + } + return false; + } +} + +// -- stream_sink ------------------------------------------------------------------------- + +void stream_sink::begin_object() { + w_.begin_object(); + frame_items_.emplace_back(0); +} + +void stream_sink::end_object() { + w_.end_object(); + FC_ASSERT(!frame_items_.empty(), "stream_sink: end_object without matching begin_object"); + frame_items_.pop_back(); + on_value_emitted(); +} + +void stream_sink::begin_array() { + w_.begin_array(); + frame_items_.emplace_back(0); +} + +void stream_sink::end_array() { + w_.end_array(); + FC_ASSERT(!frame_items_.empty(), "stream_sink: end_array without matching begin_array"); + frame_items_.pop_back(); + on_value_emitted(); +} + +void stream_sink::key(std::string_view k) { w_.key(k); } + +void stream_sink::value_string(std::string_view s) { w_.value_string(s); on_value_emitted(); } +// 64-bit integers route through the guarded fc::to_json_stream overloads (not the writer's +// raw value_* emitters) so magnitudes past json_integer_quote_magnitude emit quoted, exactly +// as variant_sink::value_int64's fc::variant(n) renders through fc::json::to_string. ABI +// built-in int64/uint64 fields already stream through the guarded generic in +// get_built_in_stream_unpacks; this pins the same quoting for the sink interface itself, so +// direct sink callers (the bytes size field today, any future caller) cannot diverge +// between the two sinks. +void stream_sink::value_int64(int64_t n) { fc::to_json_stream(n, w_); on_value_emitted(); } +void stream_sink::value_uint64(uint64_t n) { fc::to_json_stream(n, w_); on_value_emitted(); } +void stream_sink::value_bool(bool b) { w_.value_bool(b); on_value_emitted(); } +void stream_sink::value_null() { w_.value_null(); on_value_emitted(); } + +void stream_sink::value_hex(const char* data, size_t size) { + w_.value_hex(data, size); + on_value_emitted(); +} + +void stream_sink::raw_value(std::string_view raw_json) { + w_.raw_value(raw_json); + on_value_emitted(); +} + +void stream_sink::value_variant(const fc::variant& v) { + fc::to_json_stream(v, w_); + on_value_emitted(); +} + +bool stream_sink::frame_has_items() const noexcept { + return !frame_items_.empty() && frame_items_.back() > 0; +} + +void stream_sink::on_value_emitted() noexcept { + if (!frame_items_.empty()) ++frame_items_.back(); +} + +void stream_sink::unpack_built_in(std::string_view ftype, fc::datastream& stream, + bool is_array, bool is_optional, abi_traverse_context& ctx) { + const auto& dispatch = get_built_in_stream_unpacks(); + auto it = dispatch.find(ftype); + if( it != dispatch.end() ) { + it->second(stream, is_array, is_optional, ctx.get_yield_function(), w_); + on_value_emitted(); + return; + } + // Variant-side may carry user overrides via add_specialized_unpack_pack that the + // streaming dispatch does not mirror today. Fall back to the variant unpack + + // to_json_stream(variant, w) bridge so those overrides remain effective. + FC_ASSERT(abi_, "stream_sink::unpack_built_in requires the abi_serializer-bound constructor"); + auto bv = abi_->find_built_in(ftype); + FC_ASSERT(bv, "stream_sink::unpack_built_in: '{}' is not a built-in", std::string(ftype)); + auto v = bv->first(stream, is_array, is_optional, ctx.get_yield_function()); + value_variant(v); +} + +void stream_sink::unpack_protobuf(std::string_view ftype, fc::datastream& stream) { + // MessageToJsonString already produces JSON; splice it directly into the writer. + FC_ASSERT(abi_, "stream_sink::unpack_protobuf requires the abi_serializer-bound constructor"); + raw_value(abi_->pb_binary_to_json_string(ftype, stream)); +} + +void stream_sink::unpack_action_data(const abi_serializer& abi, std::string_view type, const bytes& data, + abi_traverse_context& ctx, bool short_path) { + action_data_to_variant_context inner(abi, ctx, type); + inner.short_path = short_path; + fc::datastream ds(data.data(), data.size()); + abi._binary_walk(type, ds, *this, inner); +} + +bool stream_sink::unpack_action_data_field(std::string_view key_name, const abi_serializer& abi, + std::string_view type, const bytes& data, + abi_traverse_context& ctx, bool short_path) { + // Snapshot writer + frame_items_ before the key emit; on throw we rewind both + // so the buffer / frame stack are byte-identical to the pre-call state. The + // walker may have opened nested object frames before throwing, so resizing + // frame_items_ back to its saved depth is required to keep it in lockstep + // with the json_writer's internal stack. + const auto writer_cp = w_.checkpoint(); + const size_t fi_save = frame_items_.size(); + const uint32_t items_save = frame_items_.empty() ? 0 : frame_items_.back(); + try { + w_.key(key_name); + action_data_to_variant_context inner(abi, ctx, type); + inner.short_path = short_path; + fc::datastream ds(data.data(), data.size()); + abi._binary_walk(type, ds, *this, inner); + on_value_emitted(); + return true; + } catch(...) { + w_.rewind(writer_cp); + frame_items_.resize(fi_save); + if (!frame_items_.empty()) frame_items_.back() = items_save; + return false; + } +} + +} // namespace sysio::chain::impl + +// -- _binary_walk et al ------------------------------------------------------------ + +namespace sysio::chain { + +template +void abi_serializer::_unpack_enum( const enum_def& e_def, fc::datastream& stream, + Sink& sink, impl::binary_to_variant_context& ctx )const +{ + auto btype = built_in_types.find(e_def.type); + SYS_ASSERT( btype != built_in_types.end(), invalid_type_inside_abi, + "Enum has unknown underlying type '{}'", impl::limit_size(e_def.type) ); + // Read the underlying integer through the variant-side functor. Both sinks emit the + // same shape (string name on hit, raw integer on miss) — the parity is structural. + auto int_var = btype->second.first(stream, false, false, ctx.get_yield_function()); + auto int_val = int_var.as_int64(); + for( const auto& ev : e_def.values ) { + if( ev.value == int_val ) { + sink.value_string(ev.name); + return; + } + } + // Unknown value -- emit as the underlying integer. Both sinks take by const-ref + // and copy internally; the int_var temporary lives until end of expression. + sink.value_variant(int_var); +} + +template +void abi_serializer::_binary_walk_struct_fields( + const map>::const_iterator& s_itr, + fc::datastream& stream, + Sink& sink, impl::binary_to_variant_context& ctx )const +{ + const auto& st = s_itr->second; + if( st.base != type_name() ) { + auto base_itr = structs.find(resolve_type(st.base)); + SYS_ASSERT( base_itr != structs.end(), invalid_type_inside_abi, + "Unknown base type {}", impl::limit_size(st.base) ); + _binary_walk_struct_fields(base_itr, stream, sink, ctx); + } + bool encountered_extension = false; + for( uint32_t i = 0; i < st.fields.size(); ++i ) { + const auto& field = st.fields[i]; + bool extension = field.type.ends_with("$"); + encountered_extension |= extension; + if( !stream.remaining() ) { + if( extension ) { + continue; + } + if( encountered_extension ) { + SYS_THROW( abi_exception, "Encountered field '{}' without binary extension designation while processing struct '{}'", + ctx.maybe_shorten(field.name), ctx.get_path_string() ); + } + SYS_THROW( unpack_exception, "Stream unexpectedly ended; unable to unpack field '{}' of struct '{}'", + ctx.maybe_shorten(field.name), ctx.get_path_string() ); + } + auto h1 = ctx.push_to_path( impl::field_path_item{ .parent_struct_itr = s_itr, .field_ordinal = i } ); + auto field_type = resolve_type( extension ? _remove_bin_extension(field.type) : field.type ); + sink.key(field.name); + if( ctx.is_logging() && field_type == "bytes" ) { + // Logging-bytes special case: variant path replaces the hex string with + // {size: N, hex|trimmed_hex: ...}. Read raw bytes here and emit the + // sub-object directly so the streaming sink doesn't need to retroactively + // rewrite a previously-emitted value. + std::vector data; + try { + fc::raw::unpack(stream, data); + } SYS_RETHROW_EXCEPTIONS( unpack_exception, "Unable to unpack 'bytes' for field '{}' of struct '{}'", + ctx.maybe_shorten(field.name), ctx.get_path_string() ) + sink.begin_object(); + sink.key("size"); + sink.value_uint64(data.size()); + if( data.size() > impl::hex_log_max_size ) { + sink.key("trimmed_hex"); + sink.value_hex(data.data(), impl::hex_log_max_size); + } else { + sink.key("hex"); + sink.value_hex(data.data(), data.size()); + } + sink.end_object(); + } else { + _binary_walk(field_type, stream, sink, ctx); + } + } +} + +template +void abi_serializer::_binary_walk( const std::string_view& type, fc::datastream& stream, + Sink& sink, impl::binary_to_variant_context& ctx )const +{ + auto h = ctx.enter_scope(); + auto rtype = resolve_type(type); + auto ftype = fundamental_type(rtype); + auto fixed_array_sz = is_szarray(rtype); + + auto walk_array = [&](fc::unsigned_int::base_uint sz) { + ctx.hint_array_type_if_in_array(); + sink.begin_array(); + auto h1 = ctx.push_to_path( impl::array_index_path_item{} ); + for( fc::unsigned_int::base_uint i = 0; i < sz; ++i ) { + ctx.set_array_index_of_path_back(i); + _binary_walk(ftype, stream, sink, ctx); + } + sink.end_array(); + }; + + if( fixed_array_sz ) { + walk_array(*fixed_array_sz); + return; + } + + if( auto btype = built_in_types.find(ftype); btype != built_in_types.end() ) { + try { + sink.unpack_built_in(ftype, stream, is_array(rtype), is_optional(rtype), ctx); + return; + } SYS_RETHROW_EXCEPTIONS( unpack_exception, "Unable to unpack {} type '{}' while processing '{}'", + is_array(rtype) ? "array of built-in" : is_optional(rtype) ? "optional of built-in" : "built-in", + impl::limit_size(ftype), ctx.get_path_string() ) + } + + if( is_protobuf_type(ftype) ) { + try { + sink.unpack_protobuf(ftype, stream); + return; + } SYS_RETHROW_EXCEPTIONS( unpack_exception, "Unable to unpack protobuf type '{}' while processing '{}'", + impl::limit_size(ftype), ctx.get_path_string() ) + } + + if( is_array(rtype) ) { + fc::unsigned_int size; + try { + fc::raw::unpack(stream, size); + } SYS_RETHROW_EXCEPTIONS( unpack_exception, "Unable to unpack size of array '{}'", ctx.get_path_string() ) + walk_array(size.value); + return; + } + + if( is_optional(rtype) ) { + char flag; + try { + fc::raw::unpack(stream, flag); + } SYS_RETHROW_EXCEPTIONS( unpack_exception, "Unable to unpack presence flag of optional '{}'", ctx.get_path_string() ) + if( flag ) { + _binary_walk(ftype, stream, sink, ctx); + } else { + sink.value_null(); + } + return; + } + + if( auto e_itr = enums.find(rtype); e_itr != enums.end() ) { + _unpack_enum(e_itr->second, stream, sink, ctx); + return; + } + + if( auto v_itr = variants.find(rtype); v_itr != variants.end() ) { + ctx.hint_variant_type_if_in_array( v_itr ); + fc::unsigned_int select; + try { + fc::raw::unpack(stream, select); + } SYS_RETHROW_EXCEPTIONS( unpack_exception, "Unable to unpack tag of variant '{}'", ctx.get_path_string() ) + SYS_ASSERT( (size_t)select < v_itr->second.types.size(), unpack_exception, + "Unpacked invalid tag ({}) for variant '{}'", select.value, ctx.get_path_string() ); + auto h1 = ctx.push_to_path( impl::variant_path_item{ .variant_itr = v_itr, .variant_ordinal = static_cast(select) } ); + sink.begin_array(); + sink.value_string(v_itr->second.types[select]); + _binary_walk(v_itr->second.types[select], stream, sink, ctx); + sink.end_array(); + return; + } + + auto s_itr = structs.find(rtype); + SYS_ASSERT( s_itr != structs.end(), invalid_type_inside_abi, "Unknown type {}", ctx.maybe_shorten(rtype) ); + ctx.hint_struct_type_if_in_array( s_itr ); + sink.begin_object(); + _binary_walk_struct_fields(s_itr, stream, sink, ctx); + const bool emitted_any = sink.frame_has_items(); + sink.end_object(); + SYS_ASSERT( emitted_any, unpack_exception, "Unable to unpack '{}' from stream", ctx.get_path_string() ); +} + +// Explicit instantiations for the two sinks defined in abi_sinks.hpp. Keeps the +// member-template definitions out of headers without sacrificing the link surface. +template void abi_serializer::_binary_walk( + const std::string_view&, fc::datastream&, impl::variant_sink&, + impl::binary_to_variant_context&) const; +template void abi_serializer::_binary_walk( + const std::string_view&, fc::datastream&, impl::stream_sink&, + impl::binary_to_variant_context&) const; + +template void abi_serializer::_binary_walk_struct_fields( + const map>::const_iterator&, + fc::datastream&, impl::variant_sink&, impl::binary_to_variant_context&) const; +template void abi_serializer::_binary_walk_struct_fields( + const map>::const_iterator&, + fc::datastream&, impl::stream_sink&, impl::binary_to_variant_context&) const; + +template void abi_serializer::_unpack_enum( + const enum_def&, fc::datastream&, impl::variant_sink&, + impl::binary_to_variant_context&) const; +template void abi_serializer::_unpack_enum( + const enum_def&, fc::datastream&, impl::stream_sink&, + impl::binary_to_variant_context&) const; + +} // namespace sysio::chain diff --git a/libraries/chain/chain_id_type.cpp b/libraries/chain/chain_id_type.cpp index 0ae1ce6708..90651597a4 100644 --- a/libraries/chain/chain_id_type.cpp +++ b/libraries/chain/chain_id_type.cpp @@ -1,6 +1,9 @@ #include #include +#include +#include + namespace sysio { namespace chain { void chain_id_type::reflector_init()const { @@ -19,4 +22,8 @@ namespace fc { from_variant( v, static_cast(cid) ); } + void to_json_stream(const sysio::chain::chain_id_type& cid, fc::json_writer& w) { + to_json_stream( static_cast(cid), w ); + } + } // fc diff --git a/libraries/chain/include/sysio/chain/abi_def.hpp b/libraries/chain/include/sysio/chain/abi_def.hpp index 8c15ead50d..42418f1bf6 100644 --- a/libraries/chain/include/sysio/chain/abi_def.hpp +++ b/libraries/chain/include/sysio/chain/abi_def.hpp @@ -176,6 +176,11 @@ datastream& operator >> (datastream& s, sysio::chain::may_not_exist& return s; } +template +void to_json_stream(const sysio::chain::may_not_exist& e, fc::json_writer& w) { + to_json_stream( e.value, w ); +} + template void to_variant(const sysio::chain::may_not_exist& e, fc::variant& v) { to_variant( e.value, v); diff --git a/libraries/chain/include/sysio/chain/abi_serializer.hpp b/libraries/chain/include/sysio/chain/abi_serializer.hpp index 5fd4791e8d..fb2645c6d4 100644 --- a/libraries/chain/include/sysio/chain/abi_serializer.hpp +++ b/libraries/chain/include/sysio/chain/abi_serializer.hpp @@ -1,5 +1,6 @@ #pragma once #include +#include #include #include #include @@ -8,6 +9,7 @@ #include #include #include +#include namespace google::protobuf { class DescriptorPool; @@ -32,6 +34,9 @@ namespace impl { struct binary_to_variant_context; struct variant_to_binary_context; struct action_data_to_variant_context; + + class variant_sink; + class stream_sink; } /** @@ -80,6 +85,21 @@ struct abi_serializer { fc::variant binary_to_variant( const std::string_view& type, fc::datastream& binary, const yield_function_t& yield, bool short_path = false )const; fc::variant binary_to_variant( const std::string_view& type, fc::datastream& binary, const fc::microseconds& max_action_data_serialization_time, bool short_path = false )const; + /// Streaming counterpart to `binary_to_variant`. Decodes the binary blob and + /// emits its JSON representation directly into `w` without constructing a + /// `fc::variant` tree. The output is byte-identical to + /// `fc::json::to_string(binary_to_variant(...))` for all built-in types and + /// any reflected user struct that has a corresponding `to_json_stream` overload + /// (which `FC_REFLECT` provides automatically). + void binary_to_json_stream( const std::string_view& type, const bytes& binary, fc::json_writer& w, + const yield_function_t& yield, bool short_path = false )const; + void binary_to_json_stream( const std::string_view& type, const bytes& binary, fc::json_writer& w, + const fc::microseconds& max_action_data_serialization_time, bool short_path = false )const; + void binary_to_json_stream( const std::string_view& type, fc::datastream& binary, fc::json_writer& w, + const yield_function_t& yield, bool short_path = false )const; + void binary_to_json_stream( const std::string_view& type, fc::datastream& binary, fc::json_writer& w, + const fc::microseconds& max_action_data_serialization_time, bool short_path = false )const; + bytes variant_to_binary( const std::string_view& type, const fc::variant& var, const fc::microseconds& max_action_data_serialization_time, bool short_path = false )const; bytes variant_to_binary( const std::string_view& type, const fc::variant& var, const yield_function_t& yield, bool short_path = false )const; void variant_to_binary( const std::string_view& type, const fc::variant& var, fc::datastream& ds, const fc::microseconds& max_action_data_serialization_time, bool short_path = false )const; @@ -95,6 +115,15 @@ struct abi_serializer { template static void to_log_variant( const T& o, fc::variant& vo, const Resolver& resolver, const fc::microseconds& max_action_data_serialization_time ); + /// Streaming counterpart to `to_variant`: emits the same JSON shape directly into + /// `w` without building an intermediate `fc::variant` tree. Reflected types route + /// through the same `impl::abi_to_variant` machinery, which is sink-templated -- the + /// only difference between this path and `to_variant` is the sink instance passed in. + template + static void to_json_stream( const T& o, fc::json_writer& w, const Resolver& resolver, const yield_function_t& yield ); + template + static void to_json_stream( const T& o, fc::json_writer& w, const Resolver& resolver, const fc::microseconds& max_action_data_serialization_time ); + template static void from_variant( const fc::variant& v, T& o, const Resolver& resolver, const yield_function_t& yield ); template @@ -120,8 +149,20 @@ struct abi_serializer { typedef std::function&, bool, bool, const abi_serializer::yield_function_t&)> unpack_function; typedef std::function&, bool, bool, const abi_serializer::yield_function_t&)> pack_function; + /// Registers a per-instance (unpack, pack) override for a built-in type on the + /// VARIANT dispatch only. The streaming dispatch (`binary_to_json_stream`) uses a + /// static table consulted first, so an override registered here for a type that + /// table covers is NOT reflected on the streaming path -- the two paths would emit + /// different shapes for that type. No in-tree callers exist today; if one appears, + /// the streaming dispatch needs a per-instance map with a static fallback first. void add_specialized_unpack_pack( const string& name, std::pair unpack_pack ); + /// Lookup helper used by the streaming walker sinks. Returns a pointer to the + /// (unpack, pack) entry registered for `type`, or `nullptr` if it is not a + /// built-in. Const view; the underlying map is shared with `binary_to_variant` + /// dispatch so any `add_specialized_unpack_pack` overrides are reflected. + const std::pair* find_built_in(std::string_view type) const noexcept; + static constexpr size_t max_recursion_depth = 32; // arbitrary depth to prevent infinite recursion // create standard yield function that checks for max_serialization_time and max_recursion_depth. @@ -167,12 +208,39 @@ struct abi_serializer { bool is_protobuf_type(const std::string_view& type) const; const google::protobuf::Descriptor* get_pb_descriptor(const std::string_view& type) const; fc::variant pb_binary_to_variant(const std::string_view& type, fc::datastream& stream) const; + /// Streaming protobuf bridge: decodes the protobuf blob and returns the JSON + /// string produced by `MessageToJsonString` (which `pb_binary_to_variant` + /// otherwise round-trips through `fc::json::from_string`). Used by + /// `stream_sink::unpack_protobuf` to splice via `raw_value` without building + /// an intermediate variant tree. + std::string pb_binary_to_json_string(const std::string_view& type, fc::datastream& stream) const; void pb_variant_to_binary(const std::string_view& type, const fc::variant& var, fc::datastream& ds) const; fc::variant _binary_to_variant( const std::string_view& type, const bytes& binary, impl::binary_to_variant_context& ctx )const; fc::variant _binary_to_variant( const std::string_view& type, fc::datastream& binary, impl::binary_to_variant_context& ctx )const; - void _binary_to_variant( const std::string_view& type, fc::datastream& stream, - fc::mutable_variant_object& obj, impl::binary_to_variant_context& ctx )const; + + /// Templated recursive walker that emits a value of `type` from `stream` into `sink`. + /// Mirrors the recursion shape of `_binary_to_variant`: fixed-array, built-in, + /// protobuf, dynamic array, optional, enum, variant, struct. Two explicit + /// instantiations (variant_sink, stream_sink) live in `abi_serializer.cpp`. + template + void _binary_walk( const std::string_view& type, fc::datastream& stream, + Sink& sink, impl::binary_to_variant_context& ctx )const; + + /// Walk struct fields without emitting `begin_object`/`end_object`. Recurses + /// into the base struct (if any) so its fields land in the same object frame + /// the caller already opened. + template + void _binary_walk_struct_fields( const map>::const_iterator& s_itr, + fc::datastream& stream, + Sink& sink, impl::binary_to_variant_context& ctx )const; + + /// Read an enum's underlying integer from `stream` and emit either the matching + /// member name as a string, or the raw integer for unknown values. Templated + /// over sink to avoid a `fc::variant` round-trip on the streaming path. + template + void _unpack_enum( const enum_def& e_def, fc::datastream& stream, + Sink& sink, impl::binary_to_variant_context& ctx )const; bytes _variant_to_binary( const std::string_view& type, const fc::variant& var, impl::variant_to_binary_context& ctx )const; void _variant_to_binary( const std::string_view& type, const fc::variant& var, @@ -186,6 +254,8 @@ struct abi_serializer { friend struct impl::abi_from_variant; friend struct impl::abi_to_variant; friend struct impl::abi_traverse_context_with_path; + friend class impl::variant_sink; + friend class impl::stream_sink; }; namespace impl { @@ -367,373 +437,421 @@ namespace impl { template using require_abi_t = std::enable_if_t(), int>; + /// Sink-parameterised emitter for ABI-aware reflected types. The same template + /// body drives both `abi_serializer::to_variant` (variant_sink) and + /// `abi_serializer::to_json_stream` (stream_sink) -- no logic is duplicated + /// across the two output paths. Each `add` overload emits one named field at + /// the current sink frame; nested objects/arrays open a sub-frame. struct abi_to_variant { - /** - * template which overloads add for types which are not relevant to ABI information - * and can be degraded to the normal ::to_variant(...) processing - */ - template = 1> - static void add( mutable_variant_object &mvo, const char* name, const M& v, const Resolver&, abi_traverse_context& ctx ) + // Non-ABI types: bottom out into the sink's generic emit (which is + // fc::variant(v) for variant_sink and fc::to_json_stream(v, w) for stream_sink). + template = 1> + static void add( Sink& sink, std::string_view name, const M& v, const Resolver&, abi_traverse_context& ctx ) { auto h = ctx.enter_scope(); - mvo(name,v); + sink.key(name); + sink.template emit(v); } - /** - * template which overloads add for types which contain ABI information in their trees - * for these types we create new ABI aware visitors - */ - template = 1> - static void add( mutable_variant_object &mvo, const char* name, const M& v, const Resolver& resolver, abi_traverse_context& ctx ); + // ABI types: walk through fc::reflector via abi_to_variant_visitor. + template = 1> + static void add( Sink& sink, std::string_view name, const M& v, const Resolver& resolver, abi_traverse_context& ctx ); - /** - * template which overloads add for vectors of types which contain ABI information in their trees - * for these members we call ::add in order to trigger further processing - */ - template = 1> - static void add( mutable_variant_object &mvo, const char* name, const vector& v, const Resolver& resolver, abi_traverse_context& ctx ) + // vector of ABI-bearing M. + template = 1> + static void add( Sink& sink, std::string_view name, const vector& v, const Resolver& resolver, abi_traverse_context& ctx ) { auto h = ctx.enter_scope(); - fc::variants array; - array.reserve(v.size()); + sink.key(name); + sink.begin_array(); + for( const auto& iter : v ) { + add_value( sink, iter, resolver, ctx ); + } + sink.end_array(); + } - for (const auto& iter: v) { - mutable_variant_object elem_mvo; - add(elem_mvo, "_", iter, resolver, ctx); - array.emplace_back(std::move(elem_mvo["_"])); + // deque of ABI-bearing M. + template = 1> + static void add( Sink& sink, std::string_view name, const deque& v, const Resolver& resolver, abi_traverse_context& ctx ) + { + auto h = ctx.enter_scope(); + sink.key(name); + sink.begin_array(); + for( const auto& iter : v ) { + add_value( sink, iter, resolver, ctx ); } - mvo(name, std::move(array)); + sink.end_array(); } - /** - * template which overloads add for deques of types which contain ABI information in their trees - * for these members we call ::add in order to trigger further processing - */ - template = 1> - static void add( mutable_variant_object &mvo, const char* name, const deque& v, const Resolver& resolver, abi_traverse_context& ctx ) + // shared_ptr of ABI-bearing M. Null pointer is a no-op (matches the + // pre-Sink behaviour of skipping the field entirely). + template = 1> + static void add( Sink& sink, std::string_view name, const std::shared_ptr& v, const Resolver& resolver, abi_traverse_context& ctx ) + { + auto h = ctx.enter_scope(); + if( !v ) return; + sink.key(name); + add_value( sink, *v, resolver, ctx ); + } + + // Element-position emitter used by container helpers above and by the + // top-level `to_variant` / `to_json_stream` entry points. Emits the + // value at the current (array or top-level) sink position with no leading + // key. Specialised for each container shape recognised by `add` so that + // recursion through nested containers does not bottom-out into the + // reflector path with a container type as `M`. + template = 1> + static void add_value( Sink& sink, const M& v, const Resolver&, abi_traverse_context& ctx ) { auto h = ctx.enter_scope(); - deque array; + sink.template emit(v); + } + template = 1> + static void add_value( Sink& sink, const M& v, const Resolver& resolver, abi_traverse_context& ctx ); - for (const auto& iter: v) { - mutable_variant_object elem_mvo; - add(elem_mvo, "_", iter, resolver, ctx); - array.emplace_back(std::move(elem_mvo["_"])); + // Container-shape specialisations: walk through to the element add_value. + template = 1> + static void add_value( Sink& sink, const vector& v, const Resolver& resolver, abi_traverse_context& ctx ) + { + auto h = ctx.enter_scope(); + sink.begin_array(); + for( const auto& iter : v ) { + add_value( sink, iter, resolver, ctx ); } - mvo(name, std::move(array)); + sink.end_array(); } - /** - * template which overloads add for shared_ptr of types which contain ABI information in their trees - * for these members we call ::add in order to trigger further processing - */ - template = 1> - static void add( mutable_variant_object &mvo, const char* name, const std::shared_ptr& v, const Resolver& resolver, abi_traverse_context& ctx ) + template = 1> + static void add_value( Sink& sink, const deque& v, const Resolver& resolver, abi_traverse_context& ctx ) { auto h = ctx.enter_scope(); - if( !v ) return; - mutable_variant_object obj_mvo; - add(obj_mvo, "_", *v, resolver, ctx); - mvo(name, std::move(obj_mvo["_"])); + sink.begin_array(); + for( const auto& iter : v ) { + add_value( sink, iter, resolver, ctx ); + } + sink.end_array(); + } + template = 1> + static void add_value( Sink& sink, const std::shared_ptr& v, const Resolver& resolver, abi_traverse_context& ctx ) + { + auto h = ctx.enter_scope(); + if( !v ) { sink.value_null(); return; } + add_value( sink, *v, resolver, ctx ); + } + template + static void add_value( Sink& sink, const std::variant& v, const Resolver& resolver, abi_traverse_context& ctx ) + { + auto h = ctx.enter_scope(); + add_static_variant adder( sink, resolver, ctx ); + std::visit( adder, v ); } - template + // std::variant<...>: dispatch the active alternative through the visitor. + template struct add_static_variant { - mutable_variant_object& obj_mvo; + Sink& sink; const Resolver& resolver; abi_traverse_context& ctx; - - add_static_variant( mutable_variant_object& o, const Resolver& r, abi_traverse_context& ctx ) - :obj_mvo(o), resolver(r), ctx(ctx) {} + add_static_variant( Sink& s, const Resolver& r, abi_traverse_context& c ) : sink(s), resolver(r), ctx(c) {} typedef void result_type; - template void operator()( T& v )const + template void operator()( T& v ) const { - add(obj_mvo, "_", v, resolver, ctx); + add_value( sink, v, resolver, ctx ); } }; - template - static void add( mutable_variant_object &mvo, const char* name, const std::variant& v, const Resolver& resolver, abi_traverse_context& ctx ) + template + static void add( Sink& sink, std::string_view name, const std::variant& v, const Resolver& resolver, abi_traverse_context& ctx ) { auto h = ctx.enter_scope(); - mutable_variant_object obj_mvo; - add_static_variant adder(obj_mvo, resolver, ctx); - std::visit(adder, v); - mvo(name, std::move(obj_mvo["_"])); + sink.key(name); + add_static_variant adder( sink, resolver, ctx ); + std::visit( adder, v ); } - template - static bool add_special_logging( mutable_variant_object& mvo, const char* name, const action& act, const Resolver& resolver, abi_traverse_context& ctx ) { + // Logging-mode side-effect for sysio::setcode -- inject `code_hash` next to + // the existing `data`/`hex_data` fields. Returns false either way; callers + // still emit `data` afterwards. No-op outside logging mode. + template + static bool add_special_logging( Sink& sink, const action& act, const Resolver& /*resolver*/, abi_traverse_context& ctx ) { if( !ctx.is_logging() ) return false; - try { - if( act.account == sysio::chain::config::system_account_name && act.name == "setcode"_n ) { auto setcode_act = act.data_as(); if( setcode_act.code.size() > 0 ) { fc::sha256 code_hash = fc::sha256::hash(setcode_act.code.data(), (uint32_t) setcode_act.code.size()); - mvo("code_hash", code_hash); + sink.key("code_hash"); + sink.template emit(code_hash); } return false; // still want the hex data included } - } catch(...) {} // return false - return false; } - /** - * overload of to_variant_object for actions - * - * This matches the FC_REFLECT for this type, but this is provided to extract the contents of act.data - * @tparam Resolver - * @param act - * @param resolver - * @return - */ - template - static void add( mutable_variant_object &out, const char* name, const action& act, const Resolver& resolver, abi_traverse_context& ctx ) + // Emit `bytes` either as raw hex (non-logging) or as the + // `{size, hex|trimmed_hex}` log shape. Used for `data` and `hex_data` + // fields on `action`. + template + static void emit_action_bytes_field( Sink& sink, std::string_view name, const bytes& data, abi_traverse_context& ctx ) + { + sink.key(name); + if( !ctx.is_logging() ) { + sink.template emit(data); + return; + } + sink.begin_object(); + sink.key("size"); + sink.value_uint64(data.size()); + if( data.size() > impl::hex_log_max_size ) { + sink.key("trimmed_hex"); + sink.template emit( bytes(&data[0], &data[0] + impl::hex_log_max_size) ); + } else { + sink.key("hex"); + sink.template emit(data); + } + sink.end_object(); + } + + /// Action: matches the FC_REFLECT shape but routes `data` through the + /// resolver-located ABI to decode action arguments inline. The body lives + /// on `add_value` so the top-level `to_variant` entry hits the + /// special handling rather than the reflector-based generic path. + template + static void add_value( Sink& sink, const action& act, const Resolver& resolver, abi_traverse_context& ctx ) { static_assert(fc::reflector::total_member_count == 4); auto h = ctx.enter_scope(); - mutable_variant_object mvo; - mvo("account", act.account); - mvo("name", act.name); - mvo("authorization", act.authorization); + sink.begin_object(); + add( sink, "account", act.account, resolver, ctx ); + add( sink, "name", act.name, resolver, ctx ); + add( sink, "authorization", act.authorization, resolver, ctx ); - if( add_special_logging(mvo, name, act, resolver, ctx) ) { - out(name, std::move(mvo)); + if( add_special_logging( sink, act, resolver, ctx ) ) { + sink.end_object(); return; } - auto set_hex_data = [&](mutable_variant_object& mvo, const char* name, const bytes& data) { - if( !ctx.is_logging() ) { - mvo(name, data); - } else { - fc::mutable_variant_object sub_obj; - sub_obj( "size", data.size() ); - if( data.size() > impl::hex_log_max_size ) { - sub_obj( "trimmed_hex", std::vector(&data[0], &data[0] + impl::hex_log_max_size) ); - } else { - sub_obj( "hex", data ); - } - mvo(name, std::move(sub_obj)); - } - }; - + // unpack_action_data_field is atomic: on failure the sink is rolled back + // so the hex fallback emits cleanly under the same key with no duplicate + // and no orphan inner frame. + bool data_emitted = false; try { auto abi_optional = resolver(act.account); if (abi_optional) { const abi_serializer& abi = *abi_optional; auto type = abi.get_action_type(act.name); if (!type.empty()) { - try { - action_data_to_variant_context _ctx(abi, ctx, type); - mvo( "data", abi._binary_to_variant( type, act.data, _ctx )); - } catch(...) { - // any failure to serialize data, then leave as not serialized - set_hex_data(mvo, "data", act.data); - } - } else { - set_hex_data(mvo, "data", act.data); + data_emitted = sink.unpack_action_data_field("data", abi, type, act.data, ctx, /*short_path*/ false); } - } else { - set_hex_data(mvo, "data", act.data); } - } catch(...) { - set_hex_data(mvo, "data", act.data); + } catch(...) { /* resolver lookup failure -- fall back to hex below */ } + if( !data_emitted ) { + emit_action_bytes_field( sink, "data", act.data, ctx ); } - set_hex_data(mvo, "hex_data", act.data); - out(name, std::move(mvo)); + emit_action_bytes_field( sink, "hex_data", act.data, ctx ); + sink.end_object(); } - /** - * overload of to_variant_object for action_trace - * - * This matches the FC_REFLECT for this type, but this is provided to extract the contents of action_trace.return_value - * @tparam Resolver - * @param action_trace - * @param resolver - * @return - */ - template - static void add( mutable_variant_object& out, const char* name, const action_trace& act_trace, const Resolver& resolver, abi_traverse_context& ctx ) + template + static void add( Sink& sink, std::string_view name, const action& act, const Resolver& resolver, abi_traverse_context& ctx ) + { + sink.key(name); + add_value( sink, act, resolver, ctx ); + } + + /// action_trace: matches FC_REFLECT shape; routes `return_value` through the + /// resolver-located ABI when an action_result type is registered. + template + static void add_value( Sink& sink, const action_trace& act_trace, const Resolver& resolver, abi_traverse_context& ctx ) { static_assert(fc::reflector::total_member_count == 19); auto h = ctx.enter_scope(); - mutable_variant_object mvo; - - mvo("action_ordinal", act_trace.action_ordinal); - mvo("creator_action_ordinal", act_trace.creator_action_ordinal); - mvo("closest_unnotified_ancestor_action_ordinal", act_trace.closest_unnotified_ancestor_action_ordinal); - mvo("receipt", act_trace.receipt); - mvo("receiver", act_trace.receiver); - add(mvo, "act", act_trace.act, resolver, ctx); - mvo("context_free", act_trace.context_free); - mvo("elapsed", act_trace.elapsed); - mvo("cpu_usage_us", act_trace.cpu_usage_us); - mvo("net_usage", act_trace.net_usage); - mvo("console", act_trace.console); - mvo("trx_id", act_trace.trx_id); - mvo("block_num", act_trace.block_num); - mvo("block_time", act_trace.block_time); - mvo("producer_block_id", act_trace.producer_block_id); - mvo("account_ram_deltas", act_trace.account_ram_deltas); - mvo("except", act_trace.except); - mvo("error_code", act_trace.error_code); - - mvo("return_value_hex_data", act_trace.return_value); - auto act = act_trace.act; + sink.begin_object(); + add( sink, "action_ordinal", act_trace.action_ordinal, resolver, ctx ); + add( sink, "creator_action_ordinal", act_trace.creator_action_ordinal, resolver, ctx ); + add( sink, "closest_unnotified_ancestor_action_ordinal", act_trace.closest_unnotified_ancestor_action_ordinal, resolver, ctx ); + add( sink, "receipt", act_trace.receipt, resolver, ctx ); + add( sink, "receiver", act_trace.receiver, resolver, ctx ); + add( sink, "act", act_trace.act, resolver, ctx ); + add( sink, "context_free", act_trace.context_free, resolver, ctx ); + add( sink, "elapsed", act_trace.elapsed, resolver, ctx ); + add( sink, "cpu_usage_us", act_trace.cpu_usage_us, resolver, ctx ); + add( sink, "net_usage", act_trace.net_usage, resolver, ctx ); + add( sink, "console", act_trace.console, resolver, ctx ); + add( sink, "trx_id", act_trace.trx_id, resolver, ctx ); + add( sink, "block_num", act_trace.block_num, resolver, ctx ); + add( sink, "block_time", act_trace.block_time, resolver, ctx ); + add( sink, "producer_block_id", act_trace.producer_block_id, resolver, ctx ); + add( sink, "account_ram_deltas", act_trace.account_ram_deltas, resolver, ctx ); + add( sink, "except", act_trace.except, resolver, ctx ); + add( sink, "error_code", act_trace.error_code, resolver, ctx ); + add( sink, "return_value_hex_data", act_trace.return_value, resolver, ctx ); + + // Atomic key + unpack: on decode failure the field is silently omitted + // (matches existing variant-path behavior), with no orphan frame or + // dangling key left in the writer. try { - auto abi_optional = resolver(act.account); + auto abi_optional = resolver(act_trace.act.account); if (abi_optional) { const abi_serializer& abi = *abi_optional; - auto type = abi.get_action_result_type(act.name); + auto type = abi.get_action_result_type(act_trace.act.name); if (!type.empty()) { - action_data_to_variant_context _ctx(abi, ctx, type); - mvo( "return_value_data", abi._binary_to_variant( type, act_trace.return_value, _ctx )); + sink.unpack_action_data_field("return_value_data", abi, type, act_trace.return_value, ctx, /*short_path*/ false); } } - } catch(...) {} - out(name, std::move(mvo)); + } catch(...) { /* resolver lookup failure -- omit field */ } + sink.end_object(); } - /** - * overload of to_variant_object for packed_transaction - * - * This matches the FC_REFLECT for this type, but this is provided to allow extracting the contents of ptrx.transaction - * @tparam Resolver - * @param act - * @param resolver - * @return - */ - template - static void add( mutable_variant_object &out, const char* name, const packed_transaction& ptrx, const Resolver& resolver, abi_traverse_context& ctx ) + template + static void add( Sink& sink, std::string_view name, const action_trace& act_trace, const Resolver& resolver, abi_traverse_context& ctx ) + { + sink.key(name); + add_value( sink, act_trace, resolver, ctx ); + } + + /// packed_transaction: extract the inner transaction so consumers see action + /// data fully decoded rather than a packed-bytes blob. + template + static void add_value( Sink& sink, const packed_transaction& ptrx, const Resolver& resolver, abi_traverse_context& ctx ) { static_assert(fc::reflector::total_member_count == 4); auto h = ctx.enter_scope(); - mutable_variant_object mvo; + sink.begin_object(); auto trx = ptrx.get_transaction(); - mvo("id", trx.id()); - mvo("signatures", ptrx.get_signatures()); - mvo("compression", ptrx.get_compression()); - mvo("packed_context_free_data", ptrx.get_packed_context_free_data()); - mvo("context_free_data", ptrx.get_context_free_data()); + add( sink, "id", trx.id(), resolver, ctx ); + add( sink, "signatures", ptrx.get_signatures(), resolver, ctx ); + add( sink, "compression", ptrx.get_compression(), resolver, ctx ); + add( sink, "packed_context_free_data", ptrx.get_packed_context_free_data(), resolver, ctx ); + add( sink, "context_free_data", ptrx.get_context_free_data(), resolver, ctx ); if( !ctx.is_logging() ) - mvo("packed_trx", ptrx.get_packed_transaction()); - add(mvo, "transaction", trx, resolver, ctx); + add( sink, "packed_trx", ptrx.get_packed_transaction(), resolver, ctx ); + add( sink, "transaction", trx, resolver, ctx ); + sink.end_object(); + } - out(name, std::move(mvo)); + template + static void add( Sink& sink, std::string_view name, const packed_transaction& ptrx, const Resolver& resolver, abi_traverse_context& ctx ) + { + sink.key(name); + add_value( sink, ptrx, resolver, ctx ); } - /** - * overload of to_variant_object for transaction - * - * This matches the FC_REFLECT for this type, but this is provided to allow extracting the contents of trx.transaction_extensions - */ - template - static void add( mutable_variant_object &out, const char* name, const transaction& trx, const Resolver& resolver, abi_traverse_context& ctx ) + /// transaction: per FC_REFLECT but recurses into context_free_actions and + /// actions through the ABI-aware action overload above. + template + static void add_value( Sink& sink, const transaction& trx, const Resolver& resolver, abi_traverse_context& ctx ) { static_assert(fc::reflector::total_member_count == 9); auto h = ctx.enter_scope(); - mutable_variant_object mvo; - mvo("expiration", trx.expiration); - mvo("ref_block_num", trx.ref_block_num); - mvo("ref_block_prefix", trx.ref_block_prefix); - mvo("max_net_usage_words", trx.max_net_usage_words); - mvo("max_cpu_usage_ms", trx.max_cpu_usage_ms); - mvo("delay_sec", trx.delay_sec); - add(mvo, "context_free_actions", trx.context_free_actions, resolver, ctx); - add(mvo, "actions", trx.actions, resolver, ctx); - + sink.begin_object(); + add( sink, "expiration", trx.expiration, resolver, ctx ); + add( sink, "ref_block_num", trx.ref_block_num, resolver, ctx ); + add( sink, "ref_block_prefix", trx.ref_block_prefix, resolver, ctx ); + add( sink, "max_net_usage_words", trx.max_net_usage_words, resolver, ctx ); + add( sink, "max_cpu_usage_ms", trx.max_cpu_usage_ms, resolver, ctx ); + add( sink, "delay_sec", trx.delay_sec, resolver, ctx ); + add( sink, "context_free_actions", trx.context_free_actions, resolver, ctx ); + add( sink, "actions", trx.actions, resolver, ctx ); // No transaction extensions are currently supported. - // When extensions are added in the future, deserialize them here. + sink.end_object(); + } - out(name, std::move(mvo)); + template + static void add( Sink& sink, std::string_view name, const transaction& trx, const Resolver& resolver, abi_traverse_context& ctx ) + { + sink.key(name); + add_value( sink, trx, resolver, ctx ); } - /** - * overload of to_variant_object for signed_block - * - * This matches the FC_REFLECT for this type, but this is provided to allow extracting the contents of - * block.header_extensions and block.block_extensions - */ - template - static void add( mutable_variant_object &out, const char* name, const signed_block& block, const Resolver& resolver, abi_traverse_context& ctx ) + /// Emit the inline (no begin/end_object) field set for `signed_block`. + /// Exposed so that callers with extra wrapper fields (eg `convert_block`'s + /// `id` / `block_num` / `ref_block_prefix`) can interleave them with the + /// reflected-block fields inside a single JSON object. + template + static void emit_signed_block_body( Sink& sink, const signed_block& block, const Resolver& resolver, abi_traverse_context& ctx ) { static_assert(fc::reflector::total_member_count == 13); - auto h = ctx.enter_scope(); - mutable_variant_object mvo; - mvo("timestamp", block.timestamp); - mvo("producer", block.producer); - mvo("previous", block.previous); - mvo("transaction_mroot", block.transaction_mroot); - mvo("finality_mroot", block.finality_mroot); - mvo("qc_claim", block.qc_claim); - mvo("new_finalizer_policy_diff", block.new_finalizer_policy_diff); - mvo("new_proposer_policy_diff", block.new_proposer_policy_diff); - - // process contents of block.header_extensions + add( sink, "timestamp", block.timestamp, resolver, ctx ); + add( sink, "producer", block.producer, resolver, ctx ); + add( sink, "previous", block.previous, resolver, ctx ); + add( sink, "transaction_mroot", block.transaction_mroot, resolver, ctx ); + add( sink, "finality_mroot", block.finality_mroot, resolver, ctx ); + add( sink, "qc_claim", block.qc_claim, resolver, ctx ); + add( sink, "new_finalizer_policy_diff", block.new_finalizer_policy_diff, resolver, ctx ); + add( sink, "new_proposer_policy_diff", block.new_proposer_policy_diff, resolver, ctx ); + flat_multimap header_exts = block.validate_and_extract_header_extensions(); if (auto it = header_exts.find(protocol_feature_activation::extension_id()); it != header_exts.end()) { const auto& new_protocol_features = std::get(it->second).protocol_features; - fc::variants pf_array; - pf_array.reserve(new_protocol_features.size()); + sink.key("new_protocol_features"); + sink.begin_array(); for (auto feature : new_protocol_features) { - mutable_variant_object feature_mvo; - add(feature_mvo, "feature_digest", feature, resolver, ctx); - pf_array.push_back(std::move(feature_mvo)); + sink.begin_object(); + add( sink, "feature_digest", feature, resolver, ctx ); + sink.end_object(); } - mvo("new_protocol_features", pf_array); + sink.end_array(); } - mvo("producer_signatures", block.producer_signatures); - add(mvo, "transactions", block.transactions, resolver, ctx); + add( sink, "producer_signatures", block.producer_signatures, resolver, ctx ); + add( sink, "transactions", block.transactions, resolver, ctx ); if (block.qc) { - mvo("qc", *block.qc); + add( sink, "qc", *block.qc, resolver, ctx ); } + } + + template + static void add_value( Sink& sink, const signed_block& block, const Resolver& resolver, abi_traverse_context& ctx ) + { + auto h = ctx.enter_scope(); + sink.begin_object(); + emit_signed_block_body( sink, block, resolver, ctx ); + sink.end_object(); + } - out(name, std::move(mvo)); + template + static void add( Sink& sink, std::string_view name, const signed_block& block, const Resolver& resolver, abi_traverse_context& ctx ) + { + sink.key(name); + add_value( sink, block, resolver, ctx ); } }; /** - * Reflection visitor that uses a resolver to resolve ABIs for nested types - * this will degrade to the common fc::to_variant as soon as the type no longer contains - * ABI related info + * Reflection visitor that drives `abi_to_variant::add` over each member of T, + * resolving ABIs for nested ABI-bearing types via the supplied Resolver. Now + * sink-templated so the same visitor walks reflected fields for both the + * variant-building and stream-emitting paths. * - * @tparam Resolver - callable with the signature (const name& code_account) -> std::optional + * @tparam Sink - emit target (variant_sink or stream_sink) + * @tparam T - the reflected type whose members are being visited + * @tparam Resolver - callable with signature (const name&) -> std::optional */ - template + template class abi_to_variant_visitor { public: - abi_to_variant_visitor( mutable_variant_object& _mvo, const T& _val, const Resolver& _resolver, abi_traverse_context& _ctx ) - :_vo(_mvo) - ,_val(_val) - ,_resolver(_resolver) - ,_ctx(_ctx) + abi_to_variant_visitor( Sink& s, const T& v, const Resolver& r, abi_traverse_context& c ) + : _sink(s) + , _val(v) + , _resolver(r) + , _ctx(c) {} - /** - * Visit a single member and add it to the variant object - * @tparam Member - the member to visit - * @tparam Class - the class we are traversing - * @tparam member - pointer to the member - * @param name - the name of the member - */ - template + template void operator()( const char* name )const { - abi_to_variant::add( _vo, name, (_val.*member), _resolver, _ctx ); + abi_to_variant::add( _sink, name, (_val.*member), _resolver, _ctx ); } private: - mutable_variant_object& _vo; - const T& _val; - const Resolver& _resolver; + Sink& _sink; + const T& _val; + const Resolver& _resolver; abi_traverse_context& _ctx; }; @@ -945,13 +1063,23 @@ namespace impl { abi_traverse_context& _ctx; }; - template> - void abi_to_variant::add( mutable_variant_object &mvo, const char* name, const M& v, const Resolver& resolver, abi_traverse_context& ctx ) + template> + void abi_to_variant::add( Sink& sink, std::string_view name, const M& v, const Resolver& resolver, abi_traverse_context& ctx ) + { + auto h = ctx.enter_scope(); + sink.key(name); + sink.begin_object(); + fc::reflector::visit( impl::abi_to_variant_visitor(sink, v, resolver, ctx) ); + sink.end_object(); + } + + template> + void abi_to_variant::add_value( Sink& sink, const M& v, const Resolver& resolver, abi_traverse_context& ctx ) { auto h = ctx.enter_scope(); - mutable_variant_object member_mvo; - fc::reflector::visit( impl::abi_to_variant_visitor( member_mvo, v, resolver, ctx) ); - mvo(name, std::move(member_mvo)); + sink.begin_object(); + fc::reflector::visit( impl::abi_to_variant_visitor(sink, v, resolver, ctx) ); + sink.end_object(); } template> @@ -965,36 +1093,50 @@ namespace impl { template void abi_serializer::to_variant( const T& o, fc::variant& vo, const Resolver& resolver, const yield_function_t& yield ) try { - mutable_variant_object mvo; + impl::variant_sink sink; impl::abi_traverse_context ctx( yield, fc::microseconds{} ); - impl::abi_to_variant::add(mvo, "_", o, resolver, ctx); - vo = std::move(mvo["_"]); + impl::abi_to_variant::add_value(sink, o, resolver, ctx); + vo = std::move(sink).take_result(); } FC_RETHROW_EXCEPTIONS(error, "Failed to serialize: {}", boost::core::demangle( typeid(o).name() )) template void abi_serializer::to_variant( const T& o, fc::variant& vo, const Resolver& resolver, const fc::microseconds& max_action_data_serialization_time ) try { - mutable_variant_object mvo; + impl::variant_sink sink; impl::abi_traverse_context ctx( create_depth_yield_function(), max_action_data_serialization_time ); - impl::abi_to_variant::add(mvo, "_", o, resolver, ctx); - vo = std::move(mvo["_"]); + impl::abi_to_variant::add_value(sink, o, resolver, ctx); + vo = std::move(sink).take_result(); } FC_RETHROW_EXCEPTIONS(error, "Failed to serialize: {}", boost::core::demangle( typeid(o).name() )) template void abi_serializer::to_log_variant( const T& o, fc::variant& vo, const Resolver& resolver, const yield_function_t& yield ) try { - mutable_variant_object mvo; - impl::abi_traverse_context ctx( yield, fc::microseconds{} ); - ctx.logging(); - impl::abi_to_variant::add(mvo, "_", o, resolver, ctx); - vo = std::move(mvo["_"]); + impl::variant_sink sink; + impl::abi_traverse_context ctx( yield, fc::microseconds{} ); + ctx.logging(); + impl::abi_to_variant::add_value(sink, o, resolver, ctx); + vo = std::move(sink).take_result(); } FC_RETHROW_EXCEPTIONS(error, "Failed to serialize: {}", boost::core::demangle( typeid(o).name() )) template void abi_serializer::to_log_variant( const T& o, fc::variant& vo, const Resolver& resolver, const fc::microseconds& max_action_data_serialization_time ) try { - mutable_variant_object mvo; + impl::variant_sink sink; impl::abi_traverse_context ctx( create_depth_yield_function(), max_action_data_serialization_time ); ctx.logging(); - impl::abi_to_variant::add(mvo, "_", o, resolver, ctx); - vo = std::move(mvo["_"]); + impl::abi_to_variant::add_value(sink, o, resolver, ctx); + vo = std::move(sink).take_result(); +} FC_RETHROW_EXCEPTIONS(error, "Failed to serialize: {}", boost::core::demangle( typeid(o).name() )) + +template +void abi_serializer::to_json_stream( const T& o, fc::json_writer& w, const Resolver& resolver, const yield_function_t& yield ) try { + impl::stream_sink sink(w); + impl::abi_traverse_context ctx( yield, fc::microseconds{} ); + impl::abi_to_variant::add_value(sink, o, resolver, ctx); +} FC_RETHROW_EXCEPTIONS(error, "Failed to serialize: {}", boost::core::demangle( typeid(o).name() )) + +template +void abi_serializer::to_json_stream( const T& o, fc::json_writer& w, const Resolver& resolver, const fc::microseconds& max_action_data_serialization_time ) try { + impl::stream_sink sink(w); + impl::abi_traverse_context ctx( create_depth_yield_function(), max_action_data_serialization_time ); + impl::abi_to_variant::add_value(sink, o, resolver, ctx); } FC_RETHROW_EXCEPTIONS(error, "Failed to serialize: {}", boost::core::demangle( typeid(o).name() )) template diff --git a/libraries/chain/include/sysio/chain/abi_sinks.hpp b/libraries/chain/include/sysio/chain/abi_sinks.hpp new file mode 100644 index 0000000000..014fdcea91 --- /dev/null +++ b/libraries/chain/include/sysio/chain/abi_sinks.hpp @@ -0,0 +1,224 @@ +#pragma once + +// Self-contained sink definitions consumed by abi_serializer's binary_walk and +// abi_to_variant templates. Deliberately forward-declares `abi_serializer` and +// `abi_traverse_context` instead of including `abi_serializer.hpp`, since +// abi_serializer.hpp includes this header before defining the templates that +// use Sink. Method bodies in abi_serializer.cpp see the full definitions. + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace sysio::chain { + struct abi_serializer; + namespace impl { + struct abi_traverse_context; + } +} + +namespace sysio::chain::impl { + +/** + * Sink consumed by `abi_serializer::_binary_walk` while it walks an ABI-typed + * binary blob. Two implementations exist in lock-step: + * + * - `variant_sink`: assembles an `fc::variant` tree. Drives the legacy + * `binary_to_variant` API after step 3 of the streaming migration. + * - `stream_sink`: emits JSON tokens directly into an `fc::json_writer` (no + * intermediate variant allocation). Drives the new `binary_to_json_stream` API. + * + * Both sinks expose the same write-side surface (`begin_object`, `value_string`, ...). + * The walker calls `unpack_built_in` / `unpack_protobuf` to delegate the type-specific + * read+emit step; each sink owns its own per-type unpack table. + */ + +/** + * Builds an `fc::variant` tree token-by-token. Object frames accumulate fields into + * a `mutable_variant_object`; array frames accumulate elements into an `fc::variants`. + * When the outermost frame closes (or a single scalar value is emitted at top level), + * `take_result()` yields the assembled variant. + */ +class variant_sink { +public: + /// Construct without an abi_serializer reference -- used by `abi_to_emit` + /// callers (to_variant template family) that don't need the binary unpack helpers. + variant_sink() noexcept = default; + + /// Construct bound to an abi_serializer -- used by `_binary_walk` callers + /// that need `unpack_built_in` and `unpack_protobuf` to dispatch built-in types + /// and protobuf decoding respectively. + explicit variant_sink(const abi_serializer& abi) noexcept : abi_(&abi) {} + + void begin_object(); + void end_object(); + void begin_array(); + void end_array(); + void key(std::string_view k); + + void value_string(std::string_view s); + void value_int64(int64_t n); + void value_uint64(uint64_t n); + void value_int32(int32_t n) { value_int64(n); } + void value_uint32(uint32_t n) { value_uint64(n); } + void value_int16(int16_t n) { value_int64(n); } + void value_uint16(uint16_t n) { value_uint64(n); } + void value_int8(int8_t n) { value_int64(n); } + void value_uint8(uint8_t n) { value_uint64(n); } + void value_bool(bool b); + void value_null(); + void value_hex(const char* data, size_t size); + void raw_value(std::string_view raw_json); + + /// Inject an already-built `fc::variant` at the current value position. Used by + /// `unpack_built_in` and `unpack_protobuf` to avoid re-tokenising values that the + /// existing variant-side unpack functions produce as a single `fc::variant`. + /// By const-ref to match `stream_sink::value_variant`; copies before emit so the + /// sink owns its frame entry. + void value_variant(const fc::variant& v) { emit_value(v); } + + /// Generic field emit used by `abi_to_emit` for non-ABI types: builds a + /// `fc::variant(v)` and emits it at the current value position. Symmetric with + /// `stream_sink::emit` which calls `fc::to_json_stream(v, w)` instead. + template + void emit(const T& v) { emit_value(fc::variant(v)); } + + /// ABI-aware action data emit: decode the binary payload to an `fc::variant` + /// via the per-account abi and inject it at the current value position. This + /// mirrors `stream_sink::unpack_action_data` which calls `binary_to_json_stream` + /// directly into the writer. + void unpack_action_data(const abi_serializer& abi, std::string_view type, const bytes& data, + abi_traverse_context& ctx, bool short_path); + + /// Atomic key + unpack_action_data. Returns true on success. On unpack failure, + /// the sink is rolled back to the state before the call -- no partial key, + /// no orphan inner frame -- so the caller can emit a hex fallback under the + /// same key without producing duplicates or malformed JSON. + bool unpack_action_data_field(std::string_view key_name, const abi_serializer& abi, + std::string_view type, const bytes& data, + abi_traverse_context& ctx, bool short_path); + + /// True if at least one item has been added to the current frame. Used by the walker + /// to enforce the legacy "Unable to unpack '...' from stream" guard on empty structs. + bool frame_has_items() const noexcept; + + void unpack_built_in(std::string_view ftype, fc::datastream& stream, + bool is_array, bool is_optional, abi_traverse_context& ctx); + void unpack_protobuf(std::string_view ftype, fc::datastream& stream); + + fc::variant take_result() && { return std::move(result_); } + +private: + void emit_value(fc::variant v); + + const abi_serializer* abi_ = nullptr; + + enum class frame_kind : uint8_t { object, array }; + struct frame { + frame_kind kind; + fc::mutable_variant_object mvo; + fc::variants arr; + std::string pending_key; + bool has_pending_key = false; + uint32_t item_count = 0; + }; + std::vector stack_; + fc::variant result_; +}; + +/** + * Emits JSON tokens directly into an `fc::json_writer`. No `fc::variant` is + * constructed for built-in scalars; container framing forwards directly to the + * writer. Per-frame item counts are tracked locally so the walker can run the + * same empty-struct guard the variant path enforces. + * + * In the initial scaffolding commit, `unpack_built_in` falls back to the variant-side + * unpack and pipes the result through `to_json_stream(variant, w)`. Subsequent commits + * replace those entries with direct streaming unpacks. + */ +class stream_sink { +public: + /// Construct without an abi_serializer reference -- used by `abi_to_emit` + /// callers (to_json_stream template family) that don't need the binary unpack + /// helpers. + explicit stream_sink(fc::json_writer& w) noexcept : w_(w) {} + + /// Construct bound to an abi_serializer -- used by `_binary_walk` callers + /// that need `unpack_built_in` and `unpack_protobuf` to dispatch built-in types + /// and protobuf decoding respectively. + stream_sink(const abi_serializer& abi, fc::json_writer& w) noexcept : abi_(&abi), w_(w) {} + + void begin_object(); + void end_object(); + void begin_array(); + void end_array(); + void key(std::string_view k); + + void value_string(std::string_view s); + void value_int64(int64_t n); + void value_uint64(uint64_t n); + void value_int32(int32_t n) { value_int64(n); } + void value_uint32(uint32_t n) { value_uint64(n); } + void value_int16(int16_t n) { value_int64(n); } + void value_uint16(uint16_t n) { value_uint64(n); } + void value_int8(int8_t n) { value_int64(n); } + void value_uint8(uint8_t n) { value_uint64(n); } + void value_bool(bool b); + void value_null(); + void value_hex(const char* data, size_t size); + void raw_value(std::string_view raw_json); + + /// Splice an already-built variant. Internally serialises via `to_json_stream` + /// so the writer state stays balanced. Used by the protobuf bridge today and + /// by the variant-fallback built-in path until commit 2 lands the direct unpacks. + void value_variant(const fc::variant& v); + + /// Generic field emit used by `abi_to_emit` for non-ABI types: dispatches + /// straight to `fc::to_json_stream(v, w)` so reflected user structs and primitives + /// emit their tokens without an intermediate variant build. + template + void emit(const T& v) { + fc::to_json_stream(v, w_); + on_value_emitted(); + } + + /// ABI-aware action data emit: decode the binary payload via + /// `abi.binary_to_json_stream` directly into the writer at the current value + /// position. Mirrors `variant_sink::unpack_action_data` which assembles a + /// `fc::variant` instead. + void unpack_action_data(const abi_serializer& abi, std::string_view type, const bytes& data, + abi_traverse_context& ctx, bool short_path); + + /// Atomic key + unpack_action_data. Returns true on success. On unpack failure, + /// rewinds the writer (truncates the buffer + restores frame state) so the caller + /// can emit a hex fallback under the same key without leaving a half-written + /// "data":{ ... orphan or producing duplicate keys. + bool unpack_action_data_field(std::string_view key_name, const abi_serializer& abi, + std::string_view type, const bytes& data, + abi_traverse_context& ctx, bool short_path); + + bool frame_has_items() const noexcept; + + void unpack_built_in(std::string_view ftype, fc::datastream& stream, + bool is_array, bool is_optional, abi_traverse_context& ctx); + void unpack_protobuf(std::string_view ftype, fc::datastream& stream); + +private: + void on_value_emitted() noexcept; + + const abi_serializer* abi_ = nullptr; + fc::json_writer& w_; + std::vector frame_items_; +}; + +} // namespace sysio::chain::impl diff --git a/libraries/chain/include/sysio/chain/asset.hpp b/libraries/chain/include/sysio/chain/asset.hpp index 234283d5d4..c825296c8d 100644 --- a/libraries/chain/include/sysio/chain/asset.hpp +++ b/libraries/chain/include/sysio/chain/asset.hpp @@ -1,4 +1,5 @@ #pragma once +#include #include #include #include @@ -103,12 +104,9 @@ bool operator <= (const asset& a, const asset& b); }} // namespace sysio::chain -namespace fc { -inline void to_variant(const sysio::chain::asset& var, fc::variant& vo) { vo = var.to_string(); } -inline void from_variant(const fc::variant& var, sysio::chain::asset& vo) { - vo = sysio::chain::asset::from_string(var.get_string()); -} -} +// Must precede the extended_asset from_variant body below; that body resolves +// fc::from_variant on asset, which instantiates the primary trait. +FC_SERIALIZE_AS_STRING(sysio::chain::asset) namespace fc { inline void from_variant(const fc::variant& var, sysio::chain::extended_asset& vo) { diff --git a/libraries/chain/include/sysio/chain/block_timestamp.hpp b/libraries/chain/include/sysio/chain/block_timestamp.hpp index 7e50cf9371..a98690ae36 100644 --- a/libraries/chain/include/sysio/chain/block_timestamp.hpp +++ b/libraries/chain/include/sysio/chain/block_timestamp.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -49,6 +50,12 @@ namespace sysio { namespace chain { return fc::time_point(fc::milliseconds(msec)); } + std::string to_string() const { return to_time_point().to_iso_string(); } + + static block_timestamp from_string(std::string_view s) { + return block_timestamp(fc::time_point::from_iso_string(std::string(s))); + } + void operator = (const fc::time_point& t ) { set_time_point(t); } @@ -76,7 +83,7 @@ namespace sysio { namespace chain { typedef block_timestamp block_timestamp_type; constexpr std::string format_as(const block_timestamp_type& t) { - return t.to_time_point().to_iso_string(); + return t.to_string(); } } } /// sysio::chain @@ -92,17 +99,8 @@ namespace std { #include FC_REFLECT(sysio::chain::block_timestamp_type, (slot)) -namespace fc { - template - void to_variant(const sysio::chain::block_timestamp& t, fc::variant& v) { - to_variant( (fc::time_point)t, v); - } - - template - void from_variant(const fc::variant& v, sysio::chain::block_timestamp& t) { - t = v.as(); - } -} +FC_SERIALIZE_AS_STRING_TEMPLATE((uint16_t IntervalMs, uint64_t EpochMs), + (sysio::chain::block_timestamp)) #ifdef _MSC_VER #pragma warning (pop) diff --git a/libraries/chain/include/sysio/chain/chain_id_type.hpp b/libraries/chain/include/sysio/chain/chain_id_type.hpp index 062311001f..8f7e995d21 100644 --- a/libraries/chain/include/sysio/chain/chain_id_type.hpp +++ b/libraries/chain/include/sysio/chain/chain_id_type.hpp @@ -59,6 +59,10 @@ namespace chain { namespace fc { class variant; + class json_writer; void to_variant(const sysio::chain::chain_id_type& cid, fc::variant& v); void from_variant(const fc::variant& v, sysio::chain::chain_id_type& cid); + /// JSON shape: lowercase-hex string -- matches the to_variant emit (which writes + /// the underlying sha256 as a string). + void to_json_stream(const sysio::chain::chain_id_type& cid, fc::json_writer& w); } // fc diff --git a/libraries/chain/include/sysio/chain/database_utils.hpp b/libraries/chain/include/sysio/chain/database_utils.hpp index 17a19c12df..babf8b4efd 100644 --- a/libraries/chain/include/sysio/chain/database_utils.hpp +++ b/libraries/chain/include/sysio/chain/database_utils.hpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -657,14 +658,29 @@ namespace fc { double_to_float64(double_f, f); } + namespace detail { + /// fc's canonical softfloat128 spelling: "0x" + 16 little-endian hex bytes. + /// Assumes platform is little endian and hex representation of the 128-bit + /// integer is in little endian order. Shared by to_variant / to_json_stream + /// so the two emission paths cannot drift. + inline std::string softfloat128_to_hex_string( const softfloat128_t& f ) { + char as_bytes[sizeof(sysio::chain::uint128_t)]; + memcpy(as_bytes, &f, sizeof(as_bytes)); + std::string s = "0x"; + s.append( to_hex( as_bytes, sizeof(as_bytes) ) ); + return s; + } + } + inline void to_variant( const softfloat128_t& f, variant& v ) { - // Assumes platform is little endian and hex representation of 128-bit integer is in little endian order. - char as_bytes[sizeof(sysio::chain::uint128_t)]; - memcpy(as_bytes, &f, sizeof(as_bytes)); - std::string s = "0x"; - s.append( to_hex( as_bytes, sizeof(as_bytes) ) ); - v = s; + v = detail::softfloat128_to_hex_string( f ); + } + + /// JSON shape mirrors to_variant: "0x" with little-endian bytes. + inline + void to_json_stream( const softfloat128_t& f, json_writer& w ) { + w.value_string( detail::softfloat128_to_hex_string( f ) ); } inline diff --git a/libraries/chain/include/sysio/chain/name.hpp b/libraries/chain/include/sysio/chain/name.hpp index 826f8fa6eb..e8649cb66f 100644 --- a/libraries/chain/include/sysio/chain/name.hpp +++ b/libraries/chain/include/sysio/chain/name.hpp @@ -1,5 +1,6 @@ #pragma once #include +#include #include #include @@ -7,15 +8,6 @@ #include #include -namespace sysio::chain { - struct name; -} -namespace fc { - class variant; - void to_variant(const sysio::chain::name& c, fc::variant& v); - void from_variant(const fc::variant& v, sysio::chain::name& check); -} // fc - namespace sysio::chain { /// Alphabet + length traits for the SYSIO account-name encoding: up to 13 @@ -47,6 +39,10 @@ namespace sysio::chain { using base::base; // inherits name(uint64_t), name(std::string_view) constexpr name() = default; + /// Factory used by the FC_SERIALIZE_AS_STRING trait: parses through the + /// validating string_view constructor inherited from fc::basic_name. + static name from_string( std::string_view str ) { return name(str); } + /** * Returns the prefix. * for example: @@ -164,3 +160,4 @@ namespace std { }; FC_REFLECT_DERIVED_EMPTY( sysio::chain::name, (fc::basic_name) ) +FC_SERIALIZE_AS_STRING(sysio::chain::name) diff --git a/libraries/chain/include/sysio/chain/symbol.hpp b/libraries/chain/include/sysio/chain/symbol.hpp index 69b92dc709..599f24c360 100644 --- a/libraries/chain/include/sysio/chain/symbol.hpp +++ b/libraries/chain/include/sysio/chain/symbol.hpp @@ -1,5 +1,6 @@ #pragma once #include +#include #include #include #include @@ -47,6 +48,25 @@ namespace sysio::chain { uint64_t value; operator uint64_t()const { return value; } + + /// Decode value as a packed-byte ASCII string (low byte first), e.g. 0x535953 -> "SYS". + /// Mirrors the long form `symbol(value << 8).name()` but without going through symbol's + /// validating constructor; serialization paths must not throw on already-resident state. + std::string to_string() const { + uint64_t v = value; + std::string result; + while (v > 0) { + char c = v & 0xFF; + result += c; + v >>= 8; + } + return result; + } + + /// Parse a bare symbol-name string (e.g. "SYS") into a symbol_code. Routes through + /// symbol's precision-zero constructor, which validates that all characters are + /// uppercase letters and length <= 7. + static symbol_code from_string(std::string_view s); }; class symbol : fc::reflect_init { @@ -83,18 +103,7 @@ namespace sysio::chain { } return p10; } - string name() const - { - uint64_t v = m_value; - v >>= 8; - string result; - while (v > 0) { - char c = v & 0xFF; - result += c; - v >>= 8; - } - return result; - } + string name() const { return to_symbol_code().to_string(); } symbol_code to_symbol_code()const { return {m_value >> 8}; } @@ -125,6 +134,10 @@ namespace sysio::chain { friend struct fc::reflector; }; // class symbol + inline symbol_code symbol_code::from_string(std::string_view s) { + return symbol(0, s).to_symbol_code(); + } + struct extended_symbol { symbol sym; account_name contract; @@ -168,22 +181,8 @@ namespace sysio::chain { } } // namespace sysio::chain -namespace fc { - inline void to_variant(const sysio::chain::symbol& var, fc::variant& vo) { vo = var.to_string(); } - inline void from_variant(const fc::variant& var, sysio::chain::symbol& vo) { - vo = sysio::chain::symbol::from_string(var.get_string()); - } -} - -namespace fc { - inline void to_variant(const sysio::chain::symbol_code& var, fc::variant& vo) { - vo = sysio::chain::symbol(var.value << 8).name(); - } - inline void from_variant(const fc::variant& var, sysio::chain::symbol_code& vo) { - vo = sysio::chain::symbol(0, var.get_string()).to_symbol_code(); - } -} - FC_REFLECT(sysio::chain::symbol_code, (value)) FC_REFLECT(sysio::chain::symbol, (m_value)) FC_REFLECT(sysio::chain::extended_symbol, (sym)(contract)) +FC_SERIALIZE_AS_STRING(sysio::chain::symbol) +FC_SERIALIZE_AS_STRING(sysio::chain::symbol_code) diff --git a/libraries/chain/include/sysio/chain/types.hpp b/libraries/chain/include/sysio/chain/types.hpp index 052ddc3ed6..7aefd30e18 100644 --- a/libraries/chain/include/sysio/chain/types.hpp +++ b/libraries/chain/include/sysio/chain/types.hpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -13,7 +14,6 @@ #include #include #include -#include #include #include @@ -518,7 +518,7 @@ namespace sysio::chain { // The third option is a function which can be executed in a multithreaded context (likely on the // http_plugin thread pool) and which completes the API processing and returns the result T. // - // The user-provided callable is held inside a std::move_only_function so consume-on-call captures + // The user-provided callable is held inside an fc::move_only_function so consume-on-call captures // (e.g. [cb = std::move(cb)] mutable {...}) work correctly. The outer wrapper is copyable via a // shared_ptr indirection - required because the callback travels through paths that copy // (boost::signals2 slot dispatch, boost::multi_index value storage, lambdas captured by [=], @@ -542,68 +542,11 @@ namespace sysio::chain { using next_function_variant = std::variant()>>; namespace detail { - /** - * @brief Move-only callable storage for standard libraries without std::move_only_function. - * - * libc++ versions shipped with current AppleClang do not always provide the C++23 - * std::move_only_function API. This wrapper preserves the same move-only capture support - * needed by next_function without requiring that library feature. - * - * This is intentionally a narrow polyfill for next_function's current - * `void(T&&)` storage shape, not a general std::move_only_function - * replacement. If next_function starts storing another callable - * signature, extend this wrapper alongside that change. - */ + // Portable move-only callable storage; fc::move_only_function selects + // std::move_only_function where the standard library provides it and a + // general type-erased polyfill otherwise (AppleClang's libc++ lacks it). template - class move_only_function; - - template - class move_only_function { - public: - move_only_function() = default; - move_only_function(std::nullptr_t) noexcept {} - - template - requires (!std::is_same_v, move_only_function> && - std::is_invocable_r_v&, Arg&&>) - move_only_function(F&& f) - : _callable(std::make_unique>>(std::forward(f))) {} - - move_only_function(move_only_function&&) noexcept = default; - move_only_function& operator=(move_only_function&&) noexcept = default; - move_only_function(const move_only_function&) = delete; - move_only_function& operator=(const move_only_function&) = delete; - - void operator()(Arg&& arg) { _callable->invoke(std::forward(arg)); } - explicit operator bool() const noexcept { return static_cast(_callable); } - - private: - struct callable_base { - virtual ~callable_base() = default; - virtual void invoke(Arg&& arg) = 0; - }; - - template - struct callable final : callable_base { - template - explicit callable(Fn&& fn) - : f(std::forward(fn)) {} - - void invoke(Arg&& arg) override { f(std::forward(arg)); } - - F f; - }; - - std::unique_ptr _callable; - }; - - template - using next_move_only_function = -#if defined(__cpp_lib_move_only_function) && __cpp_lib_move_only_function >= 202110L - std::move_only_function; -#else - move_only_function; -#endif + using next_move_only_function = fc::move_only_function; } template diff --git a/libraries/chain/name.cpp b/libraries/chain/name.cpp index 1202602613..f9c63480bb 100644 --- a/libraries/chain/name.cpp +++ b/libraries/chain/name.cpp @@ -1,5 +1,4 @@ #include -#include #include #include @@ -13,10 +12,3 @@ namespace sysio::chain { } } // namespace sysio::chain - -namespace fc { - void to_variant(const sysio::chain::name& c, fc::variant& v) { v = c.to_string(); } - void from_variant(const fc::variant& v, sysio::chain::name& check) { - check = sysio::chain::name{ v.get_string() }; - } -} // fc diff --git a/libraries/chain/symbol.cpp b/libraries/chain/symbol.cpp index 7a42fb6d8f..324b45490d 100644 --- a/libraries/chain/symbol.cpp +++ b/libraries/chain/symbol.cpp @@ -1,8 +1,11 @@ #include #include +#include +#include + namespace sysio::chain { - + symbol symbol::from_string(std::string_view from) { try { @@ -11,7 +14,12 @@ namespace sysio::chain { auto comma_pos = s.find(','); SYS_ASSERT(comma_pos != std::string_view::npos, symbol_type_exception, "missing comma in symbol"); std::string_view prec_part = s.substr(0, comma_pos); - uint8_t p = fc::to_int64(std::string{prec_part}); + // std::from_chars is locale-independent and avoids the std::string round-trip + // that fc::to_int64(const std::string&) would otherwise force. + uint8_t p = 0; + auto [ptr, ec] = std::from_chars(prec_part.data(), prec_part.data() + prec_part.size(), p); + SYS_ASSERT(ec == std::errc{} && ptr == prec_part.data() + prec_part.size(), + symbol_type_exception, "precision is not a valid unsigned integer"); std::string_view name_part = s.substr(comma_pos + 1); SYS_ASSERT( p <= max_precision, symbol_type_exception, "precision {} should be <= 18", p); return symbol(string_to_symbol(p, name_part)); diff --git a/libraries/libfc/include/fc/bitset.hpp b/libraries/libfc/include/fc/bitset.hpp index d1e80a8cf8..bd2c95b083 100644 --- a/libraries/libfc/include/fc/bitset.hpp +++ b/libraries/libfc/include/fc/bitset.hpp @@ -6,7 +6,8 @@ #include #include #include -#include +#include +#include namespace fc { @@ -153,6 +154,11 @@ struct bitset { } std::string to_string() const { + // Guard against allocating an unbounded result string for an oversize bitset. + // Mirrors the pre-FC_SERIALIZE_AS_STRING to_variant cap; enforced here so it + // applies to every JSON path (variant, streaming, datastream). + if (num_blocks() > MAX_NUM_ARRAY_ELEMENTS) + throw std::range_error("number of blocks of bitset cannot be greater than MAX_NUM_ARRAY_ELEMENTS"); std::string res; res.resize(size()); size_t idx = 0; @@ -202,18 +208,6 @@ struct bitset { }; -// ---------------- to_variant / from_variant conversions ---------------------------------------------- - -inline void to_variant(const fc::bitset& bs, fc::variant& v) { - auto num_blocks = bs.num_blocks(); - if (num_blocks > MAX_NUM_ARRAY_ELEMENTS) - throw std::range_error("number of blocks of bitset cannot be greather than MAX_NUM_ARRAY_ELEMENTS"); - - v = bs.to_string(); -} - -inline void from_variant(const fc::variant& v, fc::bitset& bs) { - bs = fc::bitset(v.get_string()); -} - } // namespace fc + +FC_SERIALIZE_AS_STRING(fc::bitset) diff --git a/libraries/libfc/include/fc/container/container_detail.hpp b/libraries/libfc/include/fc/container/container_detail.hpp index 4798851367..6c36d7c073 100644 --- a/libraries/libfc/include/fc/container/container_detail.hpp +++ b/libraries/libfc/include/fc/container/container_detail.hpp @@ -130,6 +130,33 @@ namespace fc { vo = std::move( vars ); } + /// JSON shape mirrors to_variant_from_map: an array of [key, value] pairs. + /// The element emit is deliberately UNQUALIFIED: a qualified fc::to_json_stream + /// call would fix the overload set at this header's definition point and + /// suppress ADL, so element types whose overload is declared later (e.g. + /// fc::crypto::public_key) would wrongly fall into the reflector primary. + template class Map, typename K, typename V, typename... U > + void to_json_stream_from_map( const Map< K, V, U... >& m, fc::json_writer& w ) { + FC_ASSERT( m.size() <= MAX_NUM_ARRAY_ELEMENTS ); + w.begin_array(); + for( const auto& item : m ) { + to_json_stream( item, w ); + } + w.end_array(); + } + + /// JSON shape mirrors to_variant_from_set: an array of elements. Same + /// unqualified-emit rule as to_json_stream_from_map above. + template class Set, typename T, typename... U> + void to_json_stream_from_set( const Set& s, fc::json_writer& w ) { + FC_ASSERT( s.size() <= MAX_NUM_ARRAY_ELEMENTS ); + w.begin_array(); + for( const auto& item : s ) { + to_json_stream( item, w ); + } + w.end_array(); + } + template class Map, typename K, typename V, typename... U> void from_variant_to_map( const variant& v, Map& m ) { const variants& vars = v.get_array(); diff --git a/libraries/libfc/include/fc/container/flat.hpp b/libraries/libfc/include/fc/container/flat.hpp index d9569c3daa..9f8cffdce3 100644 --- a/libraries/libfc/include/fc/container/flat.hpp +++ b/libraries/libfc/include/fc/container/flat.hpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace fc { @@ -143,21 +144,27 @@ namespace fc { } } + template + void to_json_stream( const flat_set< T, U... >& s, json_writer& w ) { + detail::to_json_stream_from_set( s, w ); + } template void to_variant( const flat_set< T, U... >& s, fc::variant& vo ) { detail::to_variant_from_set( s, vo ); } - template void from_variant( const fc::variant& v, flat_set< T, U... >& s ) { detail::from_variant_to_flat_set( v, s ); } + template + void to_json_stream( const flat_multiset< T, U... >& s, json_writer& w ) { + detail::to_json_stream_from_set( s, w ); + } template void to_variant( const flat_multiset< T, U... >& s, fc::variant& vo ) { detail::to_variant_from_set( s, vo ); } - template void from_variant( const fc::variant& v, flat_multiset< T, U... >& s ) { detail::from_variant_to_flat_set( v, s ); @@ -168,6 +175,11 @@ namespace fc { detail::to_variant_from_map( m, vo ); } + template + void to_json_stream( const flat_map< K, V, U... >& m, fc::json_writer& w ) { + detail::to_json_stream_from_map( m, w ); + } + template void from_variant( const variant& v, flat_map& m ) { detail::from_variant_to_flat_map( v, m ); @@ -178,6 +190,11 @@ namespace fc { detail::to_variant_from_map( m, vo ); } + template + void to_json_stream( const flat_multimap< K, V, U... >& m, fc::json_writer& w ) { + detail::to_json_stream_from_map( m, w ); + } + template void from_variant( const variant& v, flat_multimap& m ) { detail::from_variant_to_flat_map( v, m ); diff --git a/libraries/libfc/include/fc/crypto/base64.hpp b/libraries/libfc/include/fc/crypto/base64.hpp index 0091200445..115f4850f5 100644 --- a/libraries/libfc/include/fc/crypto/base64.hpp +++ b/libraries/libfc/include/fc/crypto/base64.hpp @@ -82,7 +82,7 @@ std::vector base64_decode( const std::string& s); std::vector base64_decode( std::string_view s); std::string base64url_encode(const char* s, size_t len); std::string base64url_encode(const std::string& s); -std::vector base64url_decode(const std::string& s); +std::vector base64url_decode(std::string_view s); namespace detail { // @@ -373,7 +373,7 @@ inline std::string base64url_encode(const std::string& s) { return base64_encode((unsigned char const*)s.data(), s.size(), true); } -inline std::vector base64url_decode(const std::string& s) { - return detail::decode>(s, false, true); +inline std::vector base64url_decode(std::string_view s) { + return detail::decode, std::string_view>(s, false, true); } } // namespace fc diff --git a/libraries/libfc/include/fc/crypto/blake3.hpp b/libraries/libfc/include/fc/crypto/blake3.hpp index 4c67433d85..6eb7892e13 100644 --- a/libraries/libfc/include/fc/crypto/blake3.hpp +++ b/libraries/libfc/include/fc/crypto/blake3.hpp @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include @@ -26,7 +26,7 @@ class blake3 { blake3() { memset(_hash, 0, sizeof(_hash)); } - explicit blake3(const std::string& hex_str) { + explicit blake3(std::string_view hex_str) { auto bytes = from_hex(hex_str); FC_ASSERT(bytes.size() == sizeof(_hash), "Invalid blake3 hex string length: {}", hex_str.size()); @@ -34,6 +34,14 @@ class blake3 { } std::string str() const { return to_hex(to_char_span()); } + std::string to_string() const { return str(); } + /// Validating parse used by the FC_SERIALIZE_AS_STRING trait: rejects odd-length + /// hex (a 63-char string would otherwise round up to 32 decoded bytes and pass the + /// constructor's size check with a zero-extended final nibble). + static blake3 from_string(std::string_view s) { + FC_ASSERT(s.size() % 2 == 0, "blake3 hex string length must be even, got {}", s.size()); + return blake3(s); + } uint8_t* data() { return _hash; } const uint8_t* data() const { return _hash; } @@ -74,17 +82,8 @@ class blake3 { } // namespace crypto -inline void to_variant( const crypto::blake3& bi, variant& v ) { - v = std::vector( bi.char_data(), bi.char_data() + bi.data_size() ); -} - -inline void from_variant( const variant& v, crypto::blake3& bi ) { - std::vector ve = v.as< std::vector >(); - FC_ASSERT(ve.size() == crypto::blake3::byte_size, "Invalid blake3 data size: {}", ve.size()); - memcpy(bi.char_data(), ve.data(), crypto::blake3::byte_size); -} - } // namespace fc #include FC_REFLECT_TYPENAME(fc::crypto::blake3) +FC_SERIALIZE_AS_STRING(fc::crypto::blake3) diff --git a/libraries/libfc/include/fc/crypto/bls_common.hpp b/libraries/libfc/include/fc/crypto/bls_common.hpp index df205092f9..bebeaf061f 100644 --- a/libraries/libfc/include/fc/crypto/bls_common.hpp +++ b/libraries/libfc/include/fc/crypto/bls_common.hpp @@ -13,7 +13,7 @@ constexpr std::string signature_prefix = "SIG_BLS_"; } // namespace constants template - Container deserialize_base64url(const std::string& data_str) { + Container deserialize_base64url(std::string_view data_str) { using wrapper = checksum_data; wrapper wrapped; @@ -40,7 +40,7 @@ constexpr std::string signature_prefix = "SIG_BLS_"; return data_str; } - inline public_key_data deserialize_bls_base64url(const std::string& base64urlstr) { + inline public_key_data deserialize_bls_base64url(std::string_view base64urlstr) { using namespace fc::crypto::bls::constants; auto res = std::mismatch(bls_public_key_prefix.begin(), bls_public_key_prefix.end(), base64urlstr.begin()); FC_ASSERT(res.first == bls_public_key_prefix.end(), "BLS Public Key has invalid format : {}", base64urlstr); @@ -48,7 +48,7 @@ constexpr std::string signature_prefix = "SIG_BLS_"; return fc::crypto::bls::deserialize_base64url(data_str); } - inline signature_data sig_parse_base64url(const std::string& base64urlstr) { + inline signature_data sig_parse_base64url(std::string_view base64urlstr) { auto res = std::mismatch(constants::signature_prefix.begin(), bls::constants::signature_prefix.end(), base64urlstr.begin()); FC_ASSERT(res.first == bls::constants::signature_prefix.end(), "BLS Signature has invalid format : {}", base64urlstr); auto data_str = base64urlstr.substr(bls::constants::signature_prefix.size()); diff --git a/libraries/libfc/include/fc/crypto/bls_private_key.hpp b/libraries/libfc/include/fc/crypto/bls_private_key.hpp index 1a0cac3e40..9b15af7306 100644 --- a/libraries/libfc/include/fc/crypto/bls_private_key.hpp +++ b/libraries/libfc/include/fc/crypto/bls_private_key.hpp @@ -39,13 +39,14 @@ class private_key { * @brief Constructs a private key from a base64url encoded string * @param base64urlstr The base64url encoded string */ - explicit private_key(const std::string& base64urlstr); + explicit private_key(std::string_view base64urlstr); private_key& operator=(const private_key& pk) = default; private_key& operator=(private_key&& pk) noexcept = default; private_key_secret get_secret() const { return _sk; }; std::string to_string() const; + static private_key from_string(std::string_view s) { return private_key(s); } public_key get_public_key() const; @@ -218,28 +219,23 @@ struct private_key_shim : crypto::shim { } // namespace fc::crypto::bls -namespace fc { -/** - * @brief Converts a BLS private key to a variant - * @param var The private key to convert - * @param vo The output variant - */ -void to_variant(const crypto::bls::private_key& var, variant& vo); - -/** - * @brief Converts a variant to a BLS private key - * @param var The variant to convert - * @param vo The output private key - */ -void from_variant(const variant& var, crypto::bls::private_key& vo); -} // namespace fc - - #include FC_REFLECT(fc::crypto::bls::private_key, (_sk)) FC_REFLECT_TYPENAME(fc::crypto::bls::public_key) +#include +FC_SERIALIZE_AS_STRING(fc::crypto::bls::private_key) + +// Override the FC_SERIALIZE_AS_STRING-generated to_json_stream for bls::private_key +// with a deleted non-template overload: streaming HTTP responses must never carry +// private key material. to_variant / from_variant remain intact so server-internal +// flows (signature_provider, wallet_api_plugin) keep working. +namespace fc { + class json_writer; + void to_json_stream(const crypto::bls::private_key& var, json_writer& w) = delete; +} + FC_REFLECT(fc::crypto::bls::public_key_shim, (shim_ptr)) FC_REFLECT(fc::crypto::bls::signature_shim, (shim_ptr)) diff --git a/libraries/libfc/include/fc/crypto/bls_public_key.hpp b/libraries/libfc/include/fc/crypto/bls_public_key.hpp index da3f9812bf..e5d5c44cc6 100644 --- a/libraries/libfc/include/fc/crypto/bls_public_key.hpp +++ b/libraries/libfc/include/fc/crypto/bls_public_key.hpp @@ -42,13 +42,14 @@ namespace fc::crypto::bls { explicit public_key(const public_key_data& affine_non_montgomery_le); // affine non-montgomery base64url with bls_public_key_prefix - explicit public_key(const std::string& base64urlstr); + explicit public_key(std::string_view base64urlstr); bool valid()const; public_key_data serialize()const; // affine non-montgomery base64url with bls_public_key_prefix std::string to_string() const; static std::string to_string(const public_key_data& key); + static public_key from_string(std::string_view s) { return public_key(s); } const bls12_381::g1& jacobian_montgomery_le() const { return _jacobian_montgomery_le; } const public_key_data& affine_non_montgomery_le() const { return _affine_non_montgomery_le; } @@ -99,7 +100,5 @@ namespace fc::crypto::bls { } // fc::crypto::bls -namespace fc { - void to_variant(const crypto::bls::public_key& var, variant& vo); - void from_variant(const variant& var, crypto::bls::public_key& vo); -} // namespace fc +#include +FC_SERIALIZE_AS_STRING(fc::crypto::bls::public_key) diff --git a/libraries/libfc/include/fc/crypto/bls_signature.hpp b/libraries/libfc/include/fc/crypto/bls_signature.hpp index 2d2a12dcf7..38ef4a4a87 100644 --- a/libraries/libfc/include/fc/crypto/bls_signature.hpp +++ b/libraries/libfc/include/fc/crypto/bls_signature.hpp @@ -30,11 +30,12 @@ namespace fc::crypto::bls { explicit signature(bls::signature_data_span affine_non_montgomery_le); // affine non-montgomery base64url with signature_prefix - explicit signature(const std::string& base64urlstr); + explicit signature(std::string_view base64urlstr); // affine non-montgomery base64url with signature_prefix std::string to_string() const; static std::string to_string(const bls::signature_data& sig); + static signature from_string(std::string_view s) { return signature(s); } const bls12_381::g2& jacobian_montgomery_le() const { return _jacobian_montgomery_le; } const bls::signature_data& affine_non_montgomery_le() const { return _affine_non_montgomery_le; } @@ -93,7 +94,7 @@ namespace fc::crypto::bls { aggregate_signature& operator=(aggregate_signature&&) = default; // affine non-montgomery base64url with signature_prefix - explicit aggregate_signature(const std::string& base64_url_str); + explicit aggregate_signature(std::string_view base64_url_str); explicit aggregate_signature(const signature& sig) : _jacobian_montgomery_le(sig.jacobian_montgomery_le()) {} @@ -110,6 +111,7 @@ namespace fc::crypto::bls { // affine non-montgomery base64url with signature_prefix // Expensive as conversion from Jacobian Montgomery to Affine Non-Montgomery needed std::string to_string() const; + static aggregate_signature from_string(std::string_view s) { return aggregate_signature(s); } const bls12_381::g2& jacobian_montgomery_le() const { return _jacobian_montgomery_le; } @@ -148,11 +150,6 @@ namespace fc::crypto::bls { } // fc::crypto::bls -namespace fc { - - void to_variant(const crypto::bls::signature& var, variant& vo); - void from_variant(const variant& var, crypto::bls::signature& vo); - void to_variant(const crypto::bls::aggregate_signature& var, variant& vo); - void from_variant(const variant& var, crypto::bls::aggregate_signature& vo); - -} // namespace fc +#include +FC_SERIALIZE_AS_STRING(fc::crypto::bls::signature) +FC_SERIALIZE_AS_STRING(fc::crypto::bls::aggregate_signature) diff --git a/libraries/libfc/include/fc/crypto/hex.hpp b/libraries/libfc/include/fc/crypto/hex.hpp index ce0a18a9fc..a54e2685cb 100644 --- a/libraries/libfc/include/fc/crypto/hex.hpp +++ b/libraries/libfc/include/fc/crypto/hex.hpp @@ -33,9 +33,9 @@ namespace fc { size_t from_hex(std::string_view hex_str, char* out_data, size_t out_data_len); - std::vector from_hex(const std::string& hex, bool trim_prefix = true); + std::vector from_hex(std::string_view hex, bool trim_prefix = true); - std::string trim_hex_prefix(const std::string& hex); + std::string_view trim_hex_prefix(std::string_view hex); /** diff --git a/libraries/libfc/include/fc/crypto/keccak256.hpp b/libraries/libfc/include/fc/crypto/keccak256.hpp index 8d4886d8e8..9106b10705 100644 --- a/libraries/libfc/include/fc/crypto/keccak256.hpp +++ b/libraries/libfc/include/fc/crypto/keccak256.hpp @@ -6,6 +6,7 @@ #include #include #include +#include namespace fc { namespace crypto { @@ -19,10 +20,14 @@ class keccak256 { static constexpr size_t byte_size = 256 / (8 * sizeof(uint8_t)); keccak256(); - explicit keccak256(const std::string& hex_str); + explicit keccak256(std::string_view hex_str); // in hex std::string str() const; + std::string to_string() const { return str(); } + /// Validating parse used by the FC_SERIALIZE_AS_STRING trait: rejects odd-length + /// hex (the strictness the pre-trait from_variant vector path enforced). + static keccak256 from_string(std::string_view s); const uint8_t* data() const { return _hash; } constexpr size_t data_size() const { return byte_size; } @@ -71,11 +76,8 @@ class keccak256 { }; } // namespace crypto -class variant; -void to_variant(const crypto::keccak256& bi, variant& v); -void from_variant(const variant& v, crypto::keccak256& bi); - } // namespace fc #include FC_REFLECT_TYPENAME(fc::crypto::keccak256) +FC_SERIALIZE_AS_STRING(fc::crypto::keccak256) diff --git a/libraries/libfc/include/fc/crypto/private_key.hpp b/libraries/libfc/include/fc/crypto/private_key.hpp index 9fbdd1c6b3..e7f116fa6e 100644 --- a/libraries/libfc/include/fc/crypto/private_key.hpp +++ b/libraries/libfc/include/fc/crypto/private_key.hpp @@ -105,6 +105,13 @@ namespace fc { void to_variant(const crypto::private_key& var, variant& vo, const fc::yield_function_t& yield = fc::yield_function_t()); void from_variant(const variant& var, crypto::private_key& vo); + + // Deleting the streaming-JSON overload is a load-bearing safety check, not a + // not-yet-implemented marker. HTTP responses must never expose private key + // material; the variant path remains available for wallet_api_plugin and other + // server-internal consumers that intentionally serialize private keys. + class json_writer; + void to_json_stream(const crypto::private_key& var, json_writer& w) = delete; } // namespace fc FC_REFLECT(fc::crypto::private_key, (_storage) ) diff --git a/libraries/libfc/include/fc/crypto/public_key.hpp b/libraries/libfc/include/fc/crypto/public_key.hpp index 93a4efe938..137a72a4dc 100644 --- a/libraries/libfc/include/fc/crypto/public_key.hpp +++ b/libraries/libfc/include/fc/crypto/public_key.hpp @@ -117,6 +117,9 @@ namespace fc { void to_variant(const crypto::public_key& var, variant& vo, const fc::yield_function_t& yield = fc::yield_function_t()); void from_variant(const variant& var, crypto::public_key& vo); + + class json_writer; + void to_json_stream(const crypto::public_key& var, json_writer& w); } // namespace fc FC_REFLECT(fc::crypto::public_key, (_storage) ) diff --git a/libraries/libfc/include/fc/crypto/ripemd160.hpp b/libraries/libfc/include/fc/crypto/ripemd160.hpp index 7a0b231fbd..697ee52a4e 100644 --- a/libraries/libfc/include/fc/crypto/ripemd160.hpp +++ b/libraries/libfc/include/fc/crypto/ripemd160.hpp @@ -4,6 +4,7 @@ #include #include #include +#include namespace fc{ class sha512; @@ -13,9 +14,13 @@ class ripemd160 : public add_packhash_to_hash { public: ripemd160(); - explicit ripemd160( const std::string& hex_str ); + explicit ripemd160( std::string_view hex_str ); std::string str()const; + std::string to_string() const { return str(); } + /// Validating parse used by the FC_SERIALIZE_AS_STRING trait: rejects odd-length + /// hex (the strictness the pre-trait from_variant vector path enforced). + static ripemd160 from_string(std::string_view s); char* data()const; size_t data_size()const { return 160/8; } @@ -69,10 +74,6 @@ class ripemd160 : public add_packhash_to_hash uint32_t _hash[5]; }; - class variant; - void to_variant( const ripemd160& bi, variant& v ); - void from_variant( const variant& v, ripemd160& bi ); - typedef ripemd160 uint160_t; typedef ripemd160 uint160; @@ -91,3 +92,5 @@ namespace std } }; } + +FC_SERIALIZE_AS_STRING(fc::ripemd160) diff --git a/libraries/libfc/include/fc/crypto/sha1.hpp b/libraries/libfc/include/fc/crypto/sha1.hpp index 080ae886e6..6e534a3588 100644 --- a/libraries/libfc/include/fc/crypto/sha1.hpp +++ b/libraries/libfc/include/fc/crypto/sha1.hpp @@ -2,6 +2,7 @@ #include #include #include +#include namespace fc{ @@ -9,9 +10,13 @@ class sha1 : public add_packhash_to_hash { public: sha1(); - explicit sha1( const std::string& hex_str ); + explicit sha1( std::string_view hex_str ); std::string str()const; + std::string to_string() const { return str(); } + /// Validating parse used by the FC_SERIALIZE_AS_STRING trait: rejects odd-length + /// hex (the strictness the pre-trait from_variant vector path enforced). + static sha1 from_string(std::string_view s); operator std::string()const; char* data(); @@ -65,10 +70,6 @@ class sha1 : public add_packhash_to_hash uint32_t _hash[5]; }; - class variant; - void to_variant( const sha1& bi, variant& v ); - void from_variant( const variant& v, sha1& bi ); - } // namespace fc namespace std @@ -82,3 +83,5 @@ namespace std } }; } + +FC_SERIALIZE_AS_STRING(fc::sha1) diff --git a/libraries/libfc/include/fc/crypto/sha224.hpp b/libraries/libfc/include/fc/crypto/sha224.hpp index f9c73b8a6a..5662fb882e 100644 --- a/libraries/libfc/include/fc/crypto/sha224.hpp +++ b/libraries/libfc/include/fc/crypto/sha224.hpp @@ -4,6 +4,7 @@ #include #include #include +#include namespace fc { @@ -12,9 +13,13 @@ class sha224 : public add_packhash_to_hash { public: sha224(); - explicit sha224( const std::string& hex_str ); + explicit sha224( std::string_view hex_str ); std::string str()const; + std::string to_string() const { return str(); } + /// Validating parse used by the FC_SERIALIZE_AS_STRING trait: rejects odd-length + /// hex (the strictness the pre-trait from_variant vector path enforced). + static sha224 from_string(std::string_view s); operator std::string()const; char* data(); @@ -70,10 +75,6 @@ class sha224 : public add_packhash_to_hash uint32_t _hash[7]; }; - class variant; - void to_variant( const sha224& bi, variant& v ); - void from_variant( const variant& v, sha224& bi ); - } // fc namespace std { @@ -88,3 +89,4 @@ namespace std } #include FC_REFLECT_TYPENAME( fc::sha224 ) +FC_SERIALIZE_AS_STRING(fc::sha224) diff --git a/libraries/libfc/include/fc/crypto/sha256.hpp b/libraries/libfc/include/fc/crypto/sha256.hpp index 242a9b43ab..ab18c36661 100644 --- a/libraries/libfc/include/fc/crypto/sha256.hpp +++ b/libraries/libfc/include/fc/crypto/sha256.hpp @@ -7,6 +7,7 @@ #include #include #include +#include #include namespace fc @@ -22,11 +23,15 @@ class sha256 : public add_packhash_to_hash using byte_array_type = std::array; sha256(); - explicit sha256( const std::string& hex_str ); + explicit sha256( std::string_view hex_str ); explicit sha256( const hash_array_type& hash_arr ); explicit sha256( const char *data, size_t size ); std::string str()const; + std::string to_string() const { return str(); } + /// Validating parse used by the FC_SERIALIZE_AS_STRING trait: rejects odd-length + /// hex (the strictness the pre-trait from_variant vector path enforced). + static sha256 from_string(std::string_view s); std::string short_id()const; const char* data()const; @@ -128,10 +133,6 @@ class sha256 : public add_packhash_to_hash uint64_t _hash[uint64_size]; }; -class variant; -void to_variant( const sha256& bi, variant& v ); -void from_variant( const variant& v, sha256& bi ); - uint64_t hash64(const char* buf, size_t len); constexpr auto format_as(const fc::sha256& h) { @@ -178,3 +179,4 @@ namespace fc { #include FC_REFLECT_TYPENAME( fc::sha256 ) +FC_SERIALIZE_AS_STRING(fc::sha256) diff --git a/libraries/libfc/include/fc/crypto/sha3.hpp b/libraries/libfc/include/fc/crypto/sha3.hpp index 32a8972040..0b87183ee0 100644 --- a/libraries/libfc/include/fc/crypto/sha3.hpp +++ b/libraries/libfc/include/fc/crypto/sha3.hpp @@ -4,6 +4,7 @@ #include #include #include +#include #include namespace fc @@ -14,10 +15,14 @@ class sha3 public: sha3(); ~sha3(){} - explicit sha3(const std::string &hex_str); + explicit sha3(std::string_view hex_str); explicit sha3(const char *data, size_t size); std::string str() const; + std::string to_string() const { return str(); } + /// Validating parse used by the FC_SERIALIZE_AS_STRING trait: rejects odd-length + /// hex (the strictness the pre-trait from_variant vector path enforced). + static sha3 from_string(std::string_view s); operator std::string() const; const char *data() const; @@ -100,10 +105,6 @@ class sha3 uint64_t _hash[4]; }; -class variant; -void to_variant(const sha3 &bi, variant &v); -void from_variant(const variant &v, sha3 &bi); - } // namespace fc namespace std @@ -132,3 +133,4 @@ struct hash } // namespace boost #include FC_REFLECT_TYPENAME(fc::sha3) +FC_SERIALIZE_AS_STRING(fc::sha3) diff --git a/libraries/libfc/include/fc/crypto/sha512.hpp b/libraries/libfc/include/fc/crypto/sha512.hpp index 060bd7da2d..c2f3205f1f 100644 --- a/libraries/libfc/include/fc/crypto/sha512.hpp +++ b/libraries/libfc/include/fc/crypto/sha512.hpp @@ -2,6 +2,7 @@ #include #include #include +#include namespace fc { @@ -10,9 +11,13 @@ class sha512 : public add_packhash_to_hash { public: sha512(); - explicit sha512( const std::string& hex_str ); + explicit sha512( std::string_view hex_str ); std::string str()const; + std::string to_string() const { return str(); } + /// Validating parse used by the FC_SERIALIZE_AS_STRING trait: rejects odd-length + /// hex (the strictness the pre-trait from_variant vector path enforced). + static sha512 from_string(std::string_view s); operator std::string()const; char* data(); @@ -68,11 +73,8 @@ class sha512 : public add_packhash_to_hash typedef fc::sha512 uint512; - class variant; - void to_variant( const sha512& bi, variant& v ); - void from_variant( const variant& v, sha512& bi ); - } // fc #include FC_REFLECT_TYPENAME( fc::sha512 ) +FC_SERIALIZE_AS_STRING(fc::sha512) diff --git a/libraries/libfc/include/fc/crypto/signature.hpp b/libraries/libfc/include/fc/crypto/signature.hpp index 36be3bc8ee..88a10bc1cf 100644 --- a/libraries/libfc/include/fc/crypto/signature.hpp +++ b/libraries/libfc/include/fc/crypto/signature.hpp @@ -115,6 +115,9 @@ namespace fc { void to_variant(const crypto::signature& var, variant& vo, const fc::yield_function_t& yield = fc::yield_function_t()); void from_variant(const variant& var, crypto::signature& vo); + +class json_writer; +void to_json_stream(const crypto::signature& var, json_writer& w); } // namespace fc template <> diff --git a/libraries/libfc/include/fc/exception/exception.hpp b/libraries/libfc/include/fc/exception/exception.hpp index 1d0abb47b1..15cb4460c7 100644 --- a/libraries/libfc/include/fc/exception/exception.hpp +++ b/libraries/libfc/include/fc/exception/exception.hpp @@ -14,6 +14,7 @@ namespace fc { namespace detail { class exception_impl; } + class json_writer; enum exception_code { @@ -134,6 +135,7 @@ namespace fc friend void to_variant( const exception& e, variant& v ); friend void from_variant( const variant& e, exception& ll ); + friend void to_json_stream( const exception& e, json_writer& w ); exception& operator=( const exception& copy ); exception& operator=( exception&& copy ); @@ -143,6 +145,8 @@ namespace fc void to_variant( const exception& e, variant& v ); void from_variant( const variant& e, exception& ll ); + /// JSON shape mirrors `to_variant(exception)`: { code, name, message, stack: [log_messages...] }. + void to_json_stream( const exception& e, json_writer& w ); typedef std::shared_ptr exception_ptr; typedef std::optional oexception; diff --git a/libraries/libfc/include/fc/fixed_string.hpp b/libraries/libfc/include/fc/fixed_string.hpp deleted file mode 100644 index 3c087fe284..0000000000 --- a/libraries/libfc/include/fc/fixed_string.hpp +++ /dev/null @@ -1,171 +0,0 @@ -#pragma once -#include - - -namespace fc { - - - /** - * This class is designed to offer in-place memory allocation of a string up to Length equal to - * sizeof(Storage). - * - * The string will serialize the same way as std::string for variant and raw formats - * The string will sort according to the comparison operators defined for Storage, this enables effecient - * sorting. - */ - template > - class fixed_string { - public: - fixed_string(){ - memset( (char*)&data, 0, sizeof(data) ); - } - fixed_string( const fixed_string& c ):data(c.data){} - - fixed_string( const std::string& str ) { - if( str.size() < sizeof(data) ) { - memset( (char*)&data, 0, sizeof(data) ); - memcpy( (char*)&data, str.c_str(), str.size() ); - } else { - memcpy( (char*)&data, str.c_str(), sizeof(data) ); - } - } - fixed_string( const char* str ) { - memset( (char*)&data, 0, sizeof(data) ); - auto l = strlen(str); - if( l < sizeof(data) ) { - memset( (char*)&data, 0, sizeof(data) ); - memcpy( (char*)&data, str, l ); - } - else { - memcpy( (char*)&data, str, sizeof(data) ); - } - } - - operator std::string()const { - const char* self = (const char*)&data; - return std::string( self, self + size() ); - } - - uint32_t size()const { - if( *(((const char*)&data)+sizeof(data) - 1) ) - return sizeof(data); - return strnlen( (const char*)&data, sizeof(data) ); - } - uint32_t length()const { return size(); } - - fixed_string& operator=( const fixed_string& str ) { - data = str.data; - return *this; - } - fixed_string& operator=( const char* str ) { - return *this = fixed_string(str); - } - - fixed_string& operator=( const std::string& str ) { - if( str.size() < sizeof(data) ) { - memset( (char*)&data, 0, sizeof(data) ); - memcpy( (char*)&data, str.c_str(), str.size() ); - } - else { - memcpy( (char*)&data, str.c_str(), sizeof(data) ); - } - return *this; - } - - friend std::string operator + ( const fixed_string& a, const std::string& b ) { - return std::string(a) + b; - } - friend std::string operator + ( const std::string& a, const fixed_string& b ) { - return a + std::string(b); - } - - friend bool operator < ( const fixed_string& a, const fixed_string& b ) { - return a.data < b.data; - } - friend bool operator <= ( const fixed_string& a, const fixed_string& b ) { - return a.data <= b.data; - } - friend bool operator > ( const fixed_string& a, const fixed_string& b ) { - return a.data > b.data; - } - friend bool operator >= ( const fixed_string& a, const fixed_string& b ) { - return a.data >= b.data; - } - friend bool operator == ( const fixed_string& a, const fixed_string& b ) { - return a.data == b.data; - } - friend bool operator != ( const fixed_string& a, const fixed_string& b ) { - return a.data != b.data; - } - - friend std::ostream& operator << ( std::ostream& out, const fixed_string& str ) { - return out << std::string(str); - } - //private: - Storage data; - }; - - namespace raw - { - template - inline void pack( Stream& s, const fc::fixed_string& u ) { - unsigned_int size = u.size(); - pack( s, size ); - s.write( (const char*)&u.data, size ); - } - - template - inline void unpack( Stream& s, fc::fixed_string& u ) { - unsigned_int size; - fc::raw::unpack( s, size ); - if( size.value > 0 ) { - if( size.value > sizeof(Storage) ) { - s.read( (char*)&u.data, sizeof(Storage) ); - char buf[1024]; - size_t left = size.value - sizeof(Storage); - while( left >= 1024 ) - { - s.read( buf, 1024 ); - left -= 1024; - } - s.read( buf, left ); - - /* - s.seekp( s.tellp() + (size.value - sizeof(Storage)) ); - char tmp; - size.value -= sizeof(storage); - while( size.value ){ s.read( &tmp, 1 ); --size.value; } - */ - // s.skip( size.value - sizeof(Storage) ); - } else { - s.read( (char*)&u.data, size.value ); - } - } - } - - /* - template - inline void pack( Stream& s, const boost::multiprecision::number& d ) { - s.write( (const char*)&d, sizeof(d) ); - } - - template - inline void unpack( Stream& s, boost::multiprecision::number& u ) { - s.read( (const char*)&u, sizeof(u) ); - } - */ - } -} - -#include -namespace fc { - template - void to_variant( const fixed_string& s, variant& v ) { - v = std::string(s); - } - - template - void from_variant( const variant& v, fixed_string& s ) { - s = v.as_string(); - } -} diff --git a/libraries/libfc/include/fc/int128.hpp b/libraries/libfc/include/fc/int128.hpp index 8a28c7d3e8..a0a0c5aebb 100644 --- a/libraries/libfc/include/fc/int128.hpp +++ b/libraries/libfc/include/fc/int128.hpp @@ -18,6 +18,7 @@ namespace fc __int128 int128_from_string( const std::string& s ); class variant; + class json_writer; fc::uint128 to_uint128(std::uint64_t hi, uint64_t lo); fc::uint128 to_uint128(const fc::variant& v); @@ -29,6 +30,11 @@ namespace fc void to_variant( const int128& var, variant& vo ); void from_variant( const variant& var, int128& vo ); + /// JSON shape: always-quoted decimal string, matching to_json_stream(variant, w) + /// for int128_type / uint128_type variants. + void to_json_stream( const uint128& var, json_writer& w ); + void to_json_stream( const int128& var, json_writer& w ); + namespace raw { template diff --git a/libraries/libfc/include/fc/io/enum_type.hpp b/libraries/libfc/include/fc/io/enum_type.hpp index 63e8a5b4b4..6921a0ccf9 100644 --- a/libraries/libfc/include/fc/io/enum_type.hpp +++ b/libraries/libfc/include/fc/io/enum_type.hpp @@ -1,5 +1,6 @@ #pragma once #include +#include #include #include @@ -58,6 +59,16 @@ namespace fc vo.value = var.as(); } + /// JSON shape mirrors to_variant: forward to the EnumType's own to_json_stream + /// so FC_REFLECT_ENUM-reflected enums emit as their member-name strings (not + /// the underlying integer). Bare numeric enums fall through to the primitive + /// overloads via integer promotion. + template + void to_json_stream( const enum_type& var, json_writer& w ) + { + fc::to_json_stream(static_cast(var.value), w); + } + /** serializes like an IntType */ namespace raw diff --git a/libraries/libfc/include/fc/io/json.hpp b/libraries/libfc/include/fc/io/json.hpp index 53d93a919b..7700060d34 100644 --- a/libraries/libfc/include/fc/io/json.hpp +++ b/libraries/libfc/include/fc/io/json.hpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #define DEFAULT_MAX_RECURSION_DEPTH 200 @@ -26,7 +27,7 @@ namespace fc relaxed_parser = 2, legacy_parser_with_string_doubles = 3 }; - using yield_function_t = fc::optional_delegate; + using yield_function_t = fc::json_yield_function_t; static constexpr uint64_t max_length_limit = std::numeric_limits::max(); static constexpr size_t escape_string_yield_check_count = 128; static variant from_string( const std::string& utf8_str, const parse_type ptype = parse_type::legacy_parser, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ); @@ -91,10 +92,6 @@ namespace fc } }; - std::string escape_string( const std::string_view& str, const json::yield_function_t& yield, bool escape_control_chars = true ); - /// Returns true if any characters were escaped or invalid UTF-8 was pruned. - bool escape_string( const std::string_view& str, std::string& out, const json::yield_function_t& yield, bool escape_control_chars = true ); - } // fc #undef DEFAULT_MAX_RECURSION_DEPTH diff --git a/libraries/libfc/include/fc/io/json_escape.hpp b/libraries/libfc/include/fc/io/json_escape.hpp new file mode 100644 index 0000000000..073638b031 --- /dev/null +++ b/libraries/libfc/include/fc/io/json_escape.hpp @@ -0,0 +1,31 @@ +#pragma once + +// JSON string-escape primitives shared by both the legacy variant-tree path and +// the streaming json_writer path. Decoupled from class json so that +// fc/io/json_stream.hpp (and downstream variant.hpp / flat.hpp container +// streaming overloads) can call escape_string without pulling in class json -- +// avoiding the cycle: variant.hpp -> json_stream.hpp -> json.hpp -> variant.hpp. +// +// The yield-function alias is the parameter type used by escape_string; +// class json::yield_function_t is an alias to this type so existing callers +// continue to compile unchanged. + +#include +#include +#include + +namespace fc { + + // The yield callback used by streaming serialisers to check deadlines and bail out + // of long-running JSON conversions. class json::yield_function_t (in fc/io/json.hpp) + // is an alias to this type so existing callers continue to compile. + using json_yield_function_t = fc::optional_delegate; + + /// Escape a UTF-8 string for inclusion in a JSON value. Returns the new string. + std::string escape_string( const std::string_view& str, const json_yield_function_t& yield, bool escape_control_chars = true ); + + /// Append the escaped form of `str` to `out`. Returns true if any characters were + /// escaped or invalid UTF-8 was pruned. + bool escape_string( const std::string_view& str, std::string& out, const json_yield_function_t& yield, bool escape_control_chars = true ); + +} // namespace fc diff --git a/libraries/libfc/include/fc/io/json_stream.hpp b/libraries/libfc/include/fc/io/json_stream.hpp new file mode 100644 index 0000000000..8abe665e0a --- /dev/null +++ b/libraries/libfc/include/fc/io/json_stream.hpp @@ -0,0 +1,435 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fc { + +/// Slack the writer guarantees in the output buffer at construction so typical responses +/// append without an early reallocation. +inline constexpr size_t json_writer_initial_reserve = 4096; + +/// Decimal-digit buffer that fits any int64/uint64 (uint64 max = 20 digits, plus sign and slack). +inline constexpr size_t int64_decimal_buf_size = std::numeric_limits::digits10 + 3; +static_assert(int64_decimal_buf_size >= std::numeric_limits::digits10 + 1); + +/// Shortest-roundtrip std::to_chars of a finite double fits well within this. +inline constexpr size_t double_shortest_buf_size = 32; + +/// 64-bit integers whose magnitude exceeds this are emitted as quoted JSON strings to +/// preserve precision past JS's 2^53 mantissa. Single source of truth for the streaming +/// emitters; matches fc::json::to_string's emission (libfc/src/io/json.cpp int64/uint64 cases). +inline constexpr int64_t json_integer_quote_magnitude = 0xffffffff; + +/// Depth cap for the fc::variant / variant_object streaming walkers; matches +/// DEFAULT_MAX_RECURSION_DEPTH in fc/io/json.hpp so the streaming path throws where +/// fc::json::to_string's yield would. +inline constexpr uint32_t json_stream_max_depth = 200; + +/// Default byte-growth stride between growth-guard invocations (see json_writer's guarded +/// constructor). Small enough that a caller enforcing a memory budget observes emission +/// growth promptly; large enough that the guard is a negligible fraction of emission cost +/// (one invocation per ~64 KiB appended). Responses smaller than one stride never invoke +/// the guard mid-emission. +inline constexpr size_t json_writer_default_guard_stride = 64 * 1024; + +/** + * Streaming JSON writer that emits tokens directly into an output std::string. + * + * Designed as a faster alternative to the fc::mutable_variant_object -> fc::variant -> + * fc::json::to_string pipeline used throughout the HTTP endpoints. Instead of allocating + * per-field variant_object entries and per-string fc::variant wrappers, callers drive the + * writer with explicit begin_object / key / value_* calls that append bytes to the output + * buffer in place. + * + * Contract: + * - Values must appear in value-position only (top-level, after a key(), or inside an + * array). Nesting is validated with asserts in debug builds. + * - end_object / end_array must balance begin_object / begin_array. + * - String content is JSON-escaped via fc::escape_string; control characters become + * \\uXXXX, surrogate pairs are produced when needed. + * - raw_value appends an already-serialized JSON fragment verbatim; callers are + * responsible for its validity (used e.g. to embed fc::variant output from legacy + * code paths). + */ +class json_writer { +public: + /// Growth-guard callback: invoked with the output buffer's size (bytes) each time the + /// buffer has grown one stride past the previous invocation. Before a string value or + /// key is escaped into the buffer, the guard instead receives the PROSPECTIVE size -- + /// current size plus the token's input length, a lower bound on the token's peak -- so + /// it can reject (and a budget enforcer charge) the bytes before any capacity is + /// reserved for them; actual growth reconciles at the next invocation. The guard may + /// throw to abort emission -- eg an HTTP memory-budget enforcer rejecting a response + /// mid-serialize. The guard must not emit through the writer it guards. + using growth_guard_t = std::function; + + /// @param out buffer tokens are appended to; also the guard's measured quantity. + /// @param growth_guard optional guard consulted as the buffer grows (empty = never). + /// @param guard_stride minimum byte growth between guard invocations; a stride of 0 + /// re-checks on every token (test hook, not for production use). + /// + /// Guard-throw semantics: if the guard throws, the writer re-arms so the NEXT token + /// emitted re-invokes the guard immediately (regardless of stride). Mid-emission + /// rollback handlers (eg abi_serializer's catch-rewind-hex-fallback path) legitimately + /// swallow exceptions from nested serializers; re-arming guarantees a guard abort + /// re-raises out of every such handler instead of being silently absorbed -- unless the + /// guard's condition has cleared, in which case emission resumes legitimately. + /// + /// Intra-token checks: guarded writers also consult the guard WHILE a single large + /// token streams -- every escape_string_yield_check_count input chars of a string value + /// or key (measuring the output buffer, so escape expansion counts) and every stride of + /// bytes appended by value_hex / raw_value (their append loops chunk at guard_stride) -- + /// so one oversized token cannot materialize past the budget behind a single + /// token-boundary check. String values and keys additionally pre-check their known + /// input length with the guard BEFORE escape_string reserves capacity for it, so an + /// over-budget string aborts with zero allocation (see guard_check_pending). A guard + /// throw mid-token leaves a partial token in the buffer; the writer's re-arm plus the + /// caller's checkpoint()/rewind() (or wholesale abandonment of the buffer, as the http + /// handler does) keep that safe. + explicit json_writer(std::string& out, growth_guard_t growth_guard = {}, + size_t guard_stride = json_writer_default_guard_stride) + : out_(out) + , guard_(std::move(growth_guard)) + , guard_stride_(guard_stride) + { + if (guard_) { + next_guard_check_ = out_.size() + guard_stride_; + // Consulted by escape_string every escape_string_yield_check_count input chars; + // routes the escape loop's periodic yield into the same stride-gated guard check + // the token boundaries use. + escape_yield_ = [this](size_t) { guard_check(); }; + } + // Enough room for a reasonable response without reallocation; callers that know + // the expected size can reserve() themselves before constructing the writer. + // Skip if the caller already has slack so we don't risk a spec-permitted shrink. + if (out_.capacity() - out_.size() < json_writer_initial_reserve) { + out_.reserve(out_.size() + json_writer_initial_reserve); + } + } + + void begin_object() { + value_prefix(); + out_.push_back('{'); + stack_.push_back(frame{context::object, false}); + } + + void end_object() { + assert(!stack_.empty() && stack_.back().ctx == context::object); + assert(!awaiting_value_); // key() without a value would leave a dangling colon + out_.push_back('}'); + stack_.pop_back(); + } + + void begin_array() { + value_prefix(); + out_.push_back('['); + stack_.push_back(frame{context::array, false}); + } + + void end_array() { + assert(!stack_.empty() && stack_.back().ctx == context::array); + out_.push_back(']'); + stack_.pop_back(); + } + + void key(std::string_view k) { + // Subsumes the plain token-boundary guard_check() and additionally approves the + // key's input length before escape_string reserves capacity for it. + guard_check_pending(k.size()); + assert(!stack_.empty() && stack_.back().ctx == context::object); + assert(!awaiting_value_); // two key() calls in a row would produce "a":,"b": (invalid JSON) + if (stack_.back().has_item) { + out_.push_back(','); + } else { + stack_.back().has_item = true; + } + out_.push_back('"'); + // key strings must also be escaped (eg field names containing special chars are rare, + // but correctness matters on untrusted input echoed into error messages). + fc::escape_string(k, out_, escape_yield_, true); + out_.push_back('"'); + out_.push_back(':'); + // A key mandates a value follows; suppress the comma from value_prefix for that value. + awaiting_value_ = true; + } + + void value_string(std::string_view s) { + // Approve the string's input length with the guard BEFORE escape_string reserves + // capacity for it: an over-budget value aborts here with zero allocation. Runs + // ahead of value_prefix so a rejection mutates nothing (no dangling separator for + // absorb-and-continue callers); the escape loop's periodic escape_yield_ then + // covers expansion beyond the approval. + guard_check_pending(s.size()); + value_prefix(); + out_.push_back('"'); + fc::escape_string(s, out_, escape_yield_, true); + out_.push_back('"'); + } + + void value_int64(int64_t n) { value_prefix(); append_signed(n); } + void value_uint64(uint64_t n) { value_prefix(); append_unsigned(n); } + void value_int32(int32_t n) { value_int64(n); } + void value_uint32(uint32_t n) { value_uint64(n); } + void value_int16(int16_t n) { value_int64(n); } + void value_uint16(uint16_t n) { value_uint64(n); } + void value_int8(int8_t n) { value_int64(n); } + void value_uint8(uint8_t n) { value_uint64(n); } + + void value_double(double d) { + // NaN / +-inf have no JSON representation: any encoder would emit non-conforming + // tokens that no parser will accept. Reject before mutating the output buffer + // so the writer's frame state stays consistent on throw. + FC_ASSERT(std::isfinite(d), "fc::json_writer::value_double: non-finite double cannot be JSON-encoded"); + value_prefix(); + // std::to_chars is locale-independent (period radix always) and shortest-roundtrip: + // the parsed-back double is bit-identical to the input. snprintf with %g would + // honor LC_NUMERIC and emit comma radix in non-C locales -- invalid JSON. + char buf[double_shortest_buf_size]; + auto r = std::to_chars(buf, buf + sizeof(buf), d); + assert(r.ec == std::errc{}); // unreachable: finite double + buffer sized for shortest-roundtrip + out_.append(buf, static_cast(r.ptr - buf)); + } + + void value_bool(bool b) { + value_prefix(); + if (b) out_.append("true", 4); + else out_.append("false", 5); + } + + void value_null() { + value_prefix(); + out_.append("null", 4); + } + + /// Emits a JSON string token whose content is the lowercase hex encoding of the + /// byte range [data, data+size). No intermediate std::string allocation; bytes + /// are looped once and 2 hex chars are appended per source byte. Equivalent + /// observable output to value_string(fc::to_hex(data, size)) but avoids the + /// per-call heap allocation in fc::to_hex. Used by streaming JSON paths that + /// previously paid the to_hex allocation on every action's data / return_value. + void value_hex(const char* data, size_t size) { + value_prefix(); + out_.push_back('"'); + static constexpr char digits[] = "0123456789abcdef"; + const auto* p = reinterpret_cast(data); + // Guarded writers emit in stride-sized chunks and consult the guard between them, so + // a single large blob is budget-checked at the writer's configured granularity while + // it streams; unguarded writers keep the one tight loop (chunk_end = size on the + // first pass). + const size_t src_chunk = std::max(1, guard_stride_ / 2); // 2 hex chars per source byte + size_t i = 0; + while (i < size) { + const size_t chunk_end = guard_ ? std::min(size, i + src_chunk) : size; + for (; i < chunk_end; ++i) { + out_.push_back(digits[p[i] >> 4]); + out_.push_back(digits[p[i] & 0x0f]); + } + if (guard_) { + guard_check(); + } + } + out_.push_back('"'); + } + + /// Append a preformatted JSON fragment at a value position. Used to embed output + /// from legacy fc::variant paths (eg abi_serializer::binary_to_variant + json::to_string) + /// without an additional parse/re-emit cycle. + void raw_value(std::string_view raw) { + value_prefix(); + if (!guard_) { + out_.append(raw.data(), raw.size()); + return; + } + // Guarded writers splice in stride-sized chunks and consult the guard between them, + // so a single large preformatted fragment is budget-checked at the writer's + // configured granularity while it streams. + const size_t chunk = std::max(1, guard_stride_); + for (size_t off = 0; off < raw.size();) { + const size_t n = std::min(raw.size() - off, chunk); + out_.append(raw.data() + off, n); + off += n; + guard_check(); + } + } + + /// Fused key + value emitter; the streaming-JSON counterpart to + /// fc::mutable_variant_object::set / operator(). Returns *this so call sites + /// chain naturally: + /// + /// w.begin_object(); + /// w.set("id", id) + /// .set("number", n) + /// .set("producer", producer); + /// w.end_object(); + /// + /// Dispatches the value via to_json_stream, so any type with a + /// to_json_stream overload (primitives, std containers, fc leaf types, + /// FC_REFLECT'd structs via the reflector path) is supported. Callers must + /// have included at the call site - that header + /// is where the to_json_stream primary template + reflector dispatch lives. + template + json_writer& set(std::string_view name, const T& value) { + key(name); + to_json_stream(value, *this); + return *this; + } + + /// Fused key + raw-value emitter. Same chaining shape as set(), but the + /// value is a preformatted JSON fragment spliced verbatim (eg an + /// abi_serializer::binary_to_variant result that has already been json'd). + json_writer& set_raw(std::string_view name, std::string_view raw_json) { + key(name); + raw_value(raw_json); + return *this; + } + + /// True when all begin_* have been paired with end_* AND no key() is awaiting + /// its value. Used by tests/sinks that wrap the writer to confirm end-state + /// integrity after a streaming sequence completes. + bool balanced() const { return stack_.empty() && !awaiting_value_; } + + /// Snapshot of the writer's mutable state. Pair with rewind() to discard + /// any tokens emitted between the two calls. Cheap (~3 words copied) and + /// non-throwing. Used by the abi_serializer streaming path to roll back + /// half-written ABI fields when the inner unpack throws partway. + struct checkpoint_t { + size_t buf_size = 0; + size_t stack_size = 0; + bool surviving_top_has_item = false; // restored value of stack_[stack_size-1].has_item + bool awaiting_value = false; + }; + checkpoint_t checkpoint() const noexcept { + return { + out_.size(), + stack_.size(), + stack_.empty() ? false : stack_.back().has_item, + awaiting_value_ + }; + } + void rewind(const checkpoint_t& cp) noexcept { + // rewind() only discards tokens APPENDED since checkpoint(); a caller that closed + // frames past the checkpoint and re-opened new ones would resize the stack UP here + // with value-initialized frames -- silent corruption, so the contract is asserted. + assert(out_.size() >= cp.buf_size); + assert(stack_.size() >= cp.stack_size); + out_.resize(cp.buf_size); + stack_.resize(cp.stack_size); + if (!stack_.empty()) stack_.back().has_item = cp.surviving_top_has_item; + awaiting_value_ = cp.awaiting_value; + } + +private: + enum class context : uint8_t { object, array }; + struct frame { + context ctx = context::object; + bool has_item = false; // true once one element or key:value has been emitted in this frame + }; + + /// Consult the growth guard if the buffer has crossed the next stride boundary (or on + /// every token once a prior guard invocation threw -- see the constructor contract). + /// Called from value_prefix(), which -- together with the guard_check_pending calls + /// fronting key() and value_string() -- fronts every token-emitting entry point except + /// end_object/end_array (1 byte each; the final buffer size is the caller's to observe + /// after emission completes). For guarded writers it also runs WITHIN large tokens: + /// from escape_yield_ inside the string-escape loop and between chunks in value_hex / + /// raw_value, so no single token outruns the budget. + void guard_check() { + if (out_.size() < next_guard_check_) { + return; + } + // Re-arm across a guard throw: 0 makes every subsequent token re-enter here while + // the guard keeps throwing, so a catch-and-continue serializer upstream cannot + // absorb the abort. Restored to a real stride only when the guard returns cleanly. + next_guard_check_ = 0; + guard_(out_.size()); + next_guard_check_ = out_.size() + guard_stride_; + } + + /// Consult the growth guard with the buffer's PROSPECTIVE size before `pending` bytes + /// are appended. Escaped string output is at least one byte per input byte (pre + /// UTF-8-prune), so a string token's input length is a guaranteed lower bound on the + /// buffer growth it is about to cause; checking it first lets the guard reject -- and + /// a budget enforcer charge -- those bytes before escape_string reserves capacity for + /// them, so an over-budget string aborts with zero allocation instead of after a + /// full-size reserve. With pending == 0 this is exactly guard_check(), including the + /// re-arm-across-throw contract. On clean approval the next check is deferred until + /// the buffer grows a stride PAST the approved size: approved bytes do not re-fire the + /// guard, while escape expansion beyond them still does (via escape_yield_). A + /// prune-shrunk token that never reaches the approved size simply reconciles downward + /// at the guard's next invocation. + void guard_check_pending(size_t pending) { + if (out_.size() + pending < next_guard_check_) { + return; + } + next_guard_check_ = 0; + guard_(out_.size() + pending); + next_guard_check_ = out_.size() + pending + guard_stride_; + } + + void value_prefix() { + guard_check(); + if (awaiting_value_) { + // Value right after a key() - no separator, first-item bookkeeping already set. + awaiting_value_ = false; + return; + } + if (stack_.empty()) { + // Top-level value (eg a bare array/object at the root). Nothing to prefix. + return; + } + // Must be in array context when a raw value appears without a preceding key. + assert(stack_.back().ctx == context::array); + if (stack_.back().has_item) { + out_.push_back(','); + } else { + stack_.back().has_item = true; + } + } + + // std::to_chars is non-throwing, locale-independent, and avoids the format-string + // parsing overhead that snprintf pays on every call. + void append_signed(int64_t n) { + char buf[int64_decimal_buf_size]; + auto r = std::to_chars(buf, buf + sizeof(buf), n); + assert(r.ec == std::errc{}); // unreachable: buffer sized for any int64 decimal + out_.append(buf, static_cast(r.ptr - buf)); + } + void append_unsigned(uint64_t n) { + char buf[int64_decimal_buf_size]; + auto r = std::to_chars(buf, buf + sizeof(buf), n); + assert(r.ec == std::errc{}); // unreachable: buffer sized for any uint64 decimal + out_.append(buf, static_cast(r.ptr - buf)); + } + + std::string& out_; + std::vector stack_; // small; vector is fine and avoids extra deps + growth_guard_t guard_; // empty unless the guarded constructor form was used + json_yield_function_t escape_yield_; // guarded: runs guard_check inside escape_string loops + size_t guard_stride_ = json_writer_default_guard_stride; + size_t next_guard_check_ = std::numeric_limits::max(); // SIZE_MAX = no guard + bool awaiting_value_ = false; +}; + +/// to_json_stream overload for an emit-closure: a `std::function` +/// whose JSON output is whatever the closure writes when invoked. Used by callers +/// that want to deliver a streaming response through machinery (eg `bind_stream`'s +/// async/typed-T paths) that wraps the payload in `to_json_stream(payload, w)` -- +/// the closure itself is the emitter, so we just invoke it. +inline void to_json_stream(const std::function& fn, json_writer& w) { + if (fn) fn(w); + else w.value_null(); +} + +} // namespace fc diff --git a/libraries/libfc/include/fc/io/json_stream_fwd.hpp b/libraries/libfc/include/fc/io/json_stream_fwd.hpp new file mode 100644 index 0000000000..0e30dcd27e --- /dev/null +++ b/libraries/libfc/include/fc/io/json_stream_fwd.hpp @@ -0,0 +1,14 @@ +#pragma once + +// Lightweight forward-declaration of fc::json_writer for use in type headers that want to +// declare to_json_stream overloads without pulling in the full json_writer definition +// (and the json/escape_string dependencies it transitively drags in). Callers that need +// to actually emit must include ; callers that only need to +// declare the overload include this header. +// +// Pattern matches the existing `class variant;` forward declaration used by headers that +// declare to_variant without including fc/variant.hpp. + +namespace fc { + class json_writer; +} diff --git a/libraries/libfc/include/fc/io/raw.hpp b/libraries/libfc/include/fc/io/raw.hpp index 636d8fca37..062503360b 100644 --- a/libraries/libfc/include/fc/io/raw.hpp +++ b/libraries/libfc/include/fc/io/raw.hpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -259,6 +260,19 @@ namespace fc { else { v.reset(); } // in case v has already has a value } FC_RETHROW_EXCEPTIONS( warn, "optional<{}>", fc::get_typename::name() ) } + // fc::nullable -- binary serialization parity with std::optional: the wrapper only + // changes JSON emission (key always present, explicit null when disengaged), so its + // wire form delegates to the wrapped optional unchanged. + template + void pack( Stream& s, const fc::nullable& v ) { + fc::raw::pack( s, v.value ); + } + + template + void unpack( Stream& s, fc::nullable& v ) { + fc::raw::unpack( s, v.value ); + } + // std::vector template inline void pack( Stream& s, const std::vector& value ) { FC_ASSERT( value.size() <= MAX_SIZE_OF_BYTE_ARRAYS ); diff --git a/libraries/libfc/include/fc/io/raw_fwd.hpp b/libraries/libfc/include/fc/io/raw_fwd.hpp index ce9257a0fc..f789d35622 100644 --- a/libraries/libfc/include/fc/io/raw_fwd.hpp +++ b/libraries/libfc/include/fc/io/raw_fwd.hpp @@ -26,7 +26,6 @@ namespace fc { namespace ip { class endpoint; } namespace ecc { class public_key; class private_key; } - template class fixed_string; namespace raw { @@ -41,9 +40,6 @@ namespace fc { template void unpack(Stream& s, sysio::chain::signed_block& v); - template inline void pack( Stream& s, const fc::fixed_string& u ); - template inline void unpack( Stream& s, fc::fixed_string& u ); - template inline void pack( Stream& s, const fc::enum_type& tp ); template diff --git a/libraries/libfc/include/fc/io/varint.hpp b/libraries/libfc/include/fc/io/varint.hpp index cba998f37d..67d53d0eca 100644 --- a/libraries/libfc/include/fc/io/varint.hpp +++ b/libraries/libfc/include/fc/io/varint.hpp @@ -76,12 +76,18 @@ struct signed_int { constexpr auto format_as(const signed_int& si) { return si.value; } class variant; +class json_writer; void to_variant( const signed_int& var, variant& vo ); void from_variant( const variant& var, signed_int& vo ); void to_variant( const unsigned_int& var, variant& vo ); void from_variant( const variant& var, unsigned_int& vo ); +/// JSON shape: bare integer. Matches to_json_stream(variant) for variants built +/// from `signed_int::value` / `unsigned_int::value` (int32 / uint32 fit-range). +void to_json_stream( const signed_int& var, json_writer& w ); +void to_json_stream( const unsigned_int& var, json_writer& w ); + } // namespace fc #include diff --git a/libraries/libfc/include/fc/log/log_message.hpp b/libraries/libfc/include/fc/log/log_message.hpp index 31200c6a05..5402a923aa 100644 --- a/libraries/libfc/include/fc/log/log_message.hpp +++ b/libraries/libfc/include/fc/log/log_message.hpp @@ -152,6 +152,11 @@ namespace fc void to_variant( const log_message& l, variant& v ); void from_variant( const variant& l, log_message& c ); + /// JSON shape mirrors `to_variant(log_message)`; uses a per-message variant intermediate as the streaming form + /// (log_message has a private impl pointer rather than reflected fields, so we can't walk it directly). + class json_writer; + void to_json_stream( const log_message& m, json_writer& w ); + typedef std::vector log_messages; template diff --git a/libraries/libfc/include/fc/move_only_function.hpp b/libraries/libfc/include/fc/move_only_function.hpp new file mode 100644 index 0000000000..288d365a34 --- /dev/null +++ b/libraries/libfc/include/fc/move_only_function.hpp @@ -0,0 +1,82 @@ +#pragma once + +/** + * @file fc/move_only_function.hpp + * @brief `fc::move_only_function`: std::move_only_function where the standard + * library provides it, otherwise a type-erased polyfill. + * + * libc++ versions shipped with current AppleClang do not provide the C++23 + * std::move_only_function API, so portable code uses this alias instead of naming the + * std template directly. + * + * The polyfill supports the subset the tree relies on: construction from any callable + * invocable with the signature (including move-only captures), move-only semantics, + * `explicit operator bool`, and a non-const `operator()`. cv/ref/noexcept-qualified + * signatures are not implemented. + */ + +#include + +#include +#include +#include +#include + +namespace fc { + +namespace detail { + + template + class move_only_function_impl; + + template + class move_only_function_impl { + public: + move_only_function_impl() = default; + move_only_function_impl(std::nullptr_t) noexcept {} + + template + requires (!std::is_same_v, move_only_function_impl> && + std::is_invocable_r_v&, Args...>) + move_only_function_impl(F&& f) + : _callable(std::make_unique>>(std::forward(f))) {} + + move_only_function_impl(move_only_function_impl&&) noexcept = default; + move_only_function_impl& operator=(move_only_function_impl&&) noexcept = default; + move_only_function_impl(const move_only_function_impl&) = delete; + move_only_function_impl& operator=(const move_only_function_impl&) = delete; + + R operator()(Args... args) { return _callable->invoke(std::forward(args)...); } + explicit operator bool() const noexcept { return static_cast(_callable); } + + private: + struct callable_base { + virtual ~callable_base() = default; + virtual R invoke(Args... args) = 0; + }; + + template + struct callable final : callable_base { + template + explicit callable(Fn&& fn) + : f(std::forward(fn)) {} + + R invoke(Args... args) override { return f(std::forward(args)...); } + + F f; + }; + + std::unique_ptr _callable; + }; + +} // namespace detail + +template +using move_only_function = +#if defined(__cpp_lib_move_only_function) && __cpp_lib_move_only_function >= 202110L + std::move_only_function; +#else + detail::move_only_function_impl; +#endif + +} // namespace fc diff --git a/libraries/libfc/include/fc/network/url.hpp b/libraries/libfc/include/fc/network/url.hpp index a4270f03ff..56c9b84e78 100644 --- a/libraries/libfc/include/fc/network/url.hpp +++ b/libraries/libfc/include/fc/network/url.hpp @@ -3,6 +3,7 @@ #include #include #include +#include #include namespace fc { @@ -39,6 +40,8 @@ namespace fc { bool operator==( const url& cmp )const; operator std::string()const; + std::string to_string() const { return std::string(*this); } + static url from_string(std::string_view s) { return url(std::string(s)); } //// file, ssh, tcp, http, ssl, etc... std::string proto()const; @@ -55,7 +58,6 @@ namespace fc { std::shared_ptr my; }; - void to_variant( const url& u, fc::variant& v ); - void from_variant( const fc::variant& v, url& u ); - } // namespace fc + +FC_SERIALIZE_AS_STRING(fc::url) diff --git a/libraries/libfc/include/fc/nullable.hpp b/libraries/libfc/include/fc/nullable.hpp new file mode 100644 index 0000000000..16a402849e --- /dev/null +++ b/libraries/libfc/include/fc/nullable.hpp @@ -0,0 +1,49 @@ +#pragma once + +#include +#include + +namespace fc { + +/** + * Optional-valued serialization wrapper whose JSON key is ALWAYS present. + * + * Reflected std::optional members are OMITTED from serialized output when disengaged -- + * both the to_variant reflector (to_variant_visitor::add) and the streaming reflector + * (to_json_stream_visitor::emit) skip them. Some endpoint schemas instead pin + * "key always present, value null when absent"; eg get_finalizer_info's active/pending + * policy fields, whose original fc::variant representation emitted explicit nulls that + * clients dereference before testing. Wrapping such a member as fc::nullable keeps + * that shape while the member stays strongly typed: + * + * - engaged -> serializes exactly like a plain T member + * - disengaged -> serializes as an explicit JSON null under the member's key + * + * from_variant mirrors the emission: a JSON null deserializes to the disengaged state. + * + * The serializer overloads (to_variant / from_variant / to_json_stream) live in + * fc/variant.hpp alongside the std::optional trio so the two families' shapes stay in + * lock-step; this header carries only the vocabulary type. + */ +template +struct nullable { + std::optional value; + + nullable() = default; + nullable(const T& v) : value(v) {} + nullable(T&& v) : value(std::move(v)) {} + nullable& operator=(const T& v) { value = v; return *this; } + nullable& operator=(T&& v) { value = std::move(v); return *this; } + + bool has_value() const { return value.has_value(); } + explicit operator bool() const { return value.has_value(); } + const T& operator*() const { return *value; } + T& operator*() { return *value; } + const T* operator->() const { return &*value; } + T* operator->() { return &*value; } + void reset() { value.reset(); } + + friend bool operator==(const nullable& a, const nullable& b) { return a.value == b.value; } +}; + +} // namespace fc diff --git a/libraries/libfc/include/fc/reflect/json_stream.hpp b/libraries/libfc/include/fc/reflect/json_stream.hpp new file mode 100644 index 0000000000..4c569b45d5 --- /dev/null +++ b/libraries/libfc/include/fc/reflect/json_stream.hpp @@ -0,0 +1,207 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fc { + +/** + * to_json_stream is the streaming counterpart to to_variant: it emits the JSON form of a + * reflected C++ value directly into a json_writer without building an intermediate + * fc::variant tree. The primary template dispatches through fc::reflector in the same + * way fc::to_variant does, so any type already marked with FC_REFLECT(...) gets a JSON + * serializer for free. + * + * Overloads for primitives and standard containers are provided below. Domain-specific + * types (sha256, public_key_type, variant, time_point, ...) can add their own + * to_json_stream overload next to their existing to_variant overload; until that happens + * callers can fall back to raw_value(fc::json::to_string(variant(v))) for individual + * fields at measured hot spots. + */ +template +void to_json_stream(const T& o, json_writer& w); + +// -- Scalar overloads --------------------------------------------------------------------- + +inline void to_json_stream(bool b, json_writer& w) { w.value_bool(b); } +// `char` emits as an int8 number (e.g. 'A' -> 65), matching the int8_t overload's shape. +// Single-character JSON strings would surprise reflected-struct consumers; the numeric +// form is consistent with how chars are typically read back via from_variant. +inline void to_json_stream(char c, json_writer& w) { w.value_int8(static_cast(c)); } +inline void to_json_stream(int8_t n, json_writer& w) { w.value_int8(n); } +inline void to_json_stream(uint8_t n, json_writer& w) { w.value_uint8(n); } +inline void to_json_stream(int16_t n, json_writer& w) { w.value_int16(n); } +inline void to_json_stream(uint16_t n, json_writer& w) { w.value_uint16(n); } +inline void to_json_stream(int32_t n, json_writer& w) { w.value_int32(n); } +inline void to_json_stream(uint32_t n, json_writer& w) { w.value_uint32(n); } +// 64-bit integers with magnitude > json_integer_quote_magnitude are emitted as JSON +// strings to preserve precision past JS's 2^53 mantissa. Matches fc::variant's emission +// (libfc/src/io/json.cpp, int64_type / uint64_type cases). Writers stay raw; callers +// that want a literal big number use value_int64 / value_uint64 directly. +inline void to_json_stream(int64_t n, json_writer& w) { + if (n > json_integer_quote_magnitude || n < -json_integer_quote_magnitude) { + char buf[int64_decimal_buf_size]; + auto r = std::to_chars(buf, buf + sizeof(buf), n); + w.value_string(std::string_view(buf, r.ptr - buf)); + } else { + w.value_int64(n); + } +} +inline void to_json_stream(uint64_t n, json_writer& w) { + if (n > static_cast(json_integer_quote_magnitude)) { + char buf[int64_decimal_buf_size]; + auto r = std::to_chars(buf, buf + sizeof(buf), n); + w.value_string(std::string_view(buf, r.ptr - buf)); + } else { + w.value_uint64(n); + } +} +// int64_t / uint64_t are aliases of different concrete types per platform (long on 64-bit Linux, +// long long on macOS), so one of the standard 64-bit integer types is left without an overload +// above. Mirror to_variant's platform guard (fc/variant.hpp) so reflected fields of that type +// (e.g. a size_t member on macOS) stream instead of falling through to the reflector primary. +#ifdef __APPLE__ +inline void to_json_stream(size_t n, json_writer& w) { to_json_stream(static_cast(n), w); } +#elif !defined(_MSC_VER) +inline void to_json_stream(long long int n, json_writer& w) { to_json_stream(static_cast(n), w); } +inline void to_json_stream(unsigned long long int n, json_writer& w) { to_json_stream(static_cast(n), w); } +#endif +// double / float emit as a JSON-quoted fixed-precision string to match fc::variant's emission shape +// (variant.cpp s_fc_to_string + json.cpp double_type case). Reflector-driven struct fields (e.g. +// get_producers_result.total_producer_vote_weight) depend on this shape; emitting a bare JSON number +// would silently break clients parsing the string form. std::to_chars with chars_format::fixed is +// locale-independent (unlike stringstream / snprintf, which honor LC_NUMERIC). +inline constexpr int double_fixed_precision = std::numeric_limits::digits10 + 2; +// Largest finite double in fixed format: max_exponent10 integer digits + '.' + precision frac + sign + NUL slack. +inline constexpr size_t double_fixed_buf_size = std::numeric_limits::max_exponent10 + double_fixed_precision + 8; +inline void to_json_stream(double d, json_writer& w) { + if (!std::isfinite(d)) { + // Variant (s_fc_to_string, variant.cpp) emits the explicit "nan" / "-nan" / "inf" / + // "-inf" spellings for non-finite doubles -- deliberately NOT the C library's, whose + // negative-NaN formatting is platform-dependent (glibc "-nan", Apple "nan"). Match + // that shape. NaN never compares, so its sign must come from signbit -- arbitrary + // ABI float bit patterns reach here and negative NaN must round-trip byte-identically. + w.value_string(std::isnan(d) ? (std::signbit(d) ? "-nan" : "nan") + : (std::signbit(d) ? "-inf" : "inf")); + return; + } + char buf[double_fixed_buf_size]; + auto r = std::to_chars(buf, buf + sizeof(buf), d, std::chars_format::fixed, double_fixed_precision); + assert(r.ec == std::errc{}); + w.value_string(std::string_view(buf, r.ptr - buf)); +} +inline void to_json_stream(float f, json_writer& w) { + to_json_stream(static_cast(f), w); +} +inline void to_json_stream(std::string_view s, json_writer& w) { w.value_string(s); } +inline void to_json_stream(const std::string& s, json_writer& w) { w.value_string(s); } +inline void to_json_stream(const char* s, json_writer& w) { w.value_string(s ? std::string_view(s) : std::string_view()); } + +// Container overloads (std::vector, std::array, std::optional, std::map, std::set, +// std::pair, std::deque, std::unordered_map, std::unordered_set, std::multimap) live +// alongside their to_variant siblings in fc/variant.hpp. flat_set / flat_multiset live +// in fc/container/flat.hpp. Co-locating with to_variant keeps the two paths' shape +// and MAX_NUM_ARRAY_ELEMENTS guards in lock-step. + +// -- Reflector visitor -------------------------------------------------------------------- + +template +class to_json_stream_visitor { +public: + to_json_stream_visitor(json_writer& w, const T& v) + : w_(w), val_(v) {} + + template + void operator()(const char* name) const { + emit(name, val_.*member); + } + +private: + template + void emit(const char* name, const std::optional& v) const { + // Mirror to_variant_visitor::add semantics: unset optionals are omitted rather than + // emitted as explicit null, so downstream JSON matches the existing HTTP responses. + if (v) { + w_.key(name); + to_json_stream(*v, w_); + } + } + + template + void emit(const char* name, const M& v) const { + w_.key(name); + to_json_stream(v, w_); + } + + json_writer& w_; + const T& val_; +}; + +// -- Dispatch for reflected user types --------------------------------------------------- + +namespace detail { + + template + struct if_enum_json { + template + static void to_json_stream(const T& v, json_writer& w) { + static_assert(fc::reflector::is_defined::value, + "fc::to_json_stream: T must be FC_REFLECT'd or have a hand-written to_json_stream overload"); + w.begin_object(); + fc::reflector::visit(to_json_stream_visitor(w, v)); + w.end_object(); + } + }; + + template<> + struct if_enum_json { + template + static void to_json_stream(const T& o, json_writer& w) { + w.value_string(fc::reflector::to_fc_string(o)); + } + }; + +} // namespace detail + +template +void to_json_stream(const T& o, json_writer& w) { + detail::if_enum_json::is_enum>::to_json_stream(o, w); +} + +// -- Convenience entry point -------------------------------------------------------------- + +/// One-shot helper: serialize `v` into a freshly-allocated std::string. Hot-path callers +/// should construct json_writer directly against their output buffer to avoid the copy. +template +std::string to_json_string(const T& v) { + std::string out; + { + json_writer w(out); + to_json_stream(v, w); + } + return out; +} + +/// Explicit escape hatch for types that already have a to_variant overload but have not +/// yet grown a native to_json_stream. Emits the value by converting to fc::variant and +/// running fc::json::to_string, then splicing the result at a value position. Use at +/// migrated-endpoint field granularity to avoid blocking on a full library sweep. +template +void to_json_stream_via_variant(const T& v, json_writer& w) { + fc::variant tmp; + fc::to_variant(v, tmp); + w.raw_value(fc::json::to_string(tmp, fc::json::yield_function_t())); +} + +} // namespace fc diff --git a/libraries/libfc/include/fc/serialize_as_string.hpp b/libraries/libfc/include/fc/serialize_as_string.hpp new file mode 100644 index 0000000000..91a0a9c637 --- /dev/null +++ b/libraries/libfc/include/fc/serialize_as_string.hpp @@ -0,0 +1,72 @@ +#pragma once + +#include +#include + +namespace fc { + +/** + * Opt-in trait for types whose JSON form is the result of T::to_string() and + * whose JSON parse form is T::from_string(std::string_view). Specializing + * this to std::true_type for a type makes all three serializers available + * without hand-rolled overloads: + * + * fc::to_variant(t, v) -> v = t.to_string() + * fc::from_variant(v, t) -> t = T::from_string(v.get_string()) + * fc::to_json_stream(t, w) -> w.value_string(t.to_string()) + * + * Use the FC_SERIALIZE_AS_STRING macro for the one-line opt-in. + * + * Requirements on the type: + * - std::string T::to_string() const (or convertible-to-string return) + * - static T T::from_string(std::string_view) + * + * Types whose to_variant is the FC_REFLECT'd struct shape do NOT specialize + * this; they fall through to the generic reflector dispatch. Types whose + * shape is something else (number, struct, ISO date, etc.) need their own + * hand-rolled overload. + */ +template +struct serialize_as_string : std::false_type {}; + +template + requires serialize_as_string::value +inline void to_variant(const T& v, fc::variant& vo) { + vo = v.to_string(); +} + +template + requires serialize_as_string::value +inline void from_variant(const fc::variant& v, T& out) { + out = T::from_string(v.get_string()); +} + +template + requires serialize_as_string::value +inline void to_json_stream(const T& v, fc::json_writer& w) { + w.value_string(v.to_string()); +} + +} // namespace fc + +#define FC_SERIALIZE_AS_STRING(TYPE) \ + namespace fc { template<> struct serialize_as_string : std::true_type {}; } + +// Class-template variant of FC_SERIALIZE_AS_STRING. BOTH arguments must be wrapped +// in parens so internal commas (template parameter lists, template arguments) survive +// the preprocessor's argument splitting. No compile-time check exists for the parens +// themselves -- without them the expansion silently goes wrong. +// +// Correct: +// FC_SERIALIZE_AS_STRING_TEMPLATE((typename T, typename U), (my_type)) +// +// Wrong (preprocessor splits on the first comma; expansion is malformed): +// FC_SERIALIZE_AS_STRING_TEMPLATE(typename T, typename U, my_type) +// +// Wrong (single-arg form -- no parens needed but easy to forget the wrapper): +// FC_SERIALIZE_AS_STRING_TEMPLATE((typename T), my_type) // missing parens on TYPE +#define FC_SERIALIZE_AS_STRING_TEMPLATE(TPL_PARAMS, TYPE) \ + namespace fc { template \ + struct serialize_as_string : std::true_type {}; } + +#define FC_SERIALIZE_AS_STRING_UNPAREN_(...) __VA_ARGS__ diff --git a/libraries/libfc/include/fc/static_variant.hpp b/libraries/libfc/include/fc/static_variant.hpp index 37f004ac3d..f811edb50c 100644 --- a/libraries/libfc/include/fc/static_variant.hpp +++ b/libraries/libfc/include/fc/static_variant.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -80,6 +81,15 @@ template void to_variant( const std::variant& s, fc::varian v = std::move(vars); } +/// JSON shape: 2-element array `[index, value]` matching the to_variant form. +template void to_json_stream( const std::variant& s, json_writer& w ) +{ + w.begin_array(); + w.value_uint64(static_cast(s.index())); + std::visit([&w](const auto& v) { fc::to_json_stream(v, w); }, s); + w.end_array(); +} + template void from_variant( const fc::variant& v, std::variant& s ) { auto ar = v.get_array(); diff --git a/libraries/libfc/include/fc/time.hpp b/libraries/libfc/include/fc/time.hpp index bce4d25a71..c1cd5a6883 100644 --- a/libraries/libfc/include/fc/time.hpp +++ b/libraries/libfc/include/fc/time.hpp @@ -4,6 +4,7 @@ #include #include #include +#include #ifdef _MSC_VER #pragma warning (push) @@ -45,6 +46,7 @@ namespace fc { class variant; void to_variant( const fc::microseconds&, fc::variant& ); void from_variant( const fc::variant& , fc::microseconds& ); + void to_json_stream( const fc::microseconds&, fc::json_writer& ); class time_point { public: diff --git a/libraries/libfc/include/fc/variant.hpp b/libraries/libfc/include/fc/variant.hpp index 35d6bd1434..852920f6fe 100644 --- a/libraries/libfc/include/fc/variant.hpp +++ b/libraries/libfc/include/fc/variant.hpp @@ -20,6 +20,8 @@ #include #include #include +#include +#include #include #include @@ -46,10 +48,26 @@ namespace fc class microseconds; template struct safe; + // Streaming JSON writer overloads for variant / variant_object; implemented in + // variant.cpp / variant_object.cpp. Walks the variant tree directly into the writer + // -- no intermediate fc::json::to_string string allocation. Lets reflected structs + // that embed a variant or variant_object field stream through the same writer as + // their other members. `max_depth` bounds the recursion the same way + // fc::json::to_string's yield does; a deeper tree throws instead of overflowing the + // stack. + void to_json_stream( const variant& v, json_writer& w, uint32_t max_depth = json_stream_max_depth ); + void to_json_stream( const variant_object& vo, json_writer& w, uint32_t max_depth = json_stream_max_depth ); + void to_json_stream( const mutable_variant_object& vo, json_writer& w, uint32_t max_depth = json_stream_max_depth ); + struct blob { std::vector data; }; void to_variant( const blob& var, fc::variant& vo ); void from_variant( const fc::variant& var, blob& vo ); + /// JSON shape: a base64-encoded string -- matches `to_variant(blob)` (which produces a string variant). Necessary + /// because `fc::blob` is FC_REFLECT'd as `(data)`, so without this explicit overload the reflector-based + /// `to_json_stream` would emit `{"data": ""}` (object form) instead. Callers like clio's + /// `result["abi"].as_blob()` expect the string form. + void to_json_stream( const blob& var, fc::json_writer& w ); template void to_variant( const boost::multi_index_container& s, fc::variant& v ); @@ -63,6 +81,20 @@ namespace fc template void to_variant( const std::variant& s, fc::variant& v ); template void from_variant( const fc::variant& v, std::variant& s ); + /// Explicit bool overload so std::variant elements of type bool route here, not into the + /// reflected `to_variant` template (which expects T to have an `FC_REFLECT`). Without + /// this overload the call resolves to the template once `fc/reflect/variant.hpp` is in + /// scope, then fails on `fc::reflector::visit`. + /// + /// Constrained on `T == bool` exactly so implicitly-convertible-to-bool types (raw + /// pointers, types with `operator bool()`, etc.) do NOT silently route here -- those + /// fall through to the reflected path and produce a loud build error if no reflector + /// exists. See `from_variant(variant&, bool&)` for the inverse (already type-safe by + /// virtue of taking `bool&`, which non-bools can't bind to). + template + requires std::is_same_v + void to_variant( const T& var, fc::variant& vo ) { vo = var; } + void to_variant( const uint8_t& var, fc::variant& vo ); void from_variant( const fc::variant& var, uint8_t& vo ); void to_variant( const int8_t& var, fc::variant& vo ); @@ -86,6 +118,10 @@ namespace fc void from_variant( const fc::variant& var, mutable_variant_object& vo ); void to_variant( const std::vector& var, fc::variant& vo ); void from_variant( const fc::variant& var, std::vector& vo ); + /// JSON shape: lowercase-hex string (matches to_variant which produces a string variant). + /// Concretely overrides the generic `to_json_stream(vector)` template (which emits an + /// array of int8s). Empty vector emits `""` to match the to_variant(empty_vec) result. + void to_json_stream( const std::vector& var, json_writer& w ); template void to_variant( const std::unordered_map& var, fc::variant& vo ); @@ -132,9 +168,11 @@ namespace fc void to_variant( const time_point& var, fc::variant& vo ); void from_variant( const fc::variant& var, time_point& vo ); + void to_json_stream( const time_point& var, json_writer& w ); void to_variant( const time_point_sec& var, fc::variant& vo ); void from_variant( const fc::variant& var, time_point_sec& vo ); + void to_json_stream( const time_point_sec& var, json_writer& w ); void to_variant( const microseconds& input_microseconds, fc::variant& output_variant ); void from_variant( const fc::variant& input_variant, microseconds& output_microseconds ); @@ -448,7 +486,17 @@ namespace fc void from_variant( const fc::variant& var, int32_t& vo ); /** @ingroup Serializable */ void from_variant( const fc::variant& var, uint32_t& vo ); + /** @ingroup Serializable */ + template + void to_json_stream( const std::optional& o, json_writer& w ) + { + // Top-level std::optional emits null when unset; reflected struct fields use the + // visitor's emit() overload that omits the field instead, mirroring to_variant_visitor::add. + if( o ) to_json_stream( *o, w ); + else w.value_null(); + } + template void from_variant( const variant& var, std::optional& vo ) { @@ -459,6 +507,43 @@ namespace fc from_variant( var, *vo ); } } + + /** @ingroup Serializable + * fc::nullable trio -- like std::optional except a disengaged value serializes + * as an explicit JSON null under its key instead of being omitted by the reflector + * visitors (see fc/nullable.hpp). Kept adjacent to the std::optional overloads so + * the two families' engaged-value shapes cannot drift. + */ + template + void to_variant( const nullable& n, variant& vo ) + { + if( n.value ) to_variant( *n.value, vo ); + else vo = variant(); + } + + template + void from_variant( const variant& var, nullable& n ) + { + if( var.is_null() ) { n.value.reset(); return; } + n.value = T(); + from_variant( var, *n.value ); + } + + template + void to_json_stream( const nullable& n, json_writer& w ) + { + if( n.value ) to_json_stream( *n.value, w ); + else w.value_null(); + } + + template + void to_json_stream( const std::unordered_set& var, json_writer& w ) + { + if( var.size() > MAX_NUM_ARRAY_ELEMENTS ) throw std::range_error( "too large" ); + w.begin_array(); + for( const auto& e : var ) to_json_stream( e, w ); + w.end_array(); + } template void to_variant( const std::unordered_set& var, fc::variant& vo ) { @@ -481,6 +566,16 @@ namespace fc } + template + void to_json_stream( const std::unordered_map& var, json_writer& w ) + { + // Mirrors fc::to_variant: emits as an array of [key, value] + // pairs (NOT a JSON object) so non-string keys round-trip cleanly. + if( var.size() > MAX_NUM_ARRAY_ELEMENTS ) throw std::range_error( "too large" ); + w.begin_array(); + for( const auto& kv : var ) to_json_stream( kv, w ); + w.end_array(); + } template void to_variant( const std::unordered_map& var, fc::variant& vo ) { @@ -502,6 +597,14 @@ namespace fc } template + void to_json_stream( const std::map& var, json_writer& w ) + { + if( var.size() > MAX_NUM_ARRAY_ELEMENTS ) throw std::range_error( "too large" ); + w.begin_array(); + for( const auto& kv : var ) to_json_stream( kv, w ); + w.end_array(); + } + template void to_variant( const std::map& var, fc::variant& vo ) { if( var.size() > MAX_NUM_ARRAY_ELEMENTS ) throw std::range_error( "too large" ); @@ -521,6 +624,14 @@ namespace fc vo.insert( itr->as< std::pair >() ); } + template + void to_json_stream( const std::multimap& var, json_writer& w ) + { + if( var.size() > MAX_NUM_ARRAY_ELEMENTS ) throw std::range_error( "too large" ); + w.begin_array(); + for( const auto& kv : var ) to_json_stream( kv, w ); + w.end_array(); + } template void to_variant( const std::multimap& var, fc::variant& vo ) { @@ -542,6 +653,14 @@ namespace fc } + template + void to_json_stream( const std::set& var, json_writer& w ) + { + if( var.size() > MAX_NUM_ARRAY_ELEMENTS ) throw std::range_error( "too large" ); + w.begin_array(); + for( const auto& e : var ) to_json_stream( e, w ); + w.end_array(); + } template void to_variant( const std::set& var, fc::variant& vo ) { @@ -565,15 +684,13 @@ namespace fc /** @ingroup Serializable */ template - void from_variant( const fc::variant& var, std::deque& tmp ) + void to_json_stream( const std::deque& t, json_writer& w ) { - const variants& vars = var.get_array(); - if( vars.size() > MAX_NUM_ARRAY_ELEMENTS ) throw std::range_error( "too large" ); - tmp.clear(); - for( auto itr = vars.begin(); itr != vars.end(); ++itr ) - tmp.push_back( itr->as() ); + if( t.size() > MAX_NUM_ARRAY_ELEMENTS ) throw std::range_error( "too large" ); + w.begin_array(); + for( const auto& e : t ) to_json_stream( e, w ); + w.end_array(); } - /** @ingroup Serializable */ template void to_variant( const std::deque& t, fc::variant& v ) @@ -584,20 +701,26 @@ namespace fc vars[i] = fc::variant(t[i]); v = std::move(vars); } - /** @ingroup Serializable */ - template - void from_variant( const fc::variant& v, boost::container::deque& d ) + template + void from_variant( const fc::variant& var, std::deque& tmp ) { - const variants& vars = v.get_array(); + const variants& vars = var.get_array(); if( vars.size() > MAX_NUM_ARRAY_ELEMENTS ) throw std::range_error( "too large" ); - d.clear(); - d.resize( vars.size() ); - for( uint32_t i = 0; i < vars.size(); ++i ) { - from_variant( vars[i], d[i] ); - } + tmp.clear(); + for( auto itr = vars.begin(); itr != vars.end(); ++itr ) + tmp.push_back( itr->as() ); } + /** @ingroup Serializable */ + template + void to_json_stream( const boost::container::deque& d, json_writer& w ) + { + if( d.size() > MAX_NUM_ARRAY_ELEMENTS ) throw std::range_error( "too large" ); + w.begin_array(); + for( const auto& e : d ) to_json_stream( e, w ); + w.end_array(); + } /** @ingroup Serializable */ template void to_variant( const boost::container::deque& d, fc::variant& vo ) @@ -609,19 +732,28 @@ namespace fc } vo = std::move( vars ); } - /** @ingroup Serializable */ - template - void from_variant( const fc::variant& var, std::vector& tmp ) + template + void from_variant( const fc::variant& v, boost::container::deque& d ) { - const variants& vars = var.get_array(); + const variants& vars = v.get_array(); if( vars.size() > MAX_NUM_ARRAY_ELEMENTS ) throw std::range_error( "too large" ); - tmp.clear(); - tmp.reserve( vars.size() ); - for( auto itr = vars.begin(); itr != vars.end(); ++itr ) - tmp.push_back( itr->as() ); + d.clear(); + d.resize( vars.size() ); + for( uint32_t i = 0; i < vars.size(); ++i ) { + from_variant( vars[i], d[i] ); + } } + /** @ingroup Serializable */ + template + void to_json_stream( const std::vector& v, json_writer& w ) + { + if( v.size() > MAX_NUM_ARRAY_ELEMENTS ) throw std::range_error( "too large" ); + w.begin_array(); + for( const auto& e : v ) to_json_stream( e, w ); + w.end_array(); + } /** @ingroup Serializable */ template void to_variant( const std::vector& t, fc::variant& v ) @@ -632,7 +764,26 @@ namespace fc vars[i] = fc::variant(t[i]); v = std::move(vars); } + /** @ingroup Serializable */ + template + void from_variant( const fc::variant& var, std::vector& tmp ) + { + const variants& vars = var.get_array(); + if( vars.size() > MAX_NUM_ARRAY_ELEMENTS ) throw std::range_error( "too large" ); + tmp.clear(); + tmp.reserve( vars.size() ); + for( auto itr = vars.begin(); itr != vars.end(); ++itr ) + tmp.push_back( itr->as() ); + } + /** @ingroup Serializable */ + template + void to_json_stream( const std::array& t, json_writer& w ) + { + w.begin_array(); + for( const auto& e : t ) to_json_stream( e, w ); + w.end_array(); + } /** @ingroup Serializable */ template void from_variant( const fc::variant& var, std::array& tmp ) @@ -650,6 +801,14 @@ namespace fc v = std::vector( static_cast(bi.data()), static_cast(bi.data()) + sizeof(bi) ); } + /// JSON shape mirrors to_variant: lowercase hex string (not the generic + /// std::array template's array-of-int8 form). + template + void to_json_stream( const std::array& bi, json_writer& w ) + { + w.value_hex( bi.data(), N ); + } + /** @ingroup Serializable */ template void from_variant( const variant& v, std::array& bi ) @@ -684,6 +843,16 @@ namespace fc v = std::move(vars); } + /** @ingroup Serializable */ + template + void to_json_stream( const std::pair& t, json_writer& w ) + { + // Mirrors to_variant: emits a 2-element array [first, second]. + w.begin_array(); + to_json_stream( t.first, w ); + to_json_stream( t.second, w ); + w.end_array(); + } /** @ingroup Serializable */ template void to_variant( const std::pair& t, fc::variant& v ) @@ -727,6 +896,15 @@ namespace fc else vo = nullptr; } + /// JSON shape: dereference and emit the pointee, matching to_variant; null + /// pointers emit JSON `null`. + template + void to_json_stream( const std::shared_ptr& var, json_writer& w ) + { + if( var ) to_json_stream( *var, w ); + else w.value_null(); + } + template void from_variant( const fc::variant& var, std::shared_ptr& vo ) { diff --git a/libraries/libfc/src/crypto/bls_private_key.cpp b/libraries/libfc/src/crypto/bls_private_key.cpp index b64d491449..ffd1881291 100644 --- a/libraries/libfc/src/crypto/bls_private_key.cpp +++ b/libraries/libfc/src/crypto/bls_private_key.cpp @@ -38,7 +38,7 @@ signature_shim::public_key_type signature_shim::recover(const sha256&) const { FC_THROW_EXCEPTION(fc::unsupported_exception, "BLS Signature Recovery is not supported"); } -static fc::sha256::uint64_array_type priv_parse_base64url(const std::string& base64urlstr) { +static fc::sha256::uint64_array_type priv_parse_base64url(std::string_view base64urlstr) { auto res = std::mismatch(bls::constants::bls_private_key_prefix.begin(), bls::constants::bls_private_key_prefix.end(), base64urlstr.begin()); @@ -49,7 +49,7 @@ static fc::sha256::uint64_array_type priv_parse_base64url(const std::string& bas return fc::crypto::bls::deserialize_base64url(data_str); } -private_key::private_key(const std::string& base64urlstr) +private_key::private_key(std::string_view base64urlstr) : _sk(priv_parse_base64url(base64urlstr)) {} std::string private_key::to_string() const { @@ -62,15 +62,4 @@ bool operator ==(const private_key& pk1, const private_key& pk2) { return pk1._sk == pk2._sk; } -} // fc::crypto::bls - -namespace fc { -void to_variant(const crypto::bls::private_key& var, variant& vo) { - vo = var.to_string(); -} - -void from_variant(const variant& var, crypto::bls::private_key& vo) { - vo = crypto::bls::private_key(var.as_string()); -} - -} // fc \ No newline at end of file +} // fc::crypto::bls \ No newline at end of file diff --git a/libraries/libfc/src/crypto/bls_public_key.cpp b/libraries/libfc/src/crypto/bls_public_key.cpp index d08ce4f7b3..2b492a28fe 100644 --- a/libraries/libfc/src/crypto/bls_public_key.cpp +++ b/libraries/libfc/src/crypto/bls_public_key.cpp @@ -31,7 +31,7 @@ namespace fc::crypto::bls { , _jacobian_montgomery_le(from_affine_bytes_le(_affine_non_montgomery_le)) { } - public_key::public_key(const std::string& base64urlstr) + public_key::public_key(std::string_view base64urlstr) : _affine_non_montgomery_le(deserialize_bls_base64url(base64urlstr)) , _jacobian_montgomery_le(from_affine_bytes_le(_affine_non_montgomery_le)) { } @@ -53,15 +53,3 @@ namespace fc::crypto::bls { } } // fc::crypto::bls - -namespace fc { - - void to_variant(const crypto::bls::public_key& var, variant& vo) { - vo = var.to_string(); - } - - void from_variant(const variant& var, crypto::bls::public_key& vo) { - vo = crypto::bls::public_key(var.as_string()); - } - -} // fc diff --git a/libraries/libfc/src/crypto/bls_signature.cpp b/libraries/libfc/src/crypto/bls_signature.cpp index 94f89ccfed..f260ddf220 100644 --- a/libraries/libfc/src/crypto/bls_signature.cpp +++ b/libraries/libfc/src/crypto/bls_signature.cpp @@ -36,7 +36,7 @@ signature::signature(bls::signature_data_span affine_non_montgomery_le) , _jacobian_montgomery_le(to_jacobian_montgomery_le(_affine_non_montgomery_le)) {} -signature::signature(const std::string& base64urlstr) +signature::signature(std::string_view base64urlstr) : _affine_non_montgomery_le(sig_parse_base64url(base64urlstr)) , _jacobian_montgomery_le(to_jacobian_montgomery_le(_affine_non_montgomery_le)) {} @@ -50,7 +50,7 @@ std::string signature::to_string() const { return to_string(_affine_non_montgomery_le); } -aggregate_signature::aggregate_signature(const std::string& base64_url_str) +aggregate_signature::aggregate_signature(std::string_view base64_url_str) : _jacobian_montgomery_le(signature::to_jacobian_montgomery_le(sig_parse_base64url(base64_url_str))) {} @@ -61,22 +61,3 @@ std::string aggregate_signature::to_string() const { } } // fc::crypto::bls - -namespace fc { - -void to_variant(const crypto::bls::signature& var, variant& vo) { - vo = var.to_string(); -} - -void from_variant(const variant& var, crypto::bls::signature& vo) { - vo = crypto::bls::signature(var.as_string()); -} - -void to_variant(const crypto::bls::aggregate_signature& var, variant& vo) { - vo = var.to_string(); -} - -void from_variant(const variant& var, crypto::bls::aggregate_signature& vo) { - vo = crypto::bls::aggregate_signature(var.as_string()); -} -} // fc \ No newline at end of file diff --git a/libraries/libfc/src/crypto/hex.cpp b/libraries/libfc/src/crypto/hex.cpp index be53018772..43e2034e77 100644 --- a/libraries/libfc/src/crypto/hex.cpp +++ b/libraries/libfc/src/crypto/hex.cpp @@ -42,13 +42,8 @@ size_t from_hex(std::string_view hex_str, char* out_data, size_t out_data_len) { } -std::vector from_hex(const std::string& hex, bool trim_prefix) { - auto cleaned_hex = trim_prefix ? trim_hex_prefix(hex) : hex; - if (cleaned_hex.size() % 2) { - cleaned_hex = "0" + cleaned_hex; - } - std::vector out; - out.reserve(cleaned_hex.size() / 2); +std::vector from_hex(std::string_view hex, bool trim_prefix) { + if (trim_prefix) hex = trim_hex_prefix(hex); auto from_hex = [](char c) -> int { if (c >= '0' && c <= '9') @@ -60,18 +55,25 @@ std::vector from_hex(const std::string& hex, bool trim_prefix) { throw std::runtime_error("Invalid hex char"); }; - for (size_t i = 0; i < cleaned_hex.size(); i += 2) { - uint8_t byte = - (from_hex(cleaned_hex[i]) << 4) | from_hex(cleaned_hex[i + 1]); - out.push_back(byte); + std::vector out; + out.reserve((hex.size() + 1) / 2); + + size_t i = 0; + if (hex.size() % 2) { + // Odd-length hex: the first nibble decodes to a single byte with high nibble = 0. + out.push_back(static_cast(from_hex(hex[0]))); + i = 1; + } + for (; i < hex.size(); i += 2) { + out.push_back(static_cast((from_hex(hex[i]) << 4) | from_hex(hex[i + 1]))); } return out; } -std::string trim_hex_prefix(const std::string& hex) { +std::string_view trim_hex_prefix(std::string_view hex) { if (hex.starts_with("0x") || hex.starts_with("0X")) { - return hex.substr(2); + hex.remove_prefix(2); } return hex; } diff --git a/libraries/libfc/src/crypto/keccak256.cpp b/libraries/libfc/src/crypto/keccak256.cpp index 6a7c1b47d3..7a01c16cef 100644 --- a/libraries/libfc/src/crypto/keccak256.cpp +++ b/libraries/libfc/src/crypto/keccak256.cpp @@ -10,7 +10,7 @@ keccak256::keccak256() { memset(_hash, 0, sizeof(_hash)); } -keccak256::keccak256(const std::string& hex_str) { +keccak256::keccak256(std::string_view hex_str) { auto bytes = from_hex(hex_str); FC_ASSERT(bytes.size() == sizeof(_hash), "Invalid keccak256 hex string length: {}", hex_str.size()); @@ -59,16 +59,12 @@ void keccak256::encoder::reset() { data.clear(); } -} // namespace fc::crypto - -namespace fc { - -void to_variant(const crypto::keccak256& bi, variant& v) { - v = bi.str(); -} + keccak256 keccak256::from_string(std::string_view s) { + // The pre-trait from_variant path (vector) rejected odd-length hex; + // keep that strictness so a trailing lone nibble is malformed input, not + // silently zero-extended. + FC_ASSERT(s.size() % 2 == 0, "keccak256 hex string length must be even, got {}", s.size()); + return keccak256(s); + } -void from_variant(const variant& v, crypto::keccak256& bi) { - bi = crypto::keccak256(v.as_string()); -} - -} // namespace fc \ No newline at end of file +} // namespace fc::crypto diff --git a/libraries/libfc/src/crypto/public_key.cpp b/libraries/libfc/src/crypto/public_key.cpp index 0b93fa14e4..73e198040a 100644 --- a/libraries/libfc/src/crypto/public_key.cpp +++ b/libraries/libfc/src/crypto/public_key.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include @@ -130,4 +131,10 @@ namespace fc { vo = fc::crypto::public_key::from_string(var.as_string()); } + + void to_json_stream(const fc::crypto::public_key& var, json_writer& w) + { + // Match to_variant's include-prefix=true output (eg "PUB_K1_..." / "PUB_WA_..."). + w.value_string(var.to_string(fc::yield_function_t(), true)); + } } // fc diff --git a/libraries/libfc/src/crypto/ripemd160.cpp b/libraries/libfc/src/crypto/ripemd160.cpp index 93bc4bead8..daf99fa634 100644 --- a/libraries/libfc/src/crypto/ripemd160.cpp +++ b/libraries/libfc/src/crypto/ripemd160.cpp @@ -9,12 +9,13 @@ #include #include #include "_digest_common.hpp" +#include namespace fc { ripemd160::ripemd160() { memset( _hash, 0, sizeof(_hash) ); } -ripemd160::ripemd160( const std::string& hex_str ) { +ripemd160::ripemd160( std::string_view hex_str ) { auto bytes_written = fc::from_hex( hex_str, (char*)_hash, sizeof(_hash) ); if( bytes_written < sizeof(_hash) ) memset( (char*)_hash + bytes_written, 0, (sizeof(_hash) - bytes_written) ); @@ -98,19 +99,12 @@ bool operator == ( const ripemd160& h1, const ripemd160& h2 ) { return memcmp( h1._hash, h2._hash, sizeof(h1._hash) ) == 0; } - void to_variant( const ripemd160& bi, variant& v ) - { - v = std::vector( (const char*)&bi, ((const char*)&bi) + sizeof(bi) ); - } - void from_variant( const variant& v, ripemd160& bi ) - { - std::vector ve = v.as< std::vector >(); - if( ve.size() ) - { - memcpy(bi.data(), ve.data(), fc::min(ve.size(),sizeof(bi)) ); - } - else - memset( bi.data(), char(0), sizeof(bi) ); + ripemd160 ripemd160::from_string(std::string_view s) { + // The pre-trait from_variant path (vector) rejected odd-length hex; + // keep that strictness so a trailing lone nibble is malformed input, not + // silently zero-extended. + FC_ASSERT(s.size() % 2 == 0, "ripemd160 hex string length must be even, got {}", s.size()); + return ripemd160(s); } } // fc diff --git a/libraries/libfc/src/crypto/sha1.cpp b/libraries/libfc/src/crypto/sha1.cpp index f95a15a353..71f35ee88e 100644 --- a/libraries/libfc/src/crypto/sha1.cpp +++ b/libraries/libfc/src/crypto/sha1.cpp @@ -6,12 +6,13 @@ #include #include #include "_digest_common.hpp" +#include namespace fc { sha1::sha1() { memset( _hash, 0, sizeof(_hash) ); } -sha1::sha1( const std::string& hex_str ) { +sha1::sha1( std::string_view hex_str ) { auto bytes_written = fc::from_hex( hex_str, (char*)_hash, sizeof(_hash) ); if( bytes_written < sizeof(_hash) ) memset( (char*)_hash + bytes_written, 0, (sizeof(_hash) - bytes_written) ); @@ -86,19 +87,12 @@ bool operator == ( const sha1& h1, const sha1& h2 ) { return memcmp( h1._hash, h2._hash, sizeof(h1._hash) ) == 0; } - void to_variant( const sha1& bi, variant& v ) - { - v = std::vector( (const char*)&bi, ((const char*)&bi) + sizeof(bi) ); - } - void from_variant( const variant& v, sha1& bi ) - { - std::vector ve = v.as< std::vector >(); - if( ve.size() ) - { - memcpy(bi.data(), ve.data(), fc::min(ve.size(),sizeof(bi)) ); - } - else - memset( bi.data(), char(0), sizeof(bi) ); + sha1 sha1::from_string(std::string_view s) { + // The pre-trait from_variant path (vector) rejected odd-length hex; + // keep that strictness so a trailing lone nibble is malformed input, not + // silently zero-extended. + FC_ASSERT(s.size() % 2 == 0, "sha1 hex string length must be even, got {}", s.size()); + return sha1(s); } } // fc diff --git a/libraries/libfc/src/crypto/sha224.cpp b/libraries/libfc/src/crypto/sha224.cpp index 8aa9ceaefc..2215caeb46 100644 --- a/libraries/libfc/src/crypto/sha224.cpp +++ b/libraries/libfc/src/crypto/sha224.cpp @@ -6,11 +6,12 @@ #include #include #include "_digest_common.hpp" +#include namespace fc { sha224::sha224() { memset( _hash, 0, sizeof(_hash) ); } - sha224::sha224( const std::string& hex_str ) { + sha224::sha224( std::string_view hex_str ) { auto bytes_written = fc::from_hex( hex_str, (char*)_hash, sizeof(_hash) ); if( bytes_written < sizeof(_hash) ) memset( (char*)_hash + bytes_written, 0, (sizeof(_hash) - bytes_written) ); @@ -81,21 +82,14 @@ namespace fc { return memcmp( h1._hash, h2._hash, sizeof(sha224) ) == 0; } - void to_variant( const sha224& bi, variant& v ) - { - v = std::vector( (const char*)&bi, ((const char*)&bi) + sizeof(bi) ); - } - void from_variant( const variant& v, sha224& bi ) - { - std::vector ve = v.as< std::vector >(); - if( ve.size() ) - { - memcpy(bi.data(), ve.data(), fc::min(ve.size(),sizeof(bi)) ); - } - else - memset( bi.data(), char(0), sizeof(bi) ); - } - template<> unsigned int hmac::internal_block_size() const { return 64; } + sha224 sha224::from_string(std::string_view s) { + // The pre-trait from_variant path (vector) rejected odd-length hex; + // keep that strictness so a trailing lone nibble is malformed input, not + // silently zero-extended. + FC_ASSERT(s.size() % 2 == 0, "sha224 hex string length must be even, got {}", s.size()); + return sha224(s); + } + } diff --git a/libraries/libfc/src/crypto/sha256.cpp b/libraries/libfc/src/crypto/sha256.cpp index 8d26739029..62360aaada 100644 --- a/libraries/libfc/src/crypto/sha256.cpp +++ b/libraries/libfc/src/crypto/sha256.cpp @@ -5,7 +5,6 @@ #include #include #include -#include #include #include "_digest_common.hpp" @@ -17,7 +16,7 @@ namespace fc { FC_THROW_EXCEPTION( exception, "sha256: size mismatch" ); memcpy(_hash, data, size ); } - sha256::sha256( const std::string& hex_str ) { + sha256::sha256( std::string_view hex_str ) { auto bytes_written = fc::from_hex( hex_str, (char*)_hash, sizeof(_hash) ); if( bytes_written < sizeof(_hash) ) memset( (char*)_hash + bytes_written, 0, (sizeof(_hash) - bytes_written) ); @@ -198,21 +197,6 @@ namespace fc { return lzbits; } - void to_variant( const sha256& bi, variant& v ) - { - v = std::vector( (const char*)&bi, ((const char*)&bi) + sizeof(bi) ); - } - void from_variant( const variant& v, sha256& bi ) - { - std::vector ve = v.as< std::vector >(); - if( ve.size() ) - { - memcpy(bi.data(), ve.data(), fc::min(ve.size(),sizeof(bi)) ); - } - else - memset( bi.data(), char(0), sizeof(bi) ); - } - uint64_t hash64(const char* buf, size_t len) { sha256 sha_value = sha256::hash(buf,len); @@ -221,4 +205,12 @@ namespace fc { template<> unsigned int hmac::internal_block_size() const { return 64; } + sha256 sha256::from_string(std::string_view s) { + // The pre-trait from_variant path (vector) rejected odd-length hex; + // keep that strictness so a trailing lone nibble is malformed input, not + // silently zero-extended. + FC_ASSERT(s.size() % 2 == 0, "sha256 hex string length must be even, got {}", s.size()); + return sha256(s); + } + } //end namespace fc diff --git a/libraries/libfc/src/crypto/sha3.cpp b/libraries/libfc/src/crypto/sha3.cpp index 70710567c9..4462d9b7ca 100644 --- a/libraries/libfc/src/crypto/sha3.cpp +++ b/libraries/libfc/src/crypto/sha3.cpp @@ -193,7 +193,7 @@ sha3::sha3(const char *data, size_t size) FC_THROW_EXCEPTION(exception, "sha3: size mismatch"); memcpy(_hash, data, size); } -sha3::sha3(const std::string &hex_str) +sha3::sha3(std::string_view hex_str) { auto bytes_written = fc::from_hex(hex_str, (char *)_hash, sizeof(_hash)); if (bytes_written < sizeof(_hash)) @@ -283,16 +283,12 @@ bool operator==(const sha3 &h1, const sha3 &h2) h1._hash[3] == h2._hash[3]; } -void to_variant(const sha3 &bi, variant &v) -{ - v = std::vector((const char *)&bi, ((const char *)&bi) + sizeof(bi)); -} -void from_variant(const variant &v, sha3 &bi) -{ - const auto &ve = v.as>(); - if (ve.size()) - memcpy(bi.data(), ve.data(), fc::min(ve.size(), sizeof(bi))); - else - memset(bi.data(), char(0), sizeof(bi)); -} + sha3 sha3::from_string(std::string_view s) { + // The pre-trait from_variant path (vector) rejected odd-length hex; + // keep that strictness so a trailing lone nibble is malformed input, not + // silently zero-extended. + FC_ASSERT(s.size() % 2 == 0, "sha3 hex string length must be even, got {}", s.size()); + return sha3(s); + } + } // namespace fc diff --git a/libraries/libfc/src/crypto/sha512.cpp b/libraries/libfc/src/crypto/sha512.cpp index 2a87d68b46..120dd1a3ad 100644 --- a/libraries/libfc/src/crypto/sha512.cpp +++ b/libraries/libfc/src/crypto/sha512.cpp @@ -6,11 +6,12 @@ #include #include #include "_digest_common.hpp" +#include namespace fc { sha512::sha512() { memset( _hash, 0, sizeof(_hash) ); } - sha512::sha512( const std::string& hex_str ) { + sha512::sha512( std::string_view hex_str ) { auto bytes_written = fc::from_hex( hex_str, (char*)_hash, sizeof(_hash) ); if( bytes_written < sizeof(_hash) ) memset( (char*)_hash + bytes_written, 0, (sizeof(_hash) - bytes_written) ); @@ -88,21 +89,14 @@ namespace fc { return memcmp( h1._hash, h2._hash, sizeof(h1._hash) ) == 0; } - void to_variant( const sha512& bi, variant& v ) - { - v = std::vector( (const char*)&bi, ((const char*)&bi) + sizeof(bi) ); - } - void from_variant( const variant& v, sha512& bi ) - { - std::vector ve = v.as< std::vector >(); - if( ve.size() ) - { - memcpy(bi.data(), ve.data(), fc::min(ve.size(),sizeof(bi)) ); - } - else - memset( bi.data(), char(0), sizeof(bi) ); - } - template<> unsigned int hmac::internal_block_size() const { return 128; } + sha512 sha512::from_string(std::string_view s) { + // The pre-trait from_variant path (vector) rejected odd-length hex; + // keep that strictness so a trailing lone nibble is malformed input, not + // silently zero-extended. + FC_ASSERT(s.size() % 2 == 0, "sha512 hex string length must be even, got {}", s.size()); + return sha512(s); + } + } diff --git a/libraries/libfc/src/crypto/signature.cpp b/libraries/libfc/src/crypto/signature.cpp index 1385ae97f0..fa50447699 100644 --- a/libraries/libfc/src/crypto/signature.cpp +++ b/libraries/libfc/src/crypto/signature.cpp @@ -2,6 +2,7 @@ #include #include #include +#include namespace fc { namespace crypto { struct hash_visitor : public fc::visitor { @@ -137,4 +138,10 @@ namespace fc { vo = fc::crypto::signature::from_string(var.as_string()); } + + void to_json_stream(const fc::crypto::signature& var, json_writer& w) + { + // Match to_variant's include-prefix=true output (eg "SIG_K1_..." / "SIG_WA_..."). + w.value_string(var.to_string(fc::yield_function_t(), true)); + } } // fc diff --git a/libraries/libfc/src/exception.cpp b/libraries/libfc/src/exception.cpp index 3f2f597987..960450bffa 100644 --- a/libraries/libfc/src/exception.cpp +++ b/libraries/libfc/src/exception.cpp @@ -2,6 +2,8 @@ #include #include #include +#include +#include #include #include @@ -105,6 +107,16 @@ namespace fc ( "stack", e.get_log() ); } + + void to_json_stream( const exception& e, json_writer& w ) + { + w.begin_object(); + w.set( "code", e.code() ); + w.set( "name", e.name() ); + w.set( "message", e.my->_what ); + w.set( "stack", e.get_log() ); + w.end_object(); + } void from_variant( const variant& v, exception& ll ) { const auto& obj = v.get_object(); diff --git a/libraries/libfc/src/int128.cpp b/libraries/libfc/src/int128.cpp index b27e7945d4..9b2bb0a94d 100644 --- a/libraries/libfc/src/int128.cpp +++ b/libraries/libfc/src/int128.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -83,6 +84,14 @@ void from_variant(const variant& var, fc::uint128& vo) { void to_variant(const fc::int128& var, fc::variant& vo) { vo = fc::to_string(var); } + +void to_json_stream(const fc::uint128& var, fc::json_writer& w) { + w.value_string(fc::to_string(var)); +} + +void to_json_stream(const fc::int128& var, fc::json_writer& w) { + w.value_string(fc::to_string(var)); +} void from_variant(const variant& var, fc::int128& vo) { if( var.is_int128() ) { vo = var.as_int128(); diff --git a/libraries/libfc/src/io/varint.cpp b/libraries/libfc/src/io/varint.cpp index a3eaac22aa..51a5d80270 100644 --- a/libraries/libfc/src/io/varint.cpp +++ b/libraries/libfc/src/io/varint.cpp @@ -1,4 +1,5 @@ #include +#include #include namespace fc @@ -7,4 +8,7 @@ void to_variant( const signed_int& var, variant& vo ) { vo = var.value; } void from_variant( const variant& var, signed_int& vo ) { vo.value = static_cast(var.as_int64()); } void to_variant( const unsigned_int& var, variant& vo ) { vo = var.value; } void from_variant( const variant& var, unsigned_int& vo ) { vo.value = static_cast(var.as_uint64()); } + +void to_json_stream( const signed_int& var, json_writer& w ) { w.value_int32(var.value); } +void to_json_stream( const unsigned_int& var, json_writer& w ) { w.value_uint32(var.value); } } diff --git a/libraries/libfc/src/log/log_message.cpp b/libraries/libfc/src/log/log_message.cpp index b7452e3704..f5502e5d66 100644 --- a/libraries/libfc/src/log/log_message.cpp +++ b/libraries/libfc/src/log/log_message.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace fc { @@ -106,6 +107,13 @@ namespace fc v = m.to_variant(); } + void to_json_stream( const log_message& m, json_writer& w ) + { + variant v; + to_variant(m, v); + to_json_stream(v, w); + } + void to_variant( log_level e, variant& v ) { switch( e ) diff --git a/libraries/libfc/src/network/url.cpp b/libraries/libfc/src/network/url.cpp index b96c9a1938..376c80064a 100644 --- a/libraries/libfc/src/network/url.cpp +++ b/libraries/libfc/src/network/url.cpp @@ -77,15 +77,6 @@ namespace fc }; } - void to_variant( const url& u, fc::variant& v ) - { - v = std::string(u); - } - void from_variant( const fc::variant& v, url& u ) - { - u = url( v.as_string() ); - } - url::operator std::string()const { std::stringstream ss; diff --git a/libraries/libfc/src/time.cpp b/libraries/libfc/src/time.cpp index 32f58085c6..3f46383ee3 100644 --- a/libraries/libfc/src/time.cpp +++ b/libraries/libfc/src/time.cpp @@ -1,6 +1,8 @@ #include #include #include +#include +#include #include #include #include @@ -87,12 +89,18 @@ namespace fc { void from_variant( const fc::variant& v, fc::time_point& t ) { t = fc::time_point::from_iso_string( v.as_string() ); } + void to_json_stream( const fc::time_point& t, json_writer& w ) { + w.value_string( t.to_iso_string() ); + } void to_variant( const fc::time_point_sec& t, variant& v ) { v = t.to_iso_string(); } void from_variant( const fc::variant& v, fc::time_point_sec& t ) { t = fc::time_point_sec::from_iso_string( v.as_string() ); } + void to_json_stream( const fc::time_point_sec& t, json_writer& w ) { + w.value_string( t.to_iso_string() ); + } // inspired by show_date_relative() in git's date.c std::string get_approximate_relative_time_string(const time_point_sec& event_time, @@ -165,6 +173,13 @@ namespace fc { { output_variant = input_microseconds.count(); } + void to_json_stream( const microseconds& us, json_writer& w ) + { + // Delegate to the guarded int64_t overload (fc/reflect/json_stream.hpp) so counts with + // magnitude past json_integer_quote_magnitude emit as quoted strings, exactly as the + // to_variant -> fc::json::to_string path renders the int64 count. + to_json_stream( us.count(), w ); + } void from_variant( const variant& input_variant, microseconds& output_microseconds ) { output_microseconds = microseconds(input_variant.as_int64()); diff --git a/libraries/libfc/src/variant.cpp b/libraries/libfc/src/variant.cpp index 8a68625ae9..ce2b25af10 100644 --- a/libraries/libfc/src/variant.cpp +++ b/libraries/libfc/src/variant.cpp @@ -8,9 +8,11 @@ #include #include #include +#include #include #include #include +#include #include #include @@ -785,6 +787,15 @@ bool variant::as_bool()const static std::string s_fc_to_string(double d) { + // Non-finite spellings are formatted explicitly rather than left to the C library the + // stringstream delegates to: glibc prints "-nan" for a negative NaN while Apple's libc + // drops the sign and prints "nan", so platform formatting would make the emitted bytes + // differ across platforms and diverge from the streaming path (fc/reflect/json_stream.hpp), + // which pins these exact shapes. NaN never compares, so its sign must come from signbit. + if( std::isnan( d ) ) + return std::signbit( d ) ? "-nan" : "nan"; + if( std::isinf( d ) ) + return std::signbit( d ) ? "-inf" : "inf"; // +2 is required to ensure that the double is rounded correctly when read back in. http://docs.oracle.com/cd/E19957-01/806-3568/ncg_goldberg.html std::stringstream ss; ss << std::setprecision(std::numeric_limits::digits10 + 2) << std::fixed << d; @@ -1069,6 +1080,12 @@ void to_variant( const std::vector& var, variant& vo ) vo = variant(to_hex(var.data(),var.size())); else vo = ""; } +void to_json_stream( const std::vector& var, json_writer& w ) +{ + FC_ASSERT( var.size() <= MAX_SIZE_OF_BYTE_ARRAYS ); + // value_hex with size 0 emits `""` already; no separate empty branch needed. + w.value_hex(var.data(), var.size()); +} void from_variant( const variant& var, std::vector& vo ) { std::string_view str = var.get_string(); @@ -1085,6 +1102,10 @@ void to_variant( const blob& b, variant& v ) { v = variant(base64_encode(b.data.data(), b.data.size())); } +void to_json_stream( const blob& b, json_writer& w ) { + w.value_string(base64_encode(b.data.data(), b.data.size())); +} + void from_variant( const variant& v, blob& b ) { b.data = base64_decode(v.as_string()); } @@ -1263,4 +1284,69 @@ std::string format_string( const std::string& frmt, const variant_object& args, } + void to_json_stream( const variant& v, json_writer& w, uint32_t max_depth ) { + // Walk the variant tree token-by-token directly into json_writer instead of + // building a JSON string via fc::json::to_string and splicing it back in. Mirrors + // the compact (indent=0) form of fc::json::to_string for byte-identical output. + FC_ASSERT( max_depth > 0, "to_json_stream: variant nesting exceeds max depth" ); + switch( v.get_type() ) { + case variant::null_type: + w.value_null(); + return; + case variant::int64_type: { + int64_t i = v.as_int64(); + if( i > json_integer_quote_magnitude || i < -json_integer_quote_magnitude ) { + // Quote out-of-range integers so 64-bit JS clients don't lose precision. + w.value_string( v.as_string() ); + } else { + w.value_int64( i ); + } + return; + } + case variant::uint64_type: { + uint64_t i = v.as_uint64(); + if( i > static_cast(json_integer_quote_magnitude) ) { + w.value_string( v.as_string() ); + } else { + w.value_uint64( i ); + } + return; + } + case variant::int128_type: + case variant::uint128_type: + case variant::int256_type: + case variant::uint256_type: + // Always quoted; representation is a decimal string already. + w.value_string( v.as_string() ); + return; + case variant::double_type: + // Quoted form matches fc::json::to_string's existing convention. + w.value_string( v.as_string() ); + return; + case variant::bool_type: + w.value_bool( v.as_bool() ); + return; + case variant::string_type: + case variant::string_sso_type: + w.value_string( v.get_string() ); + return; + case variant::blob_type: + // as_string() returns the base64-encoded form for blobs; matches the existing + // fc::json::to_string blob handling. + w.value_string( v.as_string() ); + return; + case variant::array_type: { + const variants& a = v.get_array(); + w.begin_array(); + for( const auto& e : a ) to_json_stream( e, w, max_depth - 1 ); + w.end_array(); + return; + } + case variant::object_type: + to_json_stream( v.get_object(), w, max_depth - 1 ); + return; + default: + FC_THROW_EXCEPTION( fc::invalid_arg_exception, "Unsupported variant type: {}", std::to_string( v.get_type() ) ); + } + } } // namespace fc diff --git a/libraries/libfc/src/variant_object.cpp b/libraries/libfc/src/variant_object.cpp index 2b86ed78f6..483eac2eef 100644 --- a/libraries/libfc/src/variant_object.cpp +++ b/libraries/libfc/src/variant_object.cpp @@ -1,6 +1,6 @@ #include #include - +#include namespace fc { @@ -504,6 +504,36 @@ namespace fc vo = variant(var); } + namespace { + // Shared walker for both object flavors (they iterate the same entry shape). + // Walks the key/value pairs directly into json_writer, mirroring the compact + // (indent=0) form of fc::json::to_string for byte-identical output; values + // recurse into to_json_stream(variant, w) which handles nested arrays / objects + // without going through fc::json::to_string at any level. `max_depth` bounds + // the recursion in lock-step with the variant walker. + template + void object_to_json_stream( const Object& vo, json_writer& w, uint32_t max_depth ) + { + FC_ASSERT( max_depth > 0, "to_json_stream: variant_object nesting exceeds max depth" ); + w.begin_object(); + for( const auto& kv : vo ) { + w.key( kv.key() ); + to_json_stream( kv.value(), w, max_depth - 1 ); + } + w.end_object(); + } + } + + void to_json_stream( const variant_object& vo, json_writer& w, uint32_t max_depth ) + { + object_to_json_stream( vo, w, max_depth ); + } + + void to_json_stream( const mutable_variant_object& vo, json_writer& w, uint32_t max_depth ) + { + object_to_json_stream( vo, w, max_depth ); + } + void from_variant( const variant& var, mutable_variant_object& vo ) { vo = var.get_object(); diff --git a/libraries/libfc/test/CMakeLists.txt b/libraries/libfc/test/CMakeLists.txt index ec1cc0e211..5679ffecd1 100644 --- a/libraries/libfc/test/CMakeLists.txt +++ b/libraries/libfc/test/CMakeLists.txt @@ -11,6 +11,8 @@ add_executable( test_fc crypto/test_webauthn.cpp io/test_cfile.cpp io/test_json.cpp + io/test_json_stream.cpp + test_move_only_function.cpp io/test_secure_file.cpp log/test_json_formatter.cpp io/test_json_variant.cpp diff --git a/libraries/libfc/test/crypto/test_hash_functions.cpp b/libraries/libfc/test/crypto/test_hash_functions.cpp index 3120a10a9b..7b21c2f779 100644 --- a/libraries/libfc/test/crypto/test_hash_functions.cpp +++ b/libraries/libfc/test/crypto/test_hash_functions.cpp @@ -1,8 +1,16 @@ #include #include -#include +#include +#include +#include #include +#include +#include +#include +#include +#include +#include #include #include @@ -12,6 +20,42 @@ using namespace fc; +namespace { + // Round-trip exercise for FC_SERIALIZE_AS_STRING-trait hash types. Verifies that: + // - Construction from a hex string round-trips through str() / to_string(). + // - The static T::from_string(string_view) factory matches construction. + // - The trait-routed fc::to_variant + fc::from_variant round-trip preserves the value + // and the variant's payload is the expected hex form. + // - fc::to_json_string emits the hex inside JSON quotes. + template + void check_hash_string_roundtrip(std::string_view hex) { + const Hash h{ hex }; + BOOST_CHECK_EQUAL(h.str(), hex); + BOOST_CHECK_EQUAL(h.to_string(), hex); + + const Hash h2 = Hash::from_string(hex); + BOOST_CHECK(h == h2); + + fc::variant v; + fc::to_variant(h, v); + BOOST_CHECK_EQUAL(v.as_string(), hex); + + Hash h3; + fc::from_variant(v, h3); + BOOST_CHECK(h == h3); + + const std::string expected_json = std::string{ "\"" } + std::string{ hex } + "\""; + BOOST_CHECK_EQUAL(fc::to_json_string(h), expected_json); + + // Odd-length hex is malformed input: from_string (and therefore the trait-routed + // fc::from_variant) must reject it rather than zero-extend the trailing nibble. + const std::string odd{ hex.substr(0, hex.size() - 1) }; + BOOST_CHECK_THROW(Hash::from_string(odd), fc::assert_exception); + Hash h4; + BOOST_CHECK_THROW(fc::from_variant(fc::variant{ odd }, h4), fc::assert_exception); + } +} + BOOST_AUTO_TEST_SUITE(hash_functions) BOOST_AUTO_TEST_CASE(sha3) try { @@ -79,6 +123,53 @@ BOOST_AUTO_TEST_CASE(keccak256) try { } FC_LOG_AND_RETHROW(); +BOOST_AUTO_TEST_CASE(sha1_string_roundtrip) try { + // 20 bytes / 40 hex chars + check_hash_string_roundtrip("0123456789abcdef0123456789abcdef01234567"); +} FC_LOG_AND_RETHROW(); + +BOOST_AUTO_TEST_CASE(sha224_string_roundtrip) try { + // 28 bytes / 56 hex chars + check_hash_string_roundtrip( + "0123456789abcdef0123456789abcdef0123456789abcdef01234567"); +} FC_LOG_AND_RETHROW(); + +BOOST_AUTO_TEST_CASE(sha256_string_roundtrip) try { + // 32 bytes / 64 hex chars + check_hash_string_roundtrip( + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"); +} FC_LOG_AND_RETHROW(); + +BOOST_AUTO_TEST_CASE(sha512_string_roundtrip) try { + // 64 bytes / 128 hex chars + check_hash_string_roundtrip( + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + "fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210"); +} FC_LOG_AND_RETHROW(); + +BOOST_AUTO_TEST_CASE(sha3_string_roundtrip) try { + // 32 bytes / 64 hex chars + check_hash_string_roundtrip( + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"); +} FC_LOG_AND_RETHROW(); + +BOOST_AUTO_TEST_CASE(ripemd160_string_roundtrip) try { + // 20 bytes / 40 hex chars + check_hash_string_roundtrip("0123456789abcdef0123456789abcdef01234567"); +} FC_LOG_AND_RETHROW(); + +BOOST_AUTO_TEST_CASE(keccak256_string_roundtrip) try { + // 32 bytes / 64 hex chars + check_hash_string_roundtrip( + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"); +} FC_LOG_AND_RETHROW(); + +BOOST_AUTO_TEST_CASE(blake3_string_roundtrip) try { + // 32 bytes / 64 hex chars + check_hash_string_roundtrip( + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"); +} FC_LOG_AND_RETHROW(); + /// Sanity-check that fc::crypto::keccak256 (the ethash-backed wrapper used by /// EM signature recovery and Ethereum tooling) returns the expected bytes for /// standard vectors. Catches wiring bugs in the wrapper independent of ethash diff --git a/libraries/libfc/test/io/test_json_stream.cpp b/libraries/libfc/test/io/test_json_stream.cpp new file mode 100644 index 0000000000..959ebe9934 --- /dev/null +++ b/libraries/libfc/test/io/test_json_stream.cpp @@ -0,0 +1,796 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +struct point_t { + int32_t x = 0; + int32_t y = 0; + std::string label; +}; + +struct nested_t { + std::string name; + std::vector values; + std::optional maybe; + point_t where; +}; + +enum class color_t { red, green, blue }; + +struct with_optional_t { + int32_t id = 0; + std::optional note; +}; + +struct with_nullable_t { + int32_t id = 0; + fc::nullable maybe_point; +}; + +struct with_map_t { + std::map counts; +}; + +struct block_like_t { + fc::time_point timestamp; + fc::sha256 digest; + std::string producer; +}; + +// int64_t/uint64_t alias different concrete types per platform (long on 64-bit Linux, long long +// on macOS), so exactly one of {size_t, long long, unsigned long long} is NOT one of the aliased +// fixed-width overloads on any given platform. A struct carrying all three exercises the +// platform-specific integer to_json_stream overload wherever the build runs (size_t on macOS; +// long long / unsigned long long on Linux) -- pins the fix for the get_unapplied_transactions_result +// size_t field that failed the macOS build. +struct platform_ints_t { + size_t sz = 0; + long long ll = 0; + unsigned long long ull = 0; +}; + +} // namespace + +FC_REFLECT(point_t, (x)(y)(label)) +FC_REFLECT(nested_t, (name)(values)(maybe)(where)) +FC_REFLECT_ENUM(color_t, (red)(green)(blue)) +FC_REFLECT(with_optional_t, (id)(note)) +FC_REFLECT(with_nullable_t, (id)(maybe_point)) +FC_REFLECT(with_map_t, (counts)) +FC_REFLECT(block_like_t, (timestamp)(digest)(producer)) +FC_REFLECT(platform_ints_t, (sz)(ll)(ull)) + +BOOST_AUTO_TEST_SUITE(json_stream_test) + +BOOST_AUTO_TEST_CASE(primitives) { + // Scalars serialize the same way fc::json::to_string(variant(v)) would. + BOOST_CHECK_EQUAL(fc::to_json_string(true), "true"); + BOOST_CHECK_EQUAL(fc::to_json_string(false), "false"); + BOOST_CHECK_EQUAL(fc::to_json_string(int32_t{0}), "0"); + BOOST_CHECK_EQUAL(fc::to_json_string(int32_t{-7}), "-7"); + BOOST_CHECK_EQUAL(fc::to_json_string(uint64_t{42}), "42"); + BOOST_CHECK_EQUAL(fc::to_json_string(std::string{"hi"}), "\"hi\""); +} + +BOOST_AUTO_TEST_CASE(array_of_char_emits_hex_string) { + // std::array serializes as a lowercase hex string in fc::variant + // (matches vector); streaming must produce the same shape, not an + // array of byte-numbers from the generic std::array template. + auto via_variant = [](const auto& v) { return fc::json::to_string(fc::variant(v), fc::json::yield_function_t()); }; + + std::array bytes{0x00, 0x01, 0x02, 0x03}; + BOOST_CHECK_EQUAL(fc::to_json_string(bytes), "\"00010203\""); + BOOST_CHECK_EQUAL(fc::to_json_string(bytes), via_variant(bytes)); + + std::array wider{}; + wider[0] = static_cast(0xff); wider[31] = 0x42; + BOOST_CHECK_EQUAL(fc::to_json_string(wider), via_variant(wider)); +} + +BOOST_AUTO_TEST_CASE(large_int_quoting_matches_variant) { + // Variant emits 64-bit integers with |v| > 0xffffffff as quoted strings to + // preserve precision past JS's 2^53 limit. Streaming must match. + auto via_variant = [](auto v) { return fc::json::to_string(fc::variant(v), fc::json::yield_function_t()); }; + + BOOST_CHECK_EQUAL(fc::to_json_string(int64_t{0xffffffff}), via_variant(int64_t{0xffffffff})); + BOOST_CHECK_EQUAL(fc::to_json_string(int64_t{0x100000000}), via_variant(int64_t{0x100000000})); + BOOST_CHECK_EQUAL(fc::to_json_string(int64_t{-0x100000000}), via_variant(int64_t{-0x100000000})); + BOOST_CHECK_EQUAL(fc::to_json_string(uint64_t{0xffffffff}), via_variant(uint64_t{0xffffffff})); + BOOST_CHECK_EQUAL(fc::to_json_string(uint64_t{0x100000000}), via_variant(uint64_t{0x100000000})); + BOOST_CHECK_EQUAL(fc::to_json_string(uint64_t{0xffffffffffffffff}), via_variant(uint64_t{0xffffffffffffffff})); +} + +BOOST_AUTO_TEST_CASE(value_double_locale_independent) { + // %g honors LC_NUMERIC: under de_DE the radix becomes ',' -> invalid JSON. + // value_double must use a locale-independent formatter. scoped_exit ensures + // the locale is restored even if a BOOST_CHECK throws. + const char* saved = std::setlocale(LC_NUMERIC, nullptr); + const std::string saved_locale = saved ? saved : "C"; + auto restore = fc::make_scoped_exit([&]{ std::setlocale(LC_NUMERIC, saved_locale.c_str()); }); + + const char* comma_locales[] = {"de_DE.UTF-8", "de_DE", "fr_FR.UTF-8", "fr_FR", "C-decimal-comma"}; + bool found = false; + for (const char* loc : comma_locales) { + if (std::setlocale(LC_NUMERIC, loc)) { found = true; break; } + } + if (!found) { + BOOST_TEST_MESSAGE("no comma-radix locale available; skipping locale-independence assertion"); + return; + } + + // Under a comma-radix locale the emitted number must (a) never contain a ',' radix + // and (b) match the variant path byte-for-byte -- doubles emit as a quoted + // fixed-precision string (digits10+2), so the check is against the reference + // emitter, not a hard-coded shortest-form spelling. std::from_chars is + // deliberately not used: AppleClang's libc++ does not implement it. + auto via_variant = [](double d) { + return fc::json::to_string(fc::variant(d), fc::json::yield_function_t()); + }; + for (double d : {1.5, -2.25, 0.1, 1.0 / 3.0, 1234.5, -9876.0}) { + const std::string s = fc::to_json_string(d); + BOOST_CHECK(s.find(',') == std::string::npos); // no comma radix leaked from LC_NUMERIC + BOOST_CHECK_EQUAL(s, via_variant(d)); // period radix + exact parity with fc::variant + } +} + +BOOST_AUTO_TEST_CASE(value_double_rejects_non_finite) { + // NaN / +-inf have no JSON representation; value_double must throw rather than + // emit unquoted nan/inf tokens that downstream parsers reject. + const auto nan = std::numeric_limits::quiet_NaN(); + const auto pos_inf = std::numeric_limits::infinity(); + const auto neg_inf = -std::numeric_limits::infinity(); + + std::string out; + { + fc::json_writer w(out); + BOOST_CHECK_THROW(w.value_double(nan), fc::assert_exception); + } + BOOST_CHECK(out.empty()); // nothing written, frame state untouched + { + fc::json_writer w(out); + BOOST_CHECK_THROW(w.value_double(pos_inf), fc::assert_exception); + BOOST_CHECK_THROW(w.value_double(neg_inf), fc::assert_exception); + } + // Finite values: to_json_stream(double) emits the quoted fixed-precision form + // matching the variant path's emission shape. + BOOST_CHECK_EQUAL(fc::to_json_string(0.0), "\"0.00000000000000000\""); + BOOST_CHECK_EQUAL(fc::to_json_string(1.5), "\"1.50000000000000000\""); +} + +BOOST_AUTO_TEST_CASE(string_escaping) { + // Control characters must be JSON-escaped. + BOOST_CHECK_EQUAL(fc::to_json_string(std::string{"a\"b"}), "\"a\\\"b\""); + BOOST_CHECK_EQUAL(fc::to_json_string(std::string{"a\nb"}), "\"a\\nb\""); + BOOST_CHECK_EQUAL(fc::to_json_string(std::string{"\t"}), "\"\\t\""); +} + +BOOST_AUTO_TEST_CASE(vector_and_optional) { + std::vector v{1, 2, 3}; + BOOST_CHECK_EQUAL(fc::to_json_string(v), "[1,2,3]"); + + std::optional unset; + BOOST_CHECK_EQUAL(fc::to_json_string(unset), "null"); + + std::optional set{7}; + BOOST_CHECK_EQUAL(fc::to_json_string(set), "7"); +} + +BOOST_AUTO_TEST_CASE(reflected_struct) { + point_t p{.x = 3, .y = -4, .label = "origin"}; + BOOST_CHECK_EQUAL(fc::to_json_string(p), "{\"x\":3,\"y\":-4,\"label\":\"origin\"}"); +} + +// The size_t / long long / unsigned long long fields must stream identically to the variant +// path, including the >2^32 quoting rule. Whichever of the three is the platform's "extra" +// 64-bit integer type hits the guarded overload in reflect/json_stream.hpp. +BOOST_AUTO_TEST_CASE(reflected_struct_platform_ints) { + platform_ints_t v{ .sz = 5, .ll = -8'000'000'000LL, .ull = 20'000'000'000ULL }; + fc::variant as_var; + fc::to_variant(v, as_var); + BOOST_CHECK_EQUAL(fc::to_json_string(v), + fc::json::to_string(as_var, fc::json::yield_function_t())); +} + +BOOST_AUTO_TEST_CASE(nested_struct) { + nested_t n{ + .name = "n", + .values = {1, 2}, + .maybe = std::nullopt, + .where = {.x = 0, .y = 1, .label = "p"} + }; + // `maybe` is std::nullopt so it's omitted per to_variant_visitor::add semantics. + BOOST_CHECK_EQUAL( + fc::to_json_string(n), + "{\"name\":\"n\",\"values\":[1,2],\"where\":{\"x\":0,\"y\":1,\"label\":\"p\"}}"); +} + +BOOST_AUTO_TEST_CASE(nested_optional_present) { + nested_t n{ + .name = "n", + .values = {}, + .maybe = 5, + .where = {.x = 0, .y = 0, .label = ""} + }; + BOOST_CHECK_EQUAL( + fc::to_json_string(n), + "{\"name\":\"n\",\"values\":[],\"maybe\":5,\"where\":{\"x\":0,\"y\":0,\"label\":\"\"}}"); +} + +BOOST_AUTO_TEST_CASE(enum_value) { + // FC_REFLECT_ENUM emits the enum name via reflector::to_fc_string. + BOOST_CHECK_EQUAL(fc::to_json_string(color_t::red), "\"red\""); + BOOST_CHECK_EQUAL(fc::to_json_string(color_t::green), "\"green\""); +} + +BOOST_AUTO_TEST_CASE(nullable_field_emits_explicit_null) { + // fc::nullable pins "key always present": a disengaged value serializes as an explicit + // JSON null under its key on BOTH paths, where a reflected std::optional omits the key + // entirely (see optional_field_omitted below). Endpoint schemas like get_finalizer_info + // depend on this shape. + auto via_variant = [](const with_nullable_t& v) { + fc::variant var; + fc::to_variant(v, var); + return fc::json::to_string(var, fc::json::yield_function_t()); + }; + + const with_nullable_t absent{.id = 1, .maybe_point = {}}; + BOOST_CHECK_EQUAL(fc::to_json_string(absent), R"({"id":1,"maybe_point":null})"); + BOOST_CHECK_EQUAL(fc::to_json_string(absent), via_variant(absent)); + + const with_nullable_t present{.id = 2, .maybe_point = point_t{.x = 3, .y = 4, .label = "p"}}; + BOOST_CHECK_EQUAL(fc::to_json_string(present), R"({"id":2,"maybe_point":{"x":3,"y":4,"label":"p"}})"); + BOOST_CHECK_EQUAL(fc::to_json_string(present), via_variant(present)); + + // from_variant mirrors the emission: null resets an engaged value, an object engages it. + fc::variant var; + fc::to_variant(absent, var); + with_nullable_t back{.id = 9, .maybe_point = point_t{}}; + fc::from_variant(var, back); + BOOST_CHECK_EQUAL(back.id, 1); + BOOST_CHECK(!back.maybe_point.has_value()); + + fc::to_variant(present, var); + fc::from_variant(var, back); + BOOST_REQUIRE(back.maybe_point.has_value()); + BOOST_CHECK_EQUAL(back.maybe_point->x, 3); + BOOST_CHECK_EQUAL(back.maybe_point->label, "p"); +} + +BOOST_AUTO_TEST_CASE(optional_field_omitted) { + // Unset optional field on a reflected struct is omitted entirely. + with_optional_t o{.id = 1, .note = std::nullopt}; + BOOST_CHECK_EQUAL(fc::to_json_string(o), "{\"id\":1}"); +} + +BOOST_AUTO_TEST_CASE(map_object_emission) { + with_map_t m; + m.counts["a"] = 1; + m.counts["b"] = 2; + // std::map iterates in sorted order so output is deterministic. Maps serialize as an + // array of [key, value] pairs (matching the long-standing to_variant_from_map shape), + // not a JSON object -- this preserves round-trip support for non-string key types. + BOOST_CHECK_EQUAL(fc::to_json_string(m), "{\"counts\":[[\"a\",1],[\"b\",2]]}"); +} + +BOOST_AUTO_TEST_CASE(raw_value_embeds_fragment) { + // raw_value should splice a pre-serialized JSON fragment at a value position. + std::string out; + { + fc::json_writer w(out); + w.begin_object(); + w.key("a"); + w.value_int32(1); + w.key("b"); + w.raw_value("{\"x\":42}"); + w.end_object(); + } + BOOST_CHECK_EQUAL(out, "{\"a\":1,\"b\":{\"x\":42}}"); +} + +BOOST_AUTO_TEST_CASE(array_of_reflected) { + std::vector pts{ + {.x = 1, .y = 2, .label = "a"}, + {.x = 3, .y = 4, .label = "b"}, + }; + BOOST_CHECK_EQUAL( + fc::to_json_string(pts), + "[{\"x\":1,\"y\":2,\"label\":\"a\"},{\"x\":3,\"y\":4,\"label\":\"b\"}]"); +} + +// -- fc type overloads -------------------------------------------------------------------- + +BOOST_AUTO_TEST_CASE(fc_microseconds) { + // fc::to_variant renders microseconds as the int64 count; match that here. + BOOST_CHECK_EQUAL(fc::to_json_string(fc::microseconds{0}), "0"); + BOOST_CHECK_EQUAL(fc::to_json_string(fc::microseconds{1'234'567}), "1234567"); + BOOST_CHECK_EQUAL(fc::to_json_string(fc::microseconds{-5}), "-5"); + + // Counts with magnitude past json_integer_quote_magnitude must emit quoted, exactly as + // the variant path renders the int64 count through fc::json::to_string. Boundary on + // both sides of +-0xffffffff (decimal literals: hex would be unsigned and break negation). + BOOST_CHECK_EQUAL(fc::to_json_string(fc::microseconds{4294967295}), "4294967295"); + BOOST_CHECK_EQUAL(fc::to_json_string(fc::microseconds{4294967296}), "\"4294967296\""); + BOOST_CHECK_EQUAL(fc::to_json_string(fc::microseconds{-4294967295}), "-4294967295"); + BOOST_CHECK_EQUAL(fc::to_json_string(fc::microseconds{-4294967296}), "\"-4294967296\""); + + auto via_variant = [](const fc::microseconds& us) { + fc::variant v; + fc::to_variant(us, v); + return fc::json::to_string(v, fc::json::yield_function_t()); + }; + for (int64_t count : {int64_t{0}, int64_t{4294967295}, int64_t{4294967296}, + int64_t{-4294967295}, int64_t{-4294967296}, + std::numeric_limits::min(), std::numeric_limits::max()}) { + BOOST_CHECK_EQUAL(fc::to_json_string(fc::microseconds{count}), via_variant(fc::microseconds{count})); + } +} + +BOOST_AUTO_TEST_CASE(fc_time_point_roundtrip_vs_variant) { + // Golden: JSON form must be identical to the existing to_variant -> json::to_string path + // so clients see no change when an endpoint migrates to the streaming writer. + const fc::time_point tp = fc::time_point::from_iso_string("2024-07-01T12:34:56.789"); + std::string stream_out = fc::to_json_string(tp); + fc::variant v; + fc::to_variant(tp, v); + std::string variant_out = fc::json::to_string(v, fc::json::yield_function_t()); + BOOST_CHECK_EQUAL(stream_out, variant_out); +} + +BOOST_AUTO_TEST_CASE(fc_time_point_sec_roundtrip_vs_variant) { + const fc::time_point_sec tps{ 1'720'000'000u }; + std::string stream_out = fc::to_json_string(tps); + fc::variant v; + fc::to_variant(tps, v); + std::string variant_out = fc::json::to_string(v, fc::json::yield_function_t()); + BOOST_CHECK_EQUAL(stream_out, variant_out); +} + +BOOST_AUTO_TEST_CASE(fc_sha256_hex_form) { + // sha256's canonical JSON form is its lowercase hex string (what .str() returns). + // to_variant stores raw bytes and relies on json::to_string base16-encoding at write + // time; to_json_stream shortcuts straight to the hex string, which must match. + const fc::sha256 h{ "0000000000000000000000000000000000000000000000000000000000000abc" }; + BOOST_CHECK_EQUAL( + fc::to_json_string(h), + "\"0000000000000000000000000000000000000000000000000000000000000abc\""); +} + +BOOST_AUTO_TEST_CASE(fc_variant_fallback) { + // fc::variant overload defers to fc::json::to_string; result must match the existing + // path byte-for-byte. + fc::variant v{ int64_t{42} }; + BOOST_CHECK_EQUAL(fc::to_json_string(v), + fc::json::to_string(v, fc::json::yield_function_t())); + v = std::string{"hello"}; + BOOST_CHECK_EQUAL(fc::to_json_string(v), + fc::json::to_string(v, fc::json::yield_function_t())); +} + +BOOST_AUTO_TEST_CASE(fc_variant_object_fallback) { + fc::mutable_variant_object mvo; + mvo("a", 1)("b", "two"); + fc::variant as_var = fc::variant(mvo); + BOOST_CHECK_EQUAL(fc::to_json_string(mvo), + fc::json::to_string(as_var, fc::json::yield_function_t())); +} + +// The variant walker bounds recursion the way fc::json::to_string's yield does: a tree +// nested deeper than fc::json_stream_max_depth throws instead of overflowing the stack. +BOOST_AUTO_TEST_CASE(fc_variant_walker_depth_guard) { + // json_stream_max_depth - 1 array wraps around a leaf emit cleanly (the leaf itself + // consumes the final depth level). + fc::variant nested{ std::string("leaf") }; + for (uint32_t i = 0; i < fc::json_stream_max_depth - 1; ++i) { + fc::variants wrap; + wrap.emplace_back(std::move(nested)); + nested = fc::variant(std::move(wrap)); + } + std::string out; + { + fc::json_writer w(out); + fc::to_json_stream(nested, w); + BOOST_REQUIRE(w.balanced()); + } + + // One more level pushes the leaf past the cap. + fc::variants wrap; + wrap.emplace_back(std::move(nested)); + fc::variant deep{ std::move(wrap) }; + std::string out2; + fc::json_writer w2(out2); + BOOST_CHECK_THROW(fc::to_json_stream(deep, w2), fc::assert_exception); +} + +// Composition test: a reflected struct that has fc::time_point and fc::sha256 fields. +BOOST_AUTO_TEST_CASE(reflector_with_fc_types) { + // The reflector-based path must find the to_json_stream overloads for fc::time_point + // and fc::sha256 via ordinary (namespace-scope) lookup. Without them, compilation + // would fail; with them, output matches the to_variant -> json::to_string golden path. + block_like_t b{ + .timestamp = fc::time_point::from_iso_string("2024-07-01T00:00:00.000"), + .digest = fc::sha256{ "deadbeef00000000000000000000000000000000000000000000000000000000" }, + .producer = "producer1", + }; + std::string stream_out = fc::to_json_string(b); + fc::variant v; + fc::to_variant(b, v); + std::string variant_out = fc::json::to_string(v, fc::json::yield_function_t()); + BOOST_CHECK_EQUAL(stream_out, variant_out); +} + +BOOST_AUTO_TEST_CASE(fc_public_key_and_signature) { + // Signatures + public keys serialize as their prefixed base58 string form ("PUB_K1_...", + // "SIG_K1_...") in existing JSON output. Round-trip via the streaming path and + // compare against the to_variant + json::to_string output. + const fc::crypto::private_key priv = fc::crypto::private_key::generate(); + const fc::crypto::public_key pub = priv.get_public_key(); + + fc::variant v_pub; + fc::to_variant(pub, v_pub); + BOOST_CHECK_EQUAL(fc::to_json_string(pub), + fc::json::to_string(v_pub, fc::json::yield_function_t())); + + const char* msg = "some-digest"; + const fc::crypto::signature sig = priv.sign(fc::sha256::hash(msg, std::strlen(msg))); + fc::variant v_sig; + fc::to_variant(sig, v_sig); + BOOST_CHECK_EQUAL(fc::to_json_string(sig), + fc::json::to_string(v_sig, fc::json::yield_function_t())); +} + +BOOST_AUTO_TEST_CASE(set_chained_object) { + // w.set("name", value) is sugar over w.key("name") + to_json_stream(value, w). + // Returns *this so call sites can chain. The expected output is byte-identical + // to the equivalent key()/value_*() sequence. + std::string out; + { + fc::json_writer w(out); + w.begin_object(); + w.set("a", 1) + .set("b", std::string{"two"}) + .set("c", true) + .set("d", 3.5); + w.end_object(); + } + BOOST_CHECK_EQUAL(out, R"({"a":1,"b":"two","c":true,"d":"3.50000000000000000"})"); +} + +BOOST_AUTO_TEST_CASE(set_dispatches_via_to_json_stream) { + // Confirms set picks up to_json_stream overloads for fc leaf types and reflected + // structs - same dispatch rule as the free-function fc::to_json_stream. + const fc::sha256 h = fc::sha256::hash(std::string("hello")); + point_t s{42, 7, "x"}; + std::vector nums{1, 2, 3}; + + std::string out; + { + fc::json_writer w(out); + w.begin_object(); + w.set("hash", h) // dispatches to_json_stream(sha256, w) + .set("inner", s) // dispatches via reflector path + .set("nums", nums); // dispatches to_json_stream(vector, w) + w.end_object(); + } + + // Cross-check against the variant + json::to_string output for the same shape. + fc::variant v_hash, v_inner, v_nums; + fc::to_variant(h, v_hash); + fc::to_variant(s, v_inner); + fc::to_variant(nums, v_nums); + const std::string expected = + std::string{"{\"hash\":"} + fc::json::to_string(v_hash, fc::json::yield_function_t()) + + ",\"inner\":" + fc::json::to_string(v_inner, fc::json::yield_function_t()) + + ",\"nums\":" + fc::json::to_string(v_nums, fc::json::yield_function_t()) + + "}"; + BOOST_CHECK_EQUAL(out, expected); +} + +// Parity helper: streaming JSON for `v` must equal the variant-built JSON. +// Catches divergence between to_json_stream(T) and to_variant(T)+json::to_string +// for libfc leaf types whose HTTP-API consumers depend on the existing shape. +template +static void check_streaming_matches_variant(const T& v, const std::string& label) { + const std::string streamed = fc::to_json_string(v); + const std::string via_var = fc::json::to_string(fc::variant(v), fc::json::yield_function_t()); + BOOST_CHECK_MESSAGE(streamed == via_var, + label << ": streaming JSON differs from variant path\n streaming: " << streamed + << "\n variant: " << via_var); +} + +BOOST_AUTO_TEST_CASE(streaming_vs_variant_parity_libfc_leaf_types) { + // exception: shape comes from FC_REFLECT(fc::exception, ...) -- stack of log_messages. + { + fc::exception e(FC_LOG_MESSAGE(error, "test exception"), + fc::std_exception_code, "test_exception", "test what"); + check_streaming_matches_variant(e, "fc::exception"); + } + // log_message: timestamp + context + formatted message. + { + fc::log_message lm(FC_LOG_CONTEXT(warn), std::string("hello world")); + check_streaming_matches_variant(lm, "fc::log_message"); + } + // 128-bit integers: emitted as decimal strings. + { + check_streaming_matches_variant(fc::int128_t{-1234567890123456789LL}, "fc::int128 negative"); + check_streaming_matches_variant(fc::uint128_t{0xffffffffffffffffULL} * 2 + 1, "fc::uint128 large"); + } + // double / float: variant emits a JSON-quoted fixed-precision string + // (digits10 + 2, std::fixed) so wire-format clients see a stable shape + // regardless of magnitude. Reflector-driven struct fields (e.g. + // get_producers_result.total_producer_vote_weight) depend on this. + { + check_streaming_matches_variant(double{1.5}, "double 1.5"); + check_streaming_matches_variant(double{-2.25}, "double -2.25"); + check_streaming_matches_variant(double{0.1}, "double 0.1 (round-trip)"); + check_streaming_matches_variant(double{1e10}, "double 1e10"); + check_streaming_matches_variant(double{0.0}, "double 0.0"); + check_streaming_matches_variant(float{3.5f}, "float 3.5"); + // Non-finite doubles: the variant path (s_fc_to_string) formats these explicitly as + // "nan" / "-nan" / "inf" / "-inf", platform-independent -- unlike the C library + // formatting it bypasses, which drops negative NaN's sign on Apple. NaN never + // compares, so its sign only reaches the output via signbit -- and arbitrary ABI + // float bit patterns can carry either sign. + const double quiet_nan = std::numeric_limits::quiet_NaN(); + const double inf = std::numeric_limits::infinity(); + check_streaming_matches_variant(quiet_nan, "double nan"); + check_streaming_matches_variant(std::copysign(quiet_nan, -1.0), "double -nan"); + check_streaming_matches_variant(inf, "double inf"); + check_streaming_matches_variant(-inf, "double -inf"); + check_streaming_matches_variant(std::copysign(std::numeric_limits::quiet_NaN(), -1.0f), + "float -nan"); + // Absolute pins on top of the parity checks: both paths must produce this exact + // spelling on every platform, not merely agree with each other. + BOOST_CHECK_EQUAL(fc::to_json_string(std::copysign(quiet_nan, -1.0)), "\"-nan\""); + BOOST_CHECK_EQUAL(fc::json::to_string(fc::variant(std::copysign(quiet_nan, -1.0)), + fc::json::yield_function_t()), "\"-nan\""); + } + // bls public_key + signature: derived from a deterministic seed via private_key::generate(). + // private_key itself has =delete'd to_json_stream so we don't include it here. + { + auto sk = fc::crypto::bls::private_key::generate(); + check_streaming_matches_variant(sk.get_public_key(), "fc::crypto::bls::public_key"); + const std::string msg = "ping"; + auto sig = sk.sign_raw(std::span(reinterpret_cast(msg.data()), msg.size())); + check_streaming_matches_variant(sig, "fc::crypto::bls::signature"); + } + // url: opaque string round-trip. + { + check_streaming_matches_variant(fc::url(std::string{"https://example.com/path?q=1"}), "fc::url"); + } + // bitset: '1'/'0' bit string via FC_SERIALIZE_AS_STRING. + { + check_streaming_matches_variant(fc::bitset(std::string_view{"1010110010"}), "fc::bitset"); + } + // enum_type<>: reflected enum should emit the member-name string, not the underlying integer. + { + using et = fc::enum_type; + check_streaming_matches_variant(et{color_t::green}, "fc::enum_type"); + } + // static_variant (std::variant alias): index + tagged value pair. + { + using sv_t = std::variant; + check_streaming_matches_variant(sv_t{int32_t{42}}, "static_variant alt 0"); + check_streaming_matches_variant(sv_t{std::string{"hi"}}, "static_variant alt 1"); + } + // vector: lowercase hex string. + { + std::vector v{0x00, 0x01, char(0xfe), char(0xff)}; + check_streaming_matches_variant(v, "std::vector"); + } + // blob: base64-encoded bytes. + { + fc::blob b{{'a', 'b', 'c'}}; + check_streaming_matches_variant(b, "fc::blob"); + } +} + +BOOST_AUTO_TEST_CASE(set_raw_splices_preformatted_fragment) { + // set_raw is "key + raw_value": for embedding a JSON fragment that was already + // serialized elsewhere (eg the abi_serializer + json::to_string path). + std::string out; + { + fc::json_writer w(out); + w.begin_object(); + w.set("name", std::string{"alice"}) + .set_raw("payload", R"({"a":1,"b":"two"})") + .set("done", true); + w.end_object(); + } + BOOST_CHECK_EQUAL(out, R"({"name":"alice","payload":{"a":1,"b":"two"},"done":true})"); +} + +// -- growth guard ------------------------------------------------------------------------- + +namespace { +/// Foreign (non-fc, non-std) exception type: models the http layer's budget abort +/// (sysio::detail::stream_response_budget_exceeded), which must fly through serializer +/// catch(fc::exception&) handlers untouched. +struct test_budget_abort {}; +} // namespace + +BOOST_AUTO_TEST_CASE(growth_guard_fires_each_stride) { + // The guard is consulted at token boundaries once the buffer has grown one stride past + // the previous invocation: observed sizes are strictly increasing by at least a stride. + std::string out; + std::vector observed; + { + fc::json_writer w(out, [&](size_t sz) { observed.push_back(sz); }, 16); + w.begin_array(); + for (int i = 0; i < 100; ++i) w.value_string("0123456789"); + w.end_array(); + } + BOOST_REQUIRE(!observed.empty()); + for (size_t i = 1; i < observed.size(); ++i) { + BOOST_CHECK_GE(observed[i], observed[i - 1] + 16); + } + + // Emissions smaller than one stride never invoke the guard: zero overhead for the + // typical small response. + std::string small; + size_t calls = 0; + { + fc::json_writer w(small, [&](size_t) { ++calls; }, 1u << 20); + w.value_string("tiny"); + } + BOOST_CHECK_EQUAL(calls, 0u); +} + +BOOST_AUTO_TEST_CASE(growth_guard_throw_aborts_emission) { + // A guard throw propagates out of the token call and stops the buffer growing: the + // emission loop below would append ~13KB unguarded, but aborts within a token or two + // of the guard's threshold. + std::string out; + fc::json_writer w(out, [&](size_t sz) { if (sz > 64) throw test_budget_abort{}; }, 8); + w.begin_array(); + auto emit_many = [&] { for (int i = 0; i < 1000; ++i) w.value_string("xxxxxxxxxx"); }; + BOOST_CHECK_THROW(emit_many(), test_budget_abort); + BOOST_CHECK_LT(out.size(), 200u); +} + +BOOST_AUTO_TEST_CASE(growth_guard_intra_token_abort) { + // A single oversized token must not materialize behind one token-boundary check. + // value_string pre-checks its known input length with the guard and aborts BEFORE + // emitting (or allocating) anything; value_hex and raw_value abort within a chunk of + // the guard's threshold while they stream. (Mid-loop coverage for string escaping -- + // where the output size is NOT known up front -- is growth_guard_measures_escape_expansion.) + constexpr size_t threshold = 64 * 1024; + // Abort bound for the chunked emitters: threshold + one stride of guarded emission + + // slack -- far below the full token size. + constexpr size_t abort_bound = threshold + 8 * 1024; + const std::string big(1u << 20, 'x'); // 1 MiB payload; hex form is 2 MiB + + { + std::string out; + fc::json_writer w(out, [&](size_t sz) { if (sz > threshold) throw test_budget_abort{}; }, 1024); + BOOST_CHECK_THROW(w.value_string(big), test_budget_abort); + BOOST_CHECK_EQUAL(out.size(), 0u); // rejected by the size pre-check: nothing emitted at all + } + { + std::string out; + fc::json_writer w(out, [&](size_t sz) { if (sz > threshold) throw test_budget_abort{}; }, 1024); + BOOST_CHECK_THROW(w.value_hex(big.data(), big.size()), test_budget_abort); + BOOST_CHECK_LT(out.size(), abort_bound); + } + { + std::string out; + fc::json_writer w(out, [&](size_t sz) { if (sz > threshold) throw test_budget_abort{}; }, 1024); + BOOST_CHECK_THROW(w.raw_value(big), test_budget_abort); + BOOST_CHECK_LT(out.size(), abort_bound); + } +} + +BOOST_AUTO_TEST_CASE(growth_guard_bounds_allocation) { + // A rejected token must not ALLOCATE its full size either: value_string / key pre-check + // the known input length with the guard BEFORE escape_string reserves capacity for it, + // so a 32 MiB string under a 64 KiB budget aborts with the buffer's capacity still at + // its initial slack -- not a ~32 MiB reservation that out.size() never reflects. + constexpr size_t threshold = 64 * 1024; + const std::string big(32u << 20, 'x'); // 32 MiB + { + std::string out; + fc::json_writer w(out, [&](size_t sz) { if (sz > threshold) throw test_budget_abort{}; }, 1024); + BOOST_CHECK_THROW(w.value_string(big), test_budget_abort); + BOOST_CHECK_LT(out.capacity(), threshold); // ~32 MiB before the size pre-check existed + BOOST_CHECK_EQUAL(out.size(), 0u); + } + { // oversized keys take the same pre-check path + std::string out; + fc::json_writer w(out, [&](size_t sz) { if (sz > threshold) throw test_budget_abort{}; }, 1024); + w.begin_object(); + BOOST_CHECK_THROW(w.key(big), test_budget_abort); + BOOST_CHECK_LT(out.capacity(), threshold); + } + { // an approved string keeps the single full-size reserve: no growth tax on the legit path + std::string out; + fc::json_writer w(out, [](size_t) { /* generous budget: never rejects */ }); + w.value_string(big); + BOOST_CHECK_EQUAL(out.size(), big.size() + 2); + BOOST_CHECK_GE(out.capacity(), big.size()); + } + { // unguarded writers are untouched: same output, same full reserve + std::string out; + fc::json_writer w(out); + w.value_string(big); + BOOST_CHECK_EQUAL(out.size(), big.size() + 2); + BOOST_CHECK_GE(out.capacity(), big.size()); + } +} + +BOOST_AUTO_TEST_CASE(growth_guard_measures_escape_expansion) { + // 32 KiB of control characters escape to ~192 KiB of output (6 bytes per input byte). + // The size pre-check approves the 32 KiB input (it is below the threshold), so this + // exercises the mid-loop machinery: a guard threshold above the input size but below + // the expanded size must still fire, proving the guard measures emitted bytes, not + // input consumed. + const std::string ctrl(32 * 1024, '\x01'); + std::string out; + fc::json_writer w(out, [&](size_t sz) { if (sz > 100 * 1024) throw test_budget_abort{}; }, 1024); + BOOST_CHECK_THROW(w.value_string(ctrl), test_budget_abort); + BOOST_CHECK_GT(out.size(), 100 * 1024); // the expanded output crossed a threshold the raw input never reaches + BOOST_CHECK_LT(out.size(), 128 * 1024); + // Allocation tracks the abort point, not some larger pre-computed estimate: capacity is + // the escape reserve for the approved input plus geometric growth to the abort. + BOOST_CHECK_LT(out.capacity(), 256 * 1024); +} + +BOOST_AUTO_TEST_CASE(growth_guard_rearms_after_throw) { + // A catch-and-continue serializer (eg the abi_serializer hex-fallback path) absorbing + // the guard's throw must NOT disable the guard: the very next token re-invokes it + // regardless of stride, so the abort re-raises until it unwinds out of the emitter. + // Once the guard passes again (budget freed), emission resumes normally. A rejected + // value_string mutates nothing (the size pre-check runs ahead of the separator), so + // absorb-and-continue leaves the buffer well-formed. + std::string out; + bool reject = false; + size_t calls = 0; + fc::json_writer w(out, [&](size_t) { ++calls; if (reject) throw test_budget_abort{}; }, 4); + w.begin_array(); + w.value_string("aaaaaaaaaa"); // pre-check crosses the stride; guard approves + BOOST_CHECK_EQUAL(calls, 1u); + reject = true; // budget exhausted + BOOST_CHECK_THROW(w.value_string("bbb"), test_budget_abort); // pre-check throws, nothing appended + BOOST_CHECK_EQUAL(calls, 2u); + BOOST_CHECK_THROW(w.value_string("ccc"), test_budget_abort); // absorbed upstream? re-fires immediately + BOOST_CHECK_EQUAL(calls, 3u); + reject = false; // budget freed: same retry now passes + w.value_string("ddd"); + w.end_array(); + BOOST_CHECK_EQUAL(calls, 4u); + BOOST_CHECK(w.balanced()); + BOOST_CHECK_EQUAL(out, R"(["aaaaaaaaaa","ddd"])"); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/libraries/libfc/test/test_bitset.cpp b/libraries/libfc/test/test_bitset.cpp index 9d86e556c0..6423b39e1c 100644 --- a/libraries/libfc/test/test_bitset.cpp +++ b/libraries/libfc/test/test_bitset.cpp @@ -1,8 +1,11 @@ #include #include +#include #include +#include + using namespace fc; BOOST_AUTO_TEST_SUITE(bitset_test_suite) @@ -167,4 +170,19 @@ BOOST_AUTO_TEST_CASE(bitset_test) test_bitset("01110011010100101111001101100100100111111111111111111001"); } +// to_string() drives every JSON path (FC_SERIALIZE_AS_STRING). Cap on the +// number of blocks must be enforced here so an oversize bitset can't allocate +// an unbounded result string. Threshold matches the pre-FC_SERIALIZE_AS_STRING +// to_variant guard (num_blocks > MAX_NUM_ARRAY_ELEMENTS). +BOOST_AUTO_TEST_CASE(bitset_to_string_caps_oversize) +{ + const size_t over_bits = (MAX_NUM_ARRAY_ELEMENTS + 1) * fc::bitset::bits_per_block; + fc::bitset bs(over_bits); + BOOST_CHECK_THROW(bs.to_string(), std::range_error); + + // Boundary: exactly MAX_NUM_ARRAY_ELEMENTS blocks must still succeed. + fc::bitset bs_at_cap(MAX_NUM_ARRAY_ELEMENTS * fc::bitset::bits_per_block); + BOOST_CHECK_NO_THROW(bs_at_cap.to_string()); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/libraries/libfc/test/test_move_only_function.cpp b/libraries/libfc/test/test_move_only_function.cpp new file mode 100644 index 0000000000..f0c4ae18d3 --- /dev/null +++ b/libraries/libfc/test/test_move_only_function.cpp @@ -0,0 +1,83 @@ +#include + +#include + +#include +#include + +// On libstdc++/newer libc++ the fc::move_only_function alias resolves to +// std::move_only_function, so the polyfill branch never compiles there. These cases +// instantiate fc::detail::move_only_function_impl directly so the polyfill gets +// compile + behavior coverage on every platform, not just AppleClang. +BOOST_AUTO_TEST_SUITE(move_only_function_tests) + +BOOST_AUTO_TEST_CASE(polyfill_void_nullary) { + int calls = 0; + fc::detail::move_only_function_impl f{ [&] { ++calls; } }; + BOOST_REQUIRE(static_cast(f)); + f(); + BOOST_CHECK_EQUAL(calls, 1); + + fc::detail::move_only_function_impl empty; + BOOST_CHECK(!static_cast(empty)); + fc::detail::move_only_function_impl null_constructed{ nullptr }; + BOOST_CHECK(!static_cast(null_constructed)); +} + +BOOST_AUTO_TEST_CASE(polyfill_returns_value_and_forwards_args) { + fc::detail::move_only_function_impl add{ [](int a, int b) { return a + b; } }; + BOOST_CHECK_EQUAL(add(2, 40), 42); + + // Move-only by-value argument must forward through the type-erasure boundary. + fc::detail::move_only_function_impl)> take{ + [](std::unique_ptr p) { return *p; } }; + BOOST_CHECK_EQUAL(take(std::make_unique(7)), 7); + + // Rvalue-reference parameter -- the shape chain::next_function stores. + fc::detail::move_only_function_impl&&)> sink{ + [](std::unique_ptr&& p) { auto consumed = std::move(p); BOOST_CHECK_EQUAL(*consumed, 9); } }; + sink(std::make_unique(9)); +} + +BOOST_AUTO_TEST_CASE(polyfill_move_only_capture_and_move_semantics) { + auto payload = std::make_unique(11); + fc::detail::move_only_function_impl f{ [p = std::move(payload)] { return *p; } }; + + // Move construction and move assignment transfer the callable. + fc::detail::move_only_function_impl g{ std::move(f) }; + BOOST_CHECK(!static_cast(f)); + BOOST_REQUIRE(static_cast(g)); + BOOST_CHECK_EQUAL(g(), 11); + + fc::detail::move_only_function_impl h; + h = std::move(g); + BOOST_REQUIRE(static_cast(h)); + BOOST_CHECK_EQUAL(h(), 11); +} + +BOOST_AUTO_TEST_CASE(polyfill_nested_shape) { + // Mirrors url_response_stream_callback: a move-only function taking a move-only + // function by value. + using inner_t = fc::detail::move_only_function_impl; + using outer_t = fc::detail::move_only_function_impl; + + int observed = 0; + outer_t outer{ [&](int code, inner_t emit) { + int v = code; + emit(v); + observed = v; + } }; + outer(200, inner_t{ [](int& v) { v += 22; } }); + BOOST_CHECK_EQUAL(observed, 222); +} + +BOOST_AUTO_TEST_CASE(alias_basic_usage) { + // Whichever implementation the alias selects on this platform, the common subset + // (construct from move-only capture, bool conversion, single invocation) must work. + auto payload = std::make_unique(5); + fc::move_only_function f{ [p = std::move(payload)] { return *p; } }; + BOOST_REQUIRE(static_cast(f)); + BOOST_CHECK_EQUAL(f(), 5); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/plugins/chain_api_plugin/src/chain_api_plugin.cpp b/plugins/chain_api_plugin/src/chain_api_plugin.cpp index 4231b0cdb8..782c060430 100644 --- a/plugins/chain_api_plugin/src/chain_api_plugin.cpp +++ b/plugins/chain_api_plugin/src/chain_api_plugin.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include #include @@ -89,28 +89,6 @@ parse_params(body);\ - fc::variant result( api_handle.call_name( std::move(params), deadline ) ); \ - cb(http_response_code, std::move(result)); \ - } catch (...) { \ - http_plugin::handle_exception(#api_name, #call_name, body, cb); \ - } \ - }} - -#define CHAIN_RO_CALL(call_name, http_response_code, params_type) CALL_WITH_400(chain, chain_ro, ro_api, chain_apis::read_only, call_name, http_response_code, params_type) -#define CHAIN_RW_CALL(call_name, http_response_code, params_type) CALL_WITH_400(chain, chain_rw, rw_api, chain_apis::read_write, call_name, http_response_code, params_type) -#define CHAIN_RO_CALL_POST(call_name, call_result, http_response_code, params_type) CALL_WITH_400_POST(chain, chain_ro, ro_api, chain_apis::read_only, call_name, call_result, http_response_code, params_type) -#define CHAIN_RO_CALL_ASYNC(call_name, call_result, http_response_code, params_type) CALL_ASYNC_WITH_400(chain, chain_ro, ro_api, chain_apis::read_only, call_name, call_result, http_response_code, params_type) -#define CHAIN_RW_CALL_ASYNC(call_name, call_result, http_response_code, params_type) CALL_ASYNC_WITH_400(chain, chain_rw, rw_api, chain_apis::read_write, call_name, call_result, http_response_code, params_type) - -#define CHAIN_RO_CALL_WITH_400(call_name, http_response_code, params_type) CALL_WITH_400(chain, chain_ro, ro_api, chain_apis::read_only, call_name, http_response_code, params_type) - void chain_api_plugin::plugin_startup() { dlog( "starting chain_api_plugin" ); my.reset(new chain_api_plugin_impl(app().get_plugin().chain())); @@ -123,56 +101,104 @@ void chain_api_plugin::plugin_startup() { ro_api.set_shorten_abi_errors( !http_plugin::verbose_errors() ); + using ro = chain_apis::read_only; + using rw = chain_apis::read_write; + using cat = api_category; + using pt = http_params_types; + // Run get_info on http thread only - _http_plugin.add_async_api({ - CALL_WITH_400(chain, node, ro_api, chain_apis::read_only, get_info, 200, http_params_types::no_params) + _http_plugin.add_async_api_stream({ + bind_stream<&ro::get_info, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_info", cat::node, pt::no_params, 200), }); - _http_plugin.add_api({ - CHAIN_RO_CALL(get_activated_protocol_features, 200, http_params_types::possible_no_params), - CHAIN_RO_CALL_POST(get_block, fc::variant, 200, http_params_types::params_required), // _POST because get_block() returns a lambda to be executed on the http thread pool - CHAIN_RO_CALL(get_block_info, 200, http_params_types::params_required), - CHAIN_RO_CALL(get_block_header_state, 200, http_params_types::params_required), - CHAIN_RO_CALL_POST(get_account, chain_apis::read_only::get_account_results, 200, http_params_types::params_required), - CHAIN_RO_CALL(get_code, 200, http_params_types::params_required), - CHAIN_RO_CALL(get_code_hash, 200, http_params_types::params_required), - CHAIN_RO_CALL(get_consensus_parameters, 200, http_params_types::no_params), - CHAIN_RO_CALL(get_abi, 200, http_params_types::params_required), - CHAIN_RO_CALL(get_raw_code_and_abi, 200, http_params_types::params_required), - CHAIN_RO_CALL(get_raw_abi, 200, http_params_types::params_required), - CHAIN_RO_CALL(get_finalizer_info, 200, http_params_types::no_params), - CHAIN_RO_CALL_POST(get_table_rows, chain_apis::read_only::get_table_rows_result, 200, http_params_types::params_required), - CHAIN_RO_CALL(get_table_by_scope, 200, http_params_types::params_required), - CHAIN_RO_CALL(get_currency_balance, 200, http_params_types::params_required), - CHAIN_RO_CALL(get_currency_stats, 200, http_params_types::params_required), - CHAIN_RO_CALL(get_producers, 200, http_params_types::params_required), - CHAIN_RO_CALL(get_producer_schedule, 200, http_params_types::no_params), - CHAIN_RO_CALL(get_required_keys, 200, http_params_types::params_required), - CHAIN_RO_CALL(get_transaction_id, 200, http_params_types::params_required), - // transaction related APIs will be posted to read_write queue after keys are recovered, they are safe to run in parallel until they post to the read_write queue - CHAIN_RO_CALL_ASYNC(compute_transaction, chain_apis::read_only::compute_transaction_results, 200, http_params_types::params_required), - CHAIN_RW_CALL_ASYNC(push_transaction, chain_apis::read_write::push_transaction_results, 202, http_params_types::params_required), - CHAIN_RW_CALL_ASYNC(push_transactions, chain_apis::read_write::push_transactions_results, 202, http_params_types::params_required), - CHAIN_RW_CALL_ASYNC(send_transaction, chain_apis::read_write::send_transaction_results, 202, http_params_types::params_required), - CHAIN_RW_CALL_ASYNC(send_transaction2, chain_apis::read_write::send_transaction_results, 202, http_params_types::params_required) + _http_plugin.add_api_stream({ + // transaction related APIs will be posted to read_write queue after keys are recovered, they are safe to run + // in parallel until they post to the read_write queue + bind_stream<&ro::compute_transaction, dispatch::async>( + _http_plugin, ro_api, "/v1/chain/compute_transaction", cat::chain_ro, pt::params_required, 200), + bind_stream<&rw::push_transaction, dispatch::async>( + _http_plugin, rw_api, "/v1/chain/push_transaction", cat::chain_rw, pt::params_required, 202), + bind_stream<&rw::push_transactions, dispatch::async>( + _http_plugin, rw_api, "/v1/chain/push_transactions", cat::chain_rw, pt::params_required, 202), + bind_stream<&rw::send_transaction, dispatch::async>( + _http_plugin, rw_api, "/v1/chain/send_transaction", cat::chain_rw, pt::params_required, 202), + bind_stream<&rw::send_transaction2, dispatch::async>( + _http_plugin, rw_api, "/v1/chain/send_transaction2", cat::chain_rw, pt::params_required, 202), + }, appbase::exec_queue::read_only); + + _http_plugin.add_api_stream({ + bind_stream<&ro::get_activated_protocol_features, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_activated_protocol_features", cat::chain_ro, pt::possible_no_params, 200), + bind_stream<&ro::get_block_header_state, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_block_header_state", cat::chain_ro, pt::params_required, 200), + bind_stream<&ro::get_code, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_code", cat::chain_ro, pt::params_required, 200), + bind_stream<&ro::get_code_hash, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_code_hash", cat::chain_ro, pt::params_required, 200), + bind_stream<&ro::get_consensus_parameters, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_consensus_parameters", cat::chain_ro, pt::no_params, 200), + bind_stream<&ro::get_abi, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_abi", cat::chain_ro, pt::params_required, 200), + bind_stream<&ro::get_raw_code_and_abi, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_raw_code_and_abi", cat::chain_ro, pt::params_required, 200), + bind_stream<&ro::get_raw_abi, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_raw_abi", cat::chain_ro, pt::params_required, 200), + bind_stream<&ro::get_finalizer_info, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_finalizer_info", cat::chain_ro, pt::no_params, 200), + bind_stream<&ro::get_table_by_scope, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_table_by_scope", cat::chain_ro, pt::params_required, 200), + bind_stream<&ro::get_currency_balance, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_currency_balance", cat::chain_ro, pt::params_required, 200), + bind_stream<&ro::get_currency_stats, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_currency_stats", cat::chain_ro, pt::params_required, 200), + bind_stream<&ro::get_producers, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_producers", cat::chain_ro, pt::params_required, 200), + bind_stream<&ro::get_producer_schedule, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_producer_schedule", cat::chain_ro, pt::no_params, 200), + bind_stream<&ro::get_required_keys, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_required_keys", cat::chain_ro, pt::params_required, 200), + bind_stream<&ro::get_transaction_id, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_transaction_id", cat::chain_ro, pt::params_required, 200), + // get_account returns Phase-2 closure (function()>) + bind_stream<&ro::get_account, dispatch::post>( + _http_plugin, ro_api, "/v1/chain/get_account", cat::chain_ro, pt::params_required, 200), + // get_table_rows_stream returns Phase-2 closure + bind_stream<&ro::get_table_rows_stream, dispatch::post_direct>( + _http_plugin, ro_api, "/v1/chain/get_table_rows", cat::chain_ro, pt::params_required, 200), }, appbase::exec_queue::read_only); if (chain.account_queries_enabled()) { - _http_plugin.add_async_api({ - CHAIN_RO_CALL_WITH_400(get_accounts_by_authorizers, 200, http_params_types::params_required), + _http_plugin.add_async_api_stream({ + bind_stream<&ro::get_accounts_by_authorizers, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_accounts_by_authorizers", + cat::chain_ro, pt::params_required, 200), }); } - _http_plugin.add_async_api({ + _http_plugin.add_async_api_stream({ // chain_plugin send_read_only_transaction will post to read_exclusive queue - CHAIN_RO_CALL_ASYNC(send_read_only_transaction, chain_apis::read_only::send_read_only_transaction_results, 200, http_params_types::params_required), - CHAIN_RO_CALL_WITH_400(get_raw_block, 200, http_params_types::params_required), - CHAIN_RO_CALL_WITH_400(get_block_header, 200, http_params_types::params_required) + bind_stream<&ro::send_read_only_transaction, dispatch::async>( + _http_plugin, ro_api, "/v1/chain/send_read_only_transaction", cat::chain_ro, pt::params_required, 200), + // get_block runs Phase 0 (block fetch) on the http pool then bounces to the read-only + // queue internally for the abi capture; see read_only::get_block_stream_async. + bind_stream<&ro::get_block_stream_async, dispatch::async>( + _http_plugin, ro_api, "/v1/chain/get_block", cat::chain_ro, pt::params_required, 200), + // get_block_info is fully off the read-only queue: fetch_block_header_by_number is + // thread-safe and the rest is CPU; see read_only::get_block_info_async. + bind_stream<&ro::get_block_info_async, dispatch::async>( + _http_plugin, ro_api, "/v1/chain/get_block_info", cat::chain_ro, pt::params_required, 200), + bind_stream<&ro::get_raw_block, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_raw_block", cat::chain_ro, pt::params_required, 200), + bind_stream<&ro::get_block_header, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_block_header", cat::chain_ro, pt::params_required, 200), }); if (chain.transaction_finality_status_enabled()) { - _http_plugin.add_api({ - CHAIN_RO_CALL_WITH_400(get_transaction_status, 200, http_params_types::params_required), + _http_plugin.add_api_stream({ + bind_stream<&ro::get_transaction_status, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_transaction_status", + cat::chain_ro, pt::params_required, 200), }, appbase::exec_queue::read_only); } diff --git a/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp b/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp index 1640c7e149..f68c7e5aa2 100644 --- a/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp +++ b/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -23,6 +24,7 @@ #include #include +#include #include #include @@ -81,9 +83,123 @@ namespace sysio { return abi_resolver(abi_serializer_cache_builder(make_resolver(db, max_time, throw_on_yield::no)).add_serializers(obj).get()); } + /// Phase-split abi capture for the streaming-cb response path. `captured_abis` + /// holds the raw `abi` byte-string for every account referenced in a block / + /// trace, copied out of chainbase on the calling (read-only-queue) thread. + /// Parsing the bytes into `abi_serializer` instances is the slow part and + /// happens later in `build_resolver_from_captured_abis`, which can run on any + /// thread (typically the http thread pool) since it does not touch chainbase. + /// + /// Empty entries (`raw_abis[name] == {}`) record "we looked, no abi" so that + /// we don't re-walk chainbase off the read-only queue if the same account + /// appears multiple times. + struct captured_abis { + std::unordered_map> raw_abis; + }; + + /// Walks the actions in `obj` (a `signed_block_ptr` or `transaction_trace_ptr`) + /// and copies each unique action.account's `abi` bytes out of chainbase. + /// Must be called from an `exec_queue::read_only` (or `read_write`) task so + /// that chainbase is not being mutated concurrently. + template + inline captured_abis capture_abis(const controller& control, const T& obj) { + captured_abis cache; + auto add = [&](const account_name& name) { + if (!name.good()) return; + if (cache.raw_abis.find(name) != cache.raw_abis.end()) return; + const auto* accnt = control.find_account_metadata(name); + if (accnt && accnt->abi.size() > 0) { + cache.raw_abis.emplace(name, + chain::vector{accnt->abi.data(), + accnt->abi.data() + accnt->abi.size()}); + } else { + cache.raw_abis.emplace(name, chain::vector{}); + } + }; + auto visit_actions = [&](const auto& actions) { + for (const auto& a : actions) add(a.account); + }; + if constexpr (std::is_same_v) { + for (const auto& receipt : obj->transactions) { + const auto& t = receipt.trx.get_transaction(); + visit_actions(t.actions); + visit_actions(t.context_free_actions); + } + } else { + // transaction_trace_ptr + for (const auto& trace : obj->action_traces) add(trace.act.account); + } + return cache; + } + + /// Parses each captured `abi` byte-string into an `abi_serializer` and returns + /// the resolver. CPU-bound; safe to run on any thread (no chainbase access). + /// On a per-account basis, an exception during `to_abi` / `abi_serializer` is + /// logged at debug level and the account's slot is left empty -- callers fall + /// back to the hex representation for actions with un-decodable ABI, matching + /// the prior `make_resolver(throw_on_yield::no)` behaviour. + inline abi_resolver build_resolver_from_captured_abis(const captured_abis& cache, + const fc::microseconds& max_time) { + chain::abi_serializer_cache_t serializers; + serializers.reserve(cache.raw_abis.size()); + for (const auto& [name, raw] : cache.raw_abis) { + if (raw.empty()) { + serializers.emplace(name, std::nullopt); + continue; + } + try { + chain::abi_def abi; + if (chain::abi_serializer::to_abi(raw, abi)) { + serializers.emplace(name, + chain::abi_serializer(std::move(abi), + chain::abi_serializer::create_yield_function(max_time))); + } else { + serializers.emplace(name, std::nullopt); + } + } catch (const fc::exception& e) { + dlog("failed to build abi_serializer for {}: {}", name, e.to_detail_string()); + serializers.emplace(name, std::nullopt); + } catch (const std::exception& e) { + dlog("failed to build abi_serializer for {}: {}", name, e.what()); + serializers.emplace(name, std::nullopt); + } catch (...) { + dlog("failed to build abi_serializer for {}: unknown exception", name); + serializers.emplace(name, std::nullopt); + } + } + return abi_resolver(std::move(serializers)); + } + + /// Heuristic byte-size estimate for memory budgeting. Counts the captured + /// raw-ABI bytes plus a small per-entry overhead -- close enough for the + /// trx_retry_db memory accounting without parsing each ABI eagerly. + inline size_t estimated_size(const captured_abis& cache) { + size_t total = 0; + for (const auto& [name, raw] : cache.raw_abis) { + total += sizeof(name) + raw.size(); + } + return total; + } + namespace chain_apis { struct empty{}; +/// Holds a transaction trace plus the ABI bytes captured at the moment the trace was produced. The trace's action data +/// is not eagerly serialized to JSON or to a variant tree -- instead, when the result is emitted, the captured ABI +/// bytes are parsed into an `abi_serializer` resolver on the http thread pool and the trace is emitted directly via +/// `abi_serializer::to_json_stream`. Compared to the prior `fc::variant processed` field this: +/// - moves the ABI parse + JSON emission off the main / chain thread; +/// - skips the intermediate variant tree allocation per response; +/// - preserves "ABIs at apply time, not fire time" semantics for `trx_retry_db` (the captured bytes are the +/// apply-time snapshot). +/// On individual actions whose contract has a corrupt ABI, the abi_serializer's streaming path falls back to hex for +/// that action's data -- same per-action fallback used by `signed_block`. +struct streamed_processed_trace { + chain::transaction_trace_ptr trace; + captured_abis raw_abis; + fc::microseconds max_serialization_time; +}; + struct linked_action { name account; std::optional action; @@ -145,6 +261,29 @@ class read_only : public api_base { const trx_finality_status_processing* trx_finality_status_proc; friend class api_base; + /// Phase-1 result for `get_table_rows` / `get_table_rows_stream`: the raw KV rows + /// collected from chainbase plus the metadata Phase 2 needs to ABI-decode them. + /// Kept as a private impl detail; the public interface returns variant- or + /// stream-emitting closures built around this struct. + struct raw_table_row { + std::vector key; + std::vector value; + chain::name payer; + }; + struct table_rows_phase1 { + chain::abi_def abi; + std::string tbl_name; + /// Resolved BE-key decode plan (one shape per declared key field); unset when the + /// table's key type is unrepresentable, in which case Phase 2 emits hex keys. + std::optional> key_shapes; + size_t scope_key_count = 0; + bool json = true; + bool show_payer = false; + bool more = false; + std::string next_key; + std::vector rows; + }; + public: static const string KEYi64; @@ -361,7 +500,23 @@ class read_only : public api_base { chain::signed_block_ptr get_raw_block(const get_raw_block_params& params, const fc::time_point& deadline) const; using get_block_params = get_raw_block_params; - std::function()> get_block(const get_block_params& params, const fc::time_point& deadline) const; + + using get_block_stream_emit_fn = std::function; + + /// Async streaming `/v1/chain/get_block` handler. Runs `get_raw_block` on the http thread pool instead of the + /// read-only queue. Threading sequence: + /// - Phase 0 (http thread pool): registration uses `add_async_api_stream`, so the api method body runs on the + /// http worker that handled the request. Does only `get_raw_block`, which is thread-safe per controller. + /// - Hop 1: the method posts onto `exec_queue::read_only` for the chainbase abi reads. + /// - Phase 1 (read-only queue): `capture_abis(block)` copies each unique account's raw `abi` bytes, then fires + /// the next callback with an http_fwd Phase-2 closure. + /// - Hop 2: bind_stream::async posts the http_fwd back to the http thread pool. + /// - Phase 2 (http thread pool): `build_resolver_from_captured_abis` parses the bytes (CPU-only) and constructs + /// the emit_fn. + /// - Phase 3 (http thread pool, asio-inlined): bind_stream wraps the emit_fn in a `to_json_stream(emit_fn, w)` + /// adapter and hands it to `cb`; emit_fn walks the block + emits JSON. + void get_block_stream_async(get_block_params params, + chain::next_function next) const; // call from app() thread abi_resolver get_block_serializers( const chain::signed_block_ptr& block, const fc::microseconds& max_time ) const; @@ -369,6 +524,8 @@ class read_only : public api_base { // call from any thread fc::variant convert_block( const chain::signed_block_ptr& block, abi_resolver& resolver ) const; + void convert_block_stream( const chain::signed_block_ptr& block, + abi_resolver& resolver, fc::json_writer& w ) const; struct get_block_header_params { string block_num_or_id; @@ -376,8 +533,8 @@ class read_only : public api_base { }; struct get_block_header_result { - chain::block_id_type id; - fc::variant signed_block_header; + chain::block_id_type id; + chain::signed_block_header signed_block_header; std::optional block_extensions; }; @@ -387,7 +544,27 @@ class read_only : public api_base { uint32_t block_num = 0; }; - fc::variant get_block_info(const get_block_info_params& params, const fc::time_point& deadline) const; + /// Typed result for `/v1/chain/get_block_info`. Field order matches the previous mutable_variant_object + /// shape so the streaming JSON output is byte-identical to the prior variant path. + struct get_block_info_results { + uint32_t block_num = 0; + uint16_t ref_block_num = 0; + chain::block_id_type id; + chain::block_timestamp_type timestamp; + chain::account_name producer; + chain::block_id_type previous; + chain::checksum256_type transaction_mroot; + chain::checksum256_type finality_mroot; + chain::qc_claim_t qc_claim; + std::vector producer_signatures; + uint32_t ref_block_prefix = 0; + }; + + /// `/v1/chain/get_block_info` -- async streaming so the body runs on the http thread pool, not the read-only queue. + /// `fetch_block_header_by_number` is thread-safe per controller and the rest is just CPU (id calc + struct build), + /// so the entire body is safe off the read-only queue. + void get_block_info_async(get_block_info_params params, + chain::next_function next) const; struct get_block_header_state_params { string block_num_or_id; @@ -436,6 +613,15 @@ class read_only : public api_base { /// `values_only` (strip the `{key, value, payer?}` wrapper). get_table_rows_return_t get_table_rows( const get_table_rows_params& params, const fc::time_point& deadline )const; + /// Streaming counterpart to `get_table_rows`. Phase 1 (main thread) runs the same row-collection + /// logic; Phase 2 (http thread pool) returns a closure that emits the response JSON directly into + /// an `fc::json_writer` via `abi_serializer::binary_to_json_stream`, eliminating per-row variant + /// allocations. `filter` is intentionally not honored on this path since it is in-tree-only and + /// requires a `fc::variant` to evaluate; in-tree callers continue using `get_table_rows()`. + using get_table_rows_stream_emit_fn = std::function; + using get_table_rows_stream_return_t = std::function()>; + get_table_rows_stream_return_t get_table_rows_stream( const get_table_rows_params& params, const fc::time_point& deadline )const; + struct get_table_by_scope_params { name code; // mandatory name table; // optional, act as filter @@ -483,8 +669,13 @@ class read_only : public api_base { }; struct get_finalizer_info_result { - fc::variant active_finalizer_policy; // current active policy - fc::variant pending_finalizer_policy; // current pending policy. Empty if not existing + // fc::nullable (not std::optional) pins the endpoint schema: both keys are always + // present and serialize as an explicit JSON null when the policy does not exist, + // matching the shape the earlier fc::variant-typed fields emitted. A reflected + // std::optional would omit the key entirely when disengaged, breaking clients that + // access the key before testing it for null. + fc::nullable active_finalizer_policy; // current active policy. null if not existing + fc::nullable pending_finalizer_policy; // current pending policy. null if not existing // Last tracked vote information for each of the finalizers in // active_finalizer_policy and pending_finalizer_policy. @@ -521,8 +712,8 @@ class read_only : public api_base { get_producer_schedule_result get_producer_schedule( const get_producer_schedule_params& params, const fc::time_point& deadline )const; struct compute_transaction_results { - chain::transaction_id_type transaction_id; - fc::variant processed; // "processed" is expected JSON for trxs in clio + chain::transaction_id_type transaction_id; + chain_apis::streamed_processed_trace processed; // "processed" is expected JSON for trxs in clio }; struct compute_transaction_params { @@ -532,8 +723,8 @@ class read_only : public api_base { void compute_transaction(compute_transaction_params params, chain::plugin_interface::next_function next ); struct send_read_only_transaction_results { - chain::transaction_id_type transaction_id; - fc::variant processed; + chain::transaction_id_type transaction_id; + chain_apis::streamed_processed_trace processed; }; struct send_read_only_transaction_params { fc::variant transaction; @@ -589,6 +780,12 @@ class read_only : public api_base { std::optional wasm_config; }; get_consensus_parameters_results get_consensus_parameters(const get_consensus_parameters_params&, const fc::time_point& deadline) const; + +private: + /// Phase 1 of `get_table_rows` / `get_table_rows_stream`: scans chainbase on the + /// main thread and returns the raw KV rows plus the ABI metadata Phase 2 needs. + /// Filter/post-pass options are applied by the caller's Phase 2 closure. + table_rows_phase1 collect_table_rows_phase1(const get_table_rows_params& params, const fc::time_point& deadline) const; }; class read_write : public api_base { @@ -625,7 +822,13 @@ class read_write : public api_base { void push_transactions(const push_transactions_params& params, chain::plugin_interface::next_function next); using send_transaction_params = push_transaction_params; - using send_transaction_results = push_transaction_results; + /// Distinct from `push_transaction_results` -- send_transaction's `processed` is emitted via streaming directly + /// from the trace + captured ABIs. push_transaction stays on `fc::variant` because it post-processes the variant + /// tree (inline_traces tree restructuring) before returning. + struct send_transaction_results { + chain::transaction_id_type transaction_id; + chain_apis::streamed_processed_trace processed; + }; void send_transaction(send_transaction_params params, chain::plugin_interface::next_function next); struct send_transaction2_params { @@ -649,6 +852,17 @@ class read_write : public api_base { constexpr const char dec[] = "dec"; constexpr const char hex[] = "hex"; +/// Queued-payload size (http_detail::source_size_estimate ADL customization point) for +/// trace-bearing results: the generic reflected raw-pack cannot traverse +/// streamed_processed_trace (it is emitted, never wire-packed), so report the dominant +/// retained memory directly -- per-action payload bytes plus the captured ABI bytes. +/// Keeps the results visible to --http-max-bytes-in-flight-mb while they sit captured in +/// a queued stream_emitter. +size_t queued_payload_size(const streamed_processed_trace& t); +inline size_t queued_payload_size(const read_only::compute_transaction_results& r) { return queued_payload_size(r.processed); } +inline size_t queued_payload_size(const read_only::send_read_only_transaction_results& r) { return queued_payload_size(r.processed); } +inline size_t queued_payload_size(const read_write::send_transaction_results& r) { return queued_payload_size(r.processed); } + } // namespace chain_apis class chain_plugin : public plugin { @@ -733,11 +947,16 @@ FC_REFLECT(sysio::chain_apis::read_only::get_activated_protocol_features_params, FC_REFLECT(sysio::chain_apis::read_only::get_activated_protocol_features_results, (activated_protocol_features)(more) ) FC_REFLECT(sysio::chain_apis::read_only::get_raw_block_params, (block_num_or_id)) FC_REFLECT(sysio::chain_apis::read_only::get_block_info_params, (block_num)) +FC_REFLECT(sysio::chain_apis::read_only::get_block_info_results, + (block_num)(ref_block_num)(id)(timestamp)(producer)(previous) + (transaction_mroot)(finality_mroot)(qc_claim) + (producer_signatures)(ref_block_prefix)) FC_REFLECT(sysio::chain_apis::read_only::get_block_header_state_params, (block_num_or_id)) FC_REFLECT(sysio::chain_apis::read_only::get_block_header_params, (block_num_or_id)(include_extensions)) FC_REFLECT(sysio::chain_apis::read_only::get_block_header_result, (id)(signed_block_header)(block_extensions)) FC_REFLECT( sysio::chain_apis::read_write::push_transaction_results, (transaction_id)(processed) ) +FC_REFLECT( sysio::chain_apis::read_write::send_transaction_results, (transaction_id)(processed) ) FC_REFLECT( sysio::chain_apis::read_write::send_transaction2_params, (return_failure_trace)(retry_trx)(retry_trx_num_blocks)(transaction) ) // `all_rows` and `filter` deliberately excluded -- gated to in-process callers so HTTP cannot trigger a full-table walk @@ -788,3 +1007,10 @@ FC_REFLECT( sysio::chain_apis::read_only::compute_transaction_results, (transact FC_REFLECT( sysio::chain_apis::read_only::send_read_only_transaction_params, (transaction)) FC_REFLECT( sysio::chain_apis::read_only::send_read_only_transaction_results, (transaction_id)(processed) ) FC_REFLECT( sysio::chain_apis::read_only::get_consensus_parameters_results, (chain_config)(wasm_config)) + +namespace fc { + /// JSON shape: emits the trace's reflected fields with action data decoded ABI-aware via the captured ABI bytes. + /// Builds a fresh `abi_resolver` from `t.raw_abis` and calls `abi_serializer::to_json_stream` -- + /// matching the prior `abi_serializer::to_variant` output but skipping the variant tree. + void to_json_stream(const sysio::chain_apis::streamed_processed_trace& t, fc::json_writer& w); +} // namespace fc diff --git a/plugins/chain_plugin/include/sysio/chain_plugin/trx_retry_db.hpp b/plugins/chain_plugin/include/sysio/chain_plugin/trx_retry_db.hpp index 1bd9b4b989..68350580e3 100644 --- a/plugins/chain_plugin/include/sysio/chain_plugin/trx_retry_db.hpp +++ b/plugins/chain_plugin/include/sysio/chain_plugin/trx_retry_db.hpp @@ -4,6 +4,8 @@ namespace sysio::chain_apis { +struct streamed_processed_trace; // defined in chain_plugin.hpp + /** * This class manages the ephemeral indices and data that provide the transaction retry feature. * It is designed to be run on an API node, as it only tracks incoming API transactions. @@ -47,7 +49,8 @@ class trx_retry_db { * @param next report result to user by calling next * @throws throw tx_resource_exhaustion if trx would exceeds max_mem_usage_size */ - void track_transaction( chain::packed_transaction_ptr ptrx, std::optional num_blocks, sysio::chain::next_function> next ); + void track_transaction( chain::packed_transaction_ptr ptrx, std::optional num_blocks, + sysio::chain::next_function> next ); /** * Attach to chain applied_transaction signal diff --git a/plugins/chain_plugin/src/chain_plugin.cpp b/plugins/chain_plugin/src/chain_plugin.cpp index 080b2cfcbb..36af32ec5b 100644 --- a/plugins/chain_plugin/src/chain_plugin.cpp +++ b/plugins/chain_plugin/src/chain_plugin.cpp @@ -2006,8 +2006,8 @@ fc::variant strip_scope_fields(fc::variant&& full_key, size_t count) { return fc::variant(std::move(stripped)); } -read_only::get_table_rows_return_t -read_only::get_table_rows( const read_only::get_table_rows_params& p, const fc::time_point& deadline ) const { +read_only::table_rows_phase1 +read_only::collect_table_rows_phase1( const read_only::get_table_rows_params& p, const fc::time_point& deadline ) const { abi_def abi = sysio::chain_apis::get_abi( db, p.code ); const auto& tbl = get_kv_table_def( abi, p.table ); @@ -2099,23 +2099,15 @@ read_only::get_table_rows( const read_only::get_table_rows_params& p, const fc:: // else: scope is empty — iterate ALL scopes (no prefix filtering) } - // Phase 1: Collect raw rows on the main thread. - // Phase 2 (the returned lambda): ABI-decode on the http thread pool. - struct raw_row { - std::vector key; - std::vector value; - name payer; - }; - struct http_params_t { - bool json; - bool show_payer; - bool more; - std::string next_key; - std::vector rows; - }; + // Phase 1: Collect raw rows on the main thread; the caller's Phase 2 closure + // ABI-decodes on the http thread pool. + using raw_row = raw_table_row; + using http_params_t = table_rows_phase1; bool show_payer = p.show_payer.has_value() && *p.show_payer; - http_params_t hp{ p.json, show_payer, false, {}, {} }; + http_params_t hp; + hp.json = p.json; + hp.show_payer = show_payer; const auto& d = db.db(); const auto& kv_idx = d.get_index(); @@ -2407,73 +2399,11 @@ read_only::get_table_rows( const read_only::get_table_rows_params& p, const fc:: } } - // Phase 2: ABI decode on http thread pool - return [p = std::move(hp), abi = std::move(abi), table_name = p.table, - key_shapes = std::move(key_shapes), - scope_key_count, - abi_serializer_max_time = abi_serializer_max_time, - shorten_abi_errors = shorten_abi_errors, - all_rows = p.all_rows, - values_only = p.values_only.value_or(false), - filter = p.filter]() mutable -> - chain::t_or_exception { - get_table_rows_result result; - result.more = p.more; - result.next_key = std::move(p.next_key); - - abi_serializer abis; - abis.set_abi(std::move(abi), abi_serializer::create_yield_function(abi_serializer_max_time)); - auto table_type = abis.get_table_type(table_name); - - for (auto& row : p.rows) { - fc::mutable_variant_object obj; - // For secondary queries, decode the primary key as the key field - if (p.json) { - try { - FC_ASSERT(key_shapes, "be_key_codec: key shape unresolved (unrepresentable key type); falling back to hex"); - auto full_key = chain::be_key_codec::decode_key( - row.key.data(), row.key.size(), *key_shapes); - obj["key"] = strip_scope_fields(std::move(full_key), scope_key_count); - } catch (...) { - obj["key"] = fc::to_hex(row.key.data(), row.key.size()); - } - } else { - obj["key"] = fc::to_hex(row.key.data(), row.key.size()); - } - // Decode value - if (p.json && !table_type.empty() && !row.value.empty()) { - try { - obj["value"] = abis.binary_to_variant(table_type, row.value, - abi_serializer::create_yield_function(abi_serializer_max_time), - shorten_abi_errors); - } catch (...) { - obj["value"] = fc::to_hex(row.value.data(), row.value.size()); - } - } else { - obj["value"] = fc::to_hex(row.value.data(), row.value.size()); - } - if (p.show_payer) - obj["payer"] = row.payer.to_string(); - result.rows.push_back(std::move(obj)); - } - - // --- Post-pass: C++-only wrapper behaviors. Same shape as the primary-path Phase 2 below. - if (all_rows) { result.more = false; result.next_key.clear(); } - if (values_only) { - for (auto& row : result.rows) { - if (!row.is_object()) continue; - const auto& row_obj = row.get_object(); - if (!row_obj.contains("value")) continue; - fc::variant value{row_obj["value"]}; - row = std::move(value); - } - } - if (filter.has_value()) { - const auto& predicate = *filter; - std::erase_if(result.rows, [&](const fc::variant& row) { return !predicate(row); }); - } - return result; - }; + hp.abi = std::move(abi); + hp.tbl_name = p.table; + hp.key_shapes = std::move(key_shapes); + hp.scope_key_count = scope_key_count; + return hp; } // --- Primary key query path --- @@ -2585,20 +2515,28 @@ read_only::get_table_rows( const read_only::get_table_rows_params& p, const fc:: } } - return [hp = std::move(hp), abi = std::move(abi), tbl_name = p.table, - key_shapes = std::move(key_shapes), - scope_key_count, + hp.abi = std::move(abi); + hp.tbl_name = p.table; + hp.key_shapes = std::move(key_shapes); + hp.scope_key_count = scope_key_count; + return hp; +} + +read_only::get_table_rows_return_t +read_only::get_table_rows( const read_only::get_table_rows_params& p, const fc::time_point& deadline ) const { + auto hp = collect_table_rows_phase1(p, deadline); + return [hp = std::move(hp), abi_serializer_max_time = abi_serializer_max_time, - shorten_abi_errors = shorten_abi_errors, - all_rows = p.all_rows, - values_only = p.values_only.value_or(false), - filter = p.filter]() mutable + shorten_abi_errors = shorten_abi_errors, + all_rows = p.all_rows, + values_only = p.values_only.value_or(false), + filter = p.filter]() mutable -> chain::t_or_exception { read_only::get_table_rows_result result; abi_serializer abis; - abis.set_abi(std::move(abi), abi_serializer::create_yield_function(abi_serializer_max_time)); - auto value_type = abis.get_table_type(tbl_name); + abis.set_abi(std::move(hp.abi), abi_serializer::create_yield_function(abi_serializer_max_time)); + auto value_type = abis.get_table_type(hp.tbl_name); for (auto& row : hp.rows) { fc::mutable_variant_object obj; @@ -2606,10 +2544,10 @@ read_only::get_table_rows( const read_only::get_table_rows_params& p, const fc:: // Decode key -- fall back to hex if BE decode fails if (hp.json) { try { - FC_ASSERT(key_shapes, "be_key_codec: key shape unresolved (unrepresentable key type); falling back to hex"); + FC_ASSERT(hp.key_shapes, "be_key_codec: key shape unresolved (unrepresentable key type); falling back to hex"); auto full_key = chain::be_key_codec::decode_key( - row.key.data(), row.key.size(), *key_shapes); - obj("key", strip_scope_fields(std::move(full_key), scope_key_count)); + row.key.data(), row.key.size(), *hp.key_shapes); + obj("key", strip_scope_fields(std::move(full_key), hp.scope_key_count)); } catch (...) { obj("key", fc::to_hex(row.key.data(), static_cast(row.key.size()))); } @@ -2618,7 +2556,7 @@ read_only::get_table_rows( const read_only::get_table_rows_params& p, const fc:: } // Decode value -- fall back to hex if ABI decode fails - if (hp.json) { + if (hp.json && !value_type.empty() && !row.value.empty()) { try { obj("value", abis.binary_to_variant(value_type, row.value, abi_serializer::create_yield_function(abi_serializer_max_time), @@ -2636,8 +2574,8 @@ read_only::get_table_rows( const read_only::get_table_rows_params& p, const fc:: result.rows.emplace_back(std::move(obj)); } - result.more = hp.more; - result.next_key = hp.next_key; + result.more = hp.more; + result.next_key = std::move(hp.next_key); // --- Post-pass: C++-only wrapper behaviors --- // `all_rows` walked the whole scan; a resume cursor is meaningless on the way out. @@ -2671,6 +2609,97 @@ read_only::get_table_rows( const read_only::get_table_rows_params& p, const fc:: }; } +read_only::get_table_rows_stream_return_t +read_only::get_table_rows_stream( const read_only::get_table_rows_params& p, const fc::time_point& deadline ) const { + SYS_ASSERT( !p.filter.has_value(), chain::contract_table_query_exception, + "get_table_rows_stream does not support the 'filter' predicate; use get_table_rows() instead" ); + auto hp = collect_table_rows_phase1(p, deadline); + return [hp = std::move(hp), + abi_serializer_max_time = abi_serializer_max_time, + shorten_abi_errors = shorten_abi_errors, + all_rows = p.all_rows, + values_only = p.values_only.value_or(false)]() mutable + -> chain::t_or_exception { + // Build the abi_serializer on the http thread pool so the on-wire emit + // closure doesn't pay the abi_def move-construction cost during emission. + auto abis = std::make_shared(); + abis->set_abi(std::move(hp.abi), abi_serializer::create_yield_function(abi_serializer_max_time)); + auto value_type = abis->get_table_type(hp.tbl_name); + + const bool effective_more = !all_rows && hp.more; + std::string effective_next_key = all_rows ? std::string{} : std::move(hp.next_key); + + return [hp = std::move(hp), + abis = std::move(abis), + value_type = std::move(value_type), + abi_serializer_max_time, + shorten_abi_errors, + values_only, + effective_more, + effective_next_key = std::move(effective_next_key)] + (fc::json_writer& w) mutable { + auto emit_value = [&](const std::vector& data) { + if (hp.json && !value_type.empty() && !data.empty()) { + // binary_to_json_stream writes tokens into `w` as it walks the ABI, so a + // mid-decode throw (truncated value, ABI/type drift) leaves a half-written + // fragment behind. Rewind every byte written before falling back so the hex + // string below lands at a clean value position instead of producing + // structurally invalid JSON on the wire. + const auto cp = w.checkpoint(); + try { + abis->binary_to_json_stream(value_type, data, w, + abi_serializer::create_yield_function(abi_serializer_max_time), + shorten_abi_errors); + return; + } catch (...) { + w.rewind(cp); + } + } + // Empty value or non-json mode: emit hex (matches the variant path's + // fc::variant(vector) -> to_hex on emit). + w.value_hex(data.data(), data.size()); + }; + + w.begin_object(); + w.key("rows"); + w.begin_array(); + for (auto& row : hp.rows) { + if (values_only) { + emit_value(row.value); + continue; + } + w.begin_object(); + w.key("key"); + if (hp.json) { + try { + FC_ASSERT(hp.key_shapes, "be_key_codec: key shape unresolved (unrepresentable key type); falling back to hex"); + auto full_key = chain::be_key_codec::decode_key( + row.key.data(), row.key.size(), *hp.key_shapes); + fc::to_json_stream(strip_scope_fields(std::move(full_key), hp.scope_key_count), w); + } catch (...) { + w.value_hex(row.key.data(), row.key.size()); + } + } else { + w.value_hex(row.key.data(), row.key.size()); + } + w.key("value"); + emit_value(row.value); + if (hp.show_payer) { + w.key("payer"); + w.value_string(row.payer.to_string()); + } + w.end_object(); + } + w.end_array(); + w.key("more"); + w.value_bool(effective_more); + w.key("next_key"); + w.value_string(effective_next_key); + w.end_object(); + }; + }; +} + read_only::get_table_by_scope_result read_only::get_table_by_scope( const read_only::get_table_by_scope_params& p, const fc::time_point& deadline )const { @@ -3030,11 +3059,10 @@ read_only::get_finalizer_info_result read_only::get_finalizer_info( const read_o std::set finalizer_keys; // Populate a particular finalizer policy - auto add_policy_to_result = [&](const finalizer_policy_ptr& from_policy, fc::variant& to_policy) { + auto add_policy_to_result = [&](const finalizer_policy_ptr& from_policy, + fc::nullable& to_policy) { if (from_policy) { - // Use string format of public key for easy uses - to_variant(*from_policy, to_policy); - + to_policy = *from_policy; for (const auto& f: from_policy->finalizers) { finalizer_keys.insert(f.public_key); } @@ -3120,17 +3148,47 @@ chain::signed_block_ptr read_only::get_raw_block(const read_only::get_raw_block_ return block; } -std::function()> read_only::get_block(const get_raw_block_params& params, const fc::time_point& deadline) const { - chain::signed_block_ptr block = get_raw_block(params, deadline); - - using return_type = t_or_exception; - return [this, - resolver = get_serializers_cache(db, block, abi_serializer_max_time), - block = std::move(block)]() mutable -> return_type { - try { - return convert_block(block, resolver); - } CATCH_AND_RETURN(return_type); - }; +void read_only::get_block_stream_async(get_block_params params, + chain::next_function next) const { + try { + // Phase 0 (http thread pool): block fetch. get_raw_block is thread-safe per + // controller. The deadline argument is unused by the current get_raw_block + // body; pass time_point::maximum so it's a no-op. + chain::signed_block_ptr block = get_raw_block(params, fc::time_point::maximum()); + + // Hop 1: post the chainbase abi capture onto the read-only queue. Phase 2 + // (parse + emit_fn build) returns to the http pool via the http_fwd alternative + // of next_function_variant. + app().executor().post(appbase::priority::medium_low, + appbase::exec_queue::read_only, + [this, block = std::move(block), + max_time = abi_serializer_max_time, + next = std::move(next)]() mutable { + try { + captured_abis abi_bytes = capture_abis(db, block); + + // Fire next with an http_fwd that builds Phase 2 + the emit_fn on + // the http pool. bind_stream::async sees the http_fwd alternative + // and posts it via post_http_thread_pool(). + next(std::function()>{ + [this, block = std::move(block), + abi_bytes = std::move(abi_bytes), + max_time]() mutable + -> chain::t_or_exception { + try { + auto resolver = build_resolver_from_captured_abis(std::move(abi_bytes), + max_time); + return get_block_stream_emit_fn{ + [this, block = std::move(block), + resolver = std::move(resolver)] + (fc::json_writer& w) mutable { + convert_block_stream(block, resolver, w); + }}; + } CATCH_AND_RETURN(chain::t_or_exception); + }}); + } CATCH_AND_CALL(next); + }); + } CATCH_AND_CALL(next); } read_only::get_block_header_result read_only::get_block_header(const read_only::get_block_header_params& params, const fc::time_point& deadline) const{ @@ -3156,7 +3214,7 @@ read_only::get_block_header_result read_only::get_block_header(const read_only:: } SYS_RETHROW_EXCEPTIONS(chain::block_id_type_exception, "Invalid block ID: {}", params.block_num_or_id) } SYS_ASSERT( header, unknown_block_exception, "Could not find block header: {}", params.block_num_or_id); - return { header->calculate_id(), fc::variant{*header}, {}}; + return { header->calculate_id(), std::move(*header), {}}; } else { signed_block_ptr block; if( block_num ) { @@ -3167,7 +3225,7 @@ read_only::get_block_header_result read_only::get_block_header(const read_only:: } SYS_RETHROW_EXCEPTIONS(chain::block_id_type_exception, "Invalid block ID: {}", params.block_num_or_id) } SYS_ASSERT( block, unknown_block_exception, "Could not find block header: {}", params.block_num_or_id); - return { block->calculate_id(), fc::variant{static_cast(*block)}, block->block_extensions}; + return { block->calculate_id(), static_cast(*block), block->block_extensions }; } } @@ -3189,32 +3247,56 @@ fc::variant read_only::convert_block( const chain::signed_block_ptr& block, abi_ ( "ref_block_prefix", ref_block_prefix ); } -fc::variant read_only::get_block_info(const read_only::get_block_info_params& params, const fc::time_point&) const { - - std::optional block; - try { - block = db.fetch_block_header_by_number( params.block_num ); - } catch (...) { - // assert below will handle the invalid block num - } +void read_only::convert_block_stream( const chain::signed_block_ptr& block, abi_resolver& resolver, fc::json_writer& w ) const { + chain::impl::stream_sink sink(w); + chain::impl::abi_traverse_context ctx(abi_serializer::create_depth_yield_function(), abi_serializer_max_time); - SYS_ASSERT( block, unknown_block_exception, "Could not find block: {}", params.block_num); + const auto block_id = block->calculate_id(); + const uint32_t ref_block_pfx = block_id._hash[1]; - const auto id = block->calculate_id(); - const uint32_t ref_block_prefix = id._hash[1]; + sink.begin_object(); + chain::impl::abi_to_variant::emit_signed_block_body(sink, *block, resolver, ctx); + sink.key("id"); + sink.emit(block_id); + sink.key("block_num"); + sink.value_uint64(block->block_num()); + sink.key("ref_block_prefix"); + sink.value_uint64(ref_block_pfx); + sink.end_object(); +} - return fc::mutable_variant_object () - ("block_num", block->block_num()) - ("ref_block_num", static_cast(block->block_num())) - ("id", id) - ("timestamp", block->timestamp) - ("producer", block->producer) - ("previous", block->previous) - ("transaction_mroot", block->transaction_mroot) - ("finality_mroot", block->finality_mroot) - ("qc_claim", block->qc_claim) - ("producer_signatures", block->producer_signatures) - ("ref_block_prefix", ref_block_prefix); +void read_only::get_block_info_async(get_block_info_params params, + chain::next_function next) const { + try { + // Runs on the http thread pool (registered via add_async_api_stream). + // fetch_block_header_by_number is thread-safe per controller, so the entire + // body stays off the read-only queue. + std::optional block; + try { + block = db.fetch_block_header_by_number(params.block_num); + } catch (...) { + // assert below will handle the invalid block num + } + + SYS_ASSERT(block, unknown_block_exception, "Could not find block: {}", params.block_num); + + const auto id = block->calculate_id(); + const uint32_t ref_block_prefix = id._hash[1]; + + next(get_block_info_results{ + .block_num = block->block_num(), + .ref_block_num = static_cast(block->block_num()), + .id = id, + .timestamp = block->timestamp, + .producer = block->producer, + .previous = block->previous, + .transaction_mroot = block->transaction_mroot, + .finality_mroot = block->finality_mroot, + .qc_claim = block->qc_claim, + .producer_signatures = block->producer_signatures, + .ref_block_prefix = ref_block_prefix, + }); + } CATCH_AND_CALL(next); } fc::variant read_only::get_block_header_state(const get_block_header_state_params& params, const fc::time_point&) const { @@ -3330,22 +3412,27 @@ void read_write::push_transaction(const read_write::push_transaction_params& par } static void push_recurse(read_write* rw, int index, const std::shared_ptr& params, const std::shared_ptr& results, const next_function& next) { - auto wrapped_next = [=](const next_function_variant& result) { + auto wrapped_next = [=](next_function_variant&& result) { if (std::holds_alternative(result)) { const auto& e = std::get(result); results->emplace_back( read_write::push_transaction_results{ transaction_id_type(), fc::mutable_variant_object( "error", e->to_detail_string() ) } ); } else if (std::holds_alternative(result)) { - const auto& r = std::get(result); - results->emplace_back( r ); + results->emplace_back( std::get(std::move(result)) ); } else { - assert(0); + // push_transaction's next_function carries an http_fwd alternative for the streaming + // dispatch path; push_transaction itself has not been migrated to emit one, so reaching + // this branch indicates a future API change that broke this assumption. Throw rather + // than assert so the failure is visible in NDEBUG builds. + SYS_THROW(plugin_exception, + "push_recurse: unsupported next_function_variant alternative (http_fwd). " + "push_transaction must emit either a fc::exception_ptr or push_transaction_results."); } size_t next_index = index + 1; if (next_index < params->size()) { push_recurse(rw, next_index, params, results, next ); } else { - next(*results); + next(read_write::push_transactions_results{*results}); } }; @@ -3404,12 +3491,12 @@ void api_base::send_transaction_gen(API &api, send_transaction_params_t params, if( retry && api.trx_retry.has_value() && !trx_trace_ptr->except) { // will be ack'ed via next later api.trx_retry->track_transaction( ptrx, retry_num_blocks, - [ptrx, next](const next_function_variant>& result ) { + [ptrx, next](const next_function_variant>& result ) { if( std::holds_alternative( result ) ) { next( std::get( result ) ); } else { - fc::variant& output = *std::get>( result ); - next( Result{ptrx->id(), std::move( output )} ); + auto& processed = *std::get>( result ); + next( Result{ptrx->id(), std::move( processed )} ); } } ); retried = true; @@ -3420,20 +3507,20 @@ void api_base::send_transaction_gen(API &api, send_transaction_params_t params, (void)retry_num_blocks; // ref variable to avoid compilation warning } if (!retried) { - // we are still on main thread here. The lambda passed to `next()` below will be executed on the http thread pool + // we are still on main thread here. The lambda passed to `next()` below will be executed on the + // http thread pool. Phase 1 (here, main thread): copy raw ABI bytes for accounts referenced + // in the trace. Phase 2 (http pool): build the streamed_processed_trace which defers ABI + // parsing + JSON emission to response-emit time. using return_type = t_or_exception; next([&api, trx_trace_ptr, - resolver = get_serializers_cache(api.db, trx_trace_ptr, api.abi_serializer_max_time)]() mutable { + raw_abis = sysio::capture_abis(api.db, trx_trace_ptr)]() mutable { try { - fc::variant output; - try { - abi_serializer::to_variant(*trx_trace_ptr, output, resolver, api.abi_serializer_max_time); - } catch( abi_exception& ) { - output = *trx_trace_ptr; - } const transaction_id_type& id = trx_trace_ptr->id; - return return_type(Result{id, std::move( output )}); + return return_type(Result{id, + chain_apis::streamed_processed_trace{ std::move(trx_trace_ptr), + std::move(raw_abis), + api.abi_serializer_max_time }}); } CATCH_AND_RETURN(return_type); }); } @@ -3650,7 +3737,11 @@ read_only::get_account_return_t read_only::get_account( const get_account_params }; http_params_t http_params; - if( abi_def abi; code_account != nullptr && abi_serializer::to_abi(code_account->abi, abi) ) { + // Phase 1 gate: ROA contract loaded with at least some abi bytes. We don't parse the + // abi here -- to_abi (deserialize) and abi_serializer construction both move to Phase 2 + // so the read-only queue isn't blocked on the ROA abi parse. The content lookups below + // (sysio.token balance + reslimit row) only need the raw bytes plus chainbase access. + if (code_account != nullptr && code_account->abi.size() > 0) { const auto token_code = "sysio.token"_n; @@ -3689,16 +3780,39 @@ read_only::get_account_return_t read_only::get_account( const get_account_params return {}; }; - http_params.total_resources = lookup_object("reslimit"_n, params.account_name); + http_params.total_resources = lookup_object("reslimit"_n, params.account_name); - return [http_params = std::move(http_params), result = std::move(result), abi=std::move(abi), shorten_abi_errors=shorten_abi_errors, - abi_serializer_max_time=abi_serializer_max_time]() mutable -> chain::t_or_exception { - auto yield = [&]() { return abi_serializer::create_yield_function(abi_serializer_max_time); }; - abi_serializer abis(std::move(abi), yield()); + // Capture raw ABI bytes for Phase 2; both to_abi (deserialize) and abi_serializer + // construction will run on the http thread pool. If parsing fails there, we skip + // the reslimit decode and return result with total_resources unset (preserving + // pre-refactor behaviour for the corrupt-abi case). + vector abi_bytes(code_account->abi.data(), + code_account->abi.data() + code_account->abi.size()); - if (http_params.total_resources) - result.total_resources = abis.binary_to_variant("reslimit", *http_params.total_resources, yield(), shorten_abi_errors); - return std::move(result); + return [http_params = std::move(http_params), result = std::move(result), + abi_bytes = std::move(abi_bytes), + shorten_abi_errors = shorten_abi_errors, + abi_serializer_max_time = abi_serializer_max_time]() mutable + -> chain::t_or_exception { + try { + // Mirror the outer SYS_RETHROW_EXCEPTIONS(account_query_exception, ...) so that a throw + // on the http thread is reported to the client with the same exception class clients + // were seeing pre-refactor. Without the inner wrap, CATCH_AND_RETURN would surface + // raw abi_exception / abi_serialization_deadline_exception instead. + try { + abi_def abi; + if (abi_serializer::to_abi(abi_bytes, abi)) { + auto yield = [&]() { return abi_serializer::create_yield_function(abi_serializer_max_time); }; + abi_serializer abis(std::move(abi), yield()); + if (http_params.total_resources) { + result.total_resources = abis.binary_to_variant("reslimit", + *http_params.total_resources, + yield(), shorten_abi_errors); + } + } + return std::move(result); + } SYS_RETHROW_EXCEPTIONS(chain::account_query_exception, "unable to retrieve account info") + } CATCH_AND_RETURN(chain::t_or_exception); }; } return [result = std::move(result)]() mutable -> chain::t_or_exception { @@ -3810,6 +3924,29 @@ read_only::get_consensus_parameters(const get_consensus_parameters_params&, cons return results; } +namespace { + /// Dominant retained memory of a transaction trace held by a queued response closure: + /// per-action payload bytes, console output, and return values. A reservation-grade + /// estimate, deliberately ignoring small fixed-size fields -- same semantics as + /// detail::in_flight_sizeof. + size_t trace_retained_size(const chain::transaction_trace& trace) { + size_t sz = sizeof(trace); + for (const auto& at : trace.action_traces) { + sz += sizeof(at) + at.act.data.size() + at.console.size() + at.return_value.size(); + } + return sz; + } +} + +size_t queued_payload_size(const streamed_processed_trace& t) { + size_t sz = 0; + if (t.trace) + sz += trace_retained_size(*t.trace); + for (const auto& [account, abi] : t.raw_abis.raw_abis) + sz += abi.size(); + return sz; +} + } // namespace chain_apis fc::variant chain_plugin::get_log_trx_trace(const transaction_trace_ptr& trx_trace ) const { @@ -3842,4 +3979,27 @@ const controller::config& chain_plugin::chain_config() const { } } // namespace sysio +namespace fc { +void to_json_stream(const sysio::chain_apis::streamed_processed_trace& t, fc::json_writer& w) { + if (!t.trace) { + w.value_null(); + return; + } + // Checkpoint before the streaming attempt so a mid-emit throw can rewind the writer + // (truncate buffer + restore frame state) before the reflector-only fallback runs. + // Without this, the fallback would continue writing into a partially-emitted object, + // producing malformed JSON. + const auto cp = w.checkpoint(); + try { + auto resolver = sysio::build_resolver_from_captured_abis(t.raw_abis, t.max_serialization_time); + sysio::chain::abi_serializer::to_json_stream(*t.trace, w, resolver, t.max_serialization_time); + } catch (sysio::chain::abi_exception&) { + w.rewind(cp); + // Fall back: emit via reflector with no ABI-aware action data decode. Matches the prior variant path's + // `output = *trx_trace_ptr;` fallback. + to_json_stream(*t.trace, w); + } +} +} // namespace fc + FC_REFLECT( sysio::chain_apis::detail::ram_market_exchange_state_t, (ignore1)(ignore2)(ignore3)(core_symbol)(ignore4) ) diff --git a/plugins/chain_plugin/src/trx_retry_db.cpp b/plugins/chain_plugin/src/trx_retry_db.cpp index 8c40ea41c7..d28858bd14 100644 --- a/plugins/chain_plugin/src/trx_retry_db.cpp +++ b/plugins/chain_plugin/src/trx_retry_db.cpp @@ -27,12 +27,13 @@ namespace { constexpr uint16_t lib_totem = std::numeric_limits::max(); struct tracked_transaction { - const packed_transaction_ptr ptrx; - const uint16_t num_blocks = 0; // lib is lib_totem - uint32_t block_num = 0; - fc::variant trx_trace_v; - fc::time_point last_try; - next_function> next; + const packed_transaction_ptr ptrx; + const uint16_t num_blocks = 0; // lib is lib_totem + uint32_t block_num = 0; + chain::transaction_trace_ptr trace_ptr; + sysio::captured_abis raw_abis; + fc::time_point last_try; + next_function> next; const transaction_id_type& id()const { return ptrx->id(); } fc::time_point_sec expiry()const { return ptrx->expiration(); } @@ -52,7 +53,29 @@ struct tracked_transaction { return block_num != 0; } - size_t memory_size()const { return ptrx->get_estimated_size() + trx_trace_v.estimated_size() + sizeof(*this); } + /// Heuristic for the memory budget. Sums the dominant heap contributors so the + /// retry container's eviction loop fires when the real footprint -- not just the + /// packed-trx size -- exceeds the cap. Per-action heap (console / data / return_value) + /// dwarfs the packed bytes for chatty contracts, so we walk the action_traces. + size_t memory_size()const { + const size_t ptrx_sz = ptrx->get_estimated_size(); + size_t trace_sz = 0; + if (trace_ptr) { + trace_sz = sizeof(*trace_ptr); + for (const auto& at : trace_ptr->action_traces) { + trace_sz += sizeof(at) + + at.console.size() + + at.act.data.size() + + at.return_value.size() + + at.act.authorization.size() * sizeof(chain::permission_level) + + at.account_ram_deltas.size() * sizeof(chain::account_delta); + } + } + return ptrx_sz + + sysio::estimated_size(raw_abis) + + trace_sz + + sizeof(*this); + } }; struct by_trx_id; @@ -105,14 +128,16 @@ struct trx_retry_db_impl { return _tracked_trxs.index().size(); } - void track_transaction( packed_transaction_ptr ptrx, std::optional num_blocks, next_function> next ) { + void track_transaction( packed_transaction_ptr ptrx, std::optional num_blocks, + next_function> next ) { SYS_ASSERT( _tracked_trxs.memory_size() < _max_mem_usage_size, tx_resource_exhaustion, - "Transaction exceeded transaction-retry-max-storage-size-gb limit: {} bytes", _tracked_trxs.memory_size() ); + "Transaction exceeded transaction-retry-max-storage-size-gb limit: {} bytes", _tracked_trxs.memory_size() ); auto i = _tracked_trxs.index().get().find( ptrx->id() ); if( i == _tracked_trxs.index().end() ) { _tracked_trxs.insert( {std::move(ptrx), !num_blocks.has_value() ? lib_totem : *num_blocks, 0, + nullptr, {}, fc::time_point::now(), std::move(next)} ); @@ -133,19 +158,15 @@ struct trx_retry_db_impl { auto& idx = _tracked_trxs.index().get(); auto itr = idx.find(trace->id); if( itr != idx.end() ) { - _tracked_trxs.modify( itr, [&trace, &control=_controller, &abi_max_time=_abi_serializer_max_time]( tracked_transaction& tt ) { + _tracked_trxs.modify( itr, [&trace, &control=_controller]( tracked_transaction& tt ) { tt.block_num = trace->block_num; - try { - // send_transaction trace output format. - // Convert to variant with abi here and now because abi could change in very next transaction. - // Alternatively, we could store off all the abis needed and do the conversion later, but as this is designed - // to run on an API node, probably the best trade off to perform the abi serialization during block processing. - auto resolver = get_serializers_cache(control, trace, abi_max_time); - tt.trx_trace_v.clear(); - abi_serializer::to_variant(*trace, tt.trx_trace_v, resolver, abi_max_time); - } catch( chain::abi_exception& ) { - tt.trx_trace_v = *trace; - } + // Capture the trace and the raw ABI bytes for accounts referenced by it. ABI parsing + JSON emission are + // deferred to response-emit time on the http thread pool (see sysio::chain_apis::streamed_processed_trace + // and `fc::to_json_stream(streamed_processed_trace, json_writer&)`). Capturing the raw bytes here + // preserves "ABIs frozen at apply time" semantics -- subsequent on-chain ABI updates won't affect the + // captured copy. + tt.trace_ptr = trace; + tt.raw_abis = sysio::capture_abis(control, trace); } ); } } @@ -190,7 +211,8 @@ struct trx_retry_db_impl { tt.last_try += fc::seconds( 10 ); } tt.block_num = 0; - tt.trx_trace_v.clear(); + tt.trace_ptr.reset(); + tt.raw_abis = {}; } ); } } @@ -228,9 +250,11 @@ struct trx_retry_db_impl { } // ack for( auto& i: to_process ) { - _tracked_trxs.modify( i, [&]( tracked_transaction& tt ) { - tt.next( std::make_unique( std::move( tt.trx_trace_v ) ) ); - tt.trx_trace_v.clear(); + _tracked_trxs.modify( i, [this]( tracked_transaction& tt ) { + tt.next( std::make_unique( + sysio::chain_apis::streamed_processed_trace{ std::move(tt.trace_ptr), + std::move(tt.raw_abis), + _abi_serializer_max_time } ) ); } ); _tracked_trxs.erase( i ); } @@ -246,9 +270,11 @@ struct trx_retry_db_impl { } // ack for( auto& i: to_process ) { - _tracked_trxs.modify( i, [&]( tracked_transaction& tt ) { - tt.next( std::make_unique( std::move( tt.trx_trace_v ) ) ); - tt.trx_trace_v.clear(); + _tracked_trxs.modify( i, [this]( tracked_transaction& tt ) { + tt.next( std::make_unique( + sysio::chain_apis::streamed_processed_trace{ std::move(tt.trace_ptr), + std::move(tt.raw_abis), + _abi_serializer_max_time } ) ); } ); _tracked_trxs.erase( i ); } @@ -290,7 +316,8 @@ trx_retry_db::trx_retry_db( const chain::controller& controller, size_t max_mem_ trx_retry_db::~trx_retry_db() = default; -void trx_retry_db::track_transaction( chain::packed_transaction_ptr ptrx, std::optional num_blocks, next_function> next ) { +void trx_retry_db::track_transaction( chain::packed_transaction_ptr ptrx, std::optional num_blocks, + next_function> next ) { _impl->track_transaction( std::move( ptrx ), num_blocks, next ); } diff --git a/plugins/chain_plugin/test/test_trx_retry_db.cpp b/plugins/chain_plugin/test/test_trx_retry_db.cpp index d7f95f3d70..1e8293c706 100644 --- a/plugins/chain_plugin/test/test_trx_retry_db.cpp +++ b/plugins/chain_plugin/test/test_trx_retry_db.cpp @@ -232,14 +232,14 @@ BOOST_AUTO_TEST_CASE(trx_retry_logic) { auto lib = std::optional{}; auto trx_1 = make_unique_trx(chain->get_chain_id(), fc::seconds(2), 1); bool trx_1_expired = false; - trx_retry.track_transaction( trx_1, lib, [&trx_1_expired](const next_function_variant>& result){ + trx_retry.track_transaction( trx_1, lib, [&trx_1_expired](next_function_variant>&& result){ BOOST_REQUIRE( std::holds_alternative(result) ); BOOST_CHECK_EQUAL( std::get(result)->code(), expired_tx_exception::code_value ); trx_1_expired = true; } ); auto trx_2 = make_unique_trx(chain->get_chain_id(), fc::seconds(4), 2); bool trx_2_expired = false; - trx_retry.track_transaction( trx_2, lib, [&trx_2_expired](const next_function_variant>& result){ + trx_retry.track_transaction( trx_2, lib, [&trx_2_expired](next_function_variant>&& result){ BOOST_REQUIRE( std::holds_alternative(result) ); BOOST_CHECK_EQUAL( std::get(result)->code(), expired_tx_exception::code_value ); trx_2_expired = true; @@ -278,7 +278,7 @@ BOOST_AUTO_TEST_CASE(trx_retry_logic) { // auto trx_3 = make_unique_trx(chain->get_chain_id(), fc::seconds(30), 3); bool trx_3_expired = false; - trx_retry.track_transaction( trx_3, lib, [&trx_3_expired](const next_function_variant>& result){ + trx_retry.track_transaction( trx_3, lib, [&trx_3_expired](next_function_variant>&& result){ BOOST_REQUIRE( std::holds_alternative(result) ); BOOST_CHECK_EQUAL( std::get(result)->code(), expired_tx_exception::code_value ); trx_3_expired = true; @@ -288,7 +288,7 @@ BOOST_AUTO_TEST_CASE(trx_retry_logic) { fc::mock_time_traits::set_now(pnow); auto trx_4 = make_unique_trx(chain->get_chain_id(), fc::seconds(30), 4); bool trx_4_expired = false; - trx_retry.track_transaction( trx_4, lib, [&trx_4_expired](const next_function_variant>& result){ + trx_retry.track_transaction( trx_4, lib, [&trx_4_expired](next_function_variant>&& result){ BOOST_REQUIRE( std::holds_alternative(result) ); BOOST_CHECK_EQUAL( std::get(result)->code(), expired_tx_exception::code_value ); trx_4_expired = true; @@ -331,9 +331,9 @@ BOOST_AUTO_TEST_CASE(trx_retry_logic) { // auto trx_5 = make_unique_trx(chain->get_chain_id(), fc::seconds(30), 5); bool trx_5_variant = false; - trx_retry.track_transaction( trx_5, lib, [&trx_5_variant](const next_function_variant>& result){ - BOOST_REQUIRE( std::holds_alternative>(result) ); - BOOST_CHECK( !!std::get>(result) ); + trx_retry.track_transaction( trx_5, lib, [&trx_5_variant](next_function_variant>&& result){ + BOOST_REQUIRE( std::holds_alternative>(result) ); + BOOST_CHECK( !!std::get>(result) ); trx_5_variant = true; } ); // increase time by 1 seconds, so trx_6 retry interval diff than 5 @@ -341,9 +341,9 @@ BOOST_AUTO_TEST_CASE(trx_retry_logic) { fc::mock_time_traits::set_now(pnow); auto trx_6 = make_unique_trx(chain->get_chain_id(), fc::seconds(30), 6); bool trx_6_variant = false; - trx_retry.track_transaction( trx_6, std::optional(2), [&trx_6_variant](const next_function_variant>& result){ - BOOST_REQUIRE( std::holds_alternative>(result) ); - BOOST_CHECK( !!std::get>(result) ); + trx_retry.track_transaction( trx_6, std::optional(2), [&trx_6_variant](next_function_variant>&& result){ + BOOST_REQUIRE( std::holds_alternative>(result) ); + BOOST_CHECK( !!std::get>(result) ); trx_6_variant = true; } ); // not in block 7, so not returned to user @@ -400,9 +400,9 @@ BOOST_AUTO_TEST_CASE(trx_retry_logic) { // auto trx_7 = make_unique_trx(chain->get_chain_id(), fc::seconds(30), 7); bool trx_7_variant = false; - trx_retry.track_transaction( trx_7, lib, [&trx_7_variant](const next_function_variant>& result){ - BOOST_REQUIRE( std::holds_alternative>(result) ); - BOOST_CHECK( !!std::get>(result) ); + trx_retry.track_transaction( trx_7, lib, [&trx_7_variant](next_function_variant>&& result){ + BOOST_REQUIRE( std::holds_alternative>(result) ); + BOOST_CHECK( !!std::get>(result) ); trx_7_variant = true; } ); // increase time by 1 seconds, so trx_8 retry interval diff than 7 @@ -410,15 +410,15 @@ BOOST_AUTO_TEST_CASE(trx_retry_logic) { fc::mock_time_traits::set_now(pnow); auto trx_8 = make_unique_trx(chain->get_chain_id(), fc::seconds(30), 8); bool trx_8_variant = false; - trx_retry.track_transaction( trx_8, std::optional(3), [&trx_8_variant](const next_function_variant>& result){ - BOOST_REQUIRE( std::holds_alternative>(result) ); - BOOST_CHECK( !!std::get>(result) ); + trx_retry.track_transaction( trx_8, std::optional(3), [&trx_8_variant](next_function_variant>&& result){ + BOOST_REQUIRE( std::holds_alternative>(result) ); + BOOST_CHECK( !!std::get>(result) ); trx_8_variant = true; } ); // one to expire, will be forked out never to return auto trx_9 = make_unique_trx(chain->get_chain_id(), fc::seconds(30), 9); bool trx_9_expired = false; - trx_retry.track_transaction( trx_9, lib, [&trx_9_expired](const next_function_variant>& result){ + trx_retry.track_transaction( trx_9, lib, [&trx_9_expired](next_function_variant>&& result){ BOOST_REQUIRE( std::holds_alternative(result) ); BOOST_CHECK_EQUAL( std::get(result)->code(), expired_tx_exception::code_value ); trx_9_expired = true; @@ -559,16 +559,16 @@ BOOST_AUTO_TEST_CASE(trx_retry_logic) { // auto trx_10 = make_unique_trx(chain->get_chain_id(), fc::seconds(30), 10); bool trx_10_variant = false; - trx_retry.track_transaction( trx_10, std::optional(0), [&trx_10_variant](const next_function_variant>& result){ - BOOST_REQUIRE( std::holds_alternative>(result) ); - BOOST_CHECK( !!std::get>(result) ); + trx_retry.track_transaction( trx_10, std::optional(0), [&trx_10_variant](next_function_variant>&& result){ + BOOST_REQUIRE( std::holds_alternative>(result) ); + BOOST_CHECK( !!std::get>(result) ); trx_10_variant = true; } ); auto trx_11 = make_unique_trx(chain->get_chain_id(), fc::seconds(30), 11); bool trx_11_variant = false; - trx_retry.track_transaction( trx_11, std::optional(1), [&trx_11_variant](const next_function_variant>& result){ - BOOST_REQUIRE( std::holds_alternative>(result) ); - BOOST_CHECK( !!std::get>(result) ); + trx_retry.track_transaction( trx_11, std::optional(1), [&trx_11_variant](next_function_variant>&& result){ + BOOST_REQUIRE( std::holds_alternative>(result) ); + BOOST_CHECK( !!std::get>(result) ); trx_11_variant = true; } ); // seen in block immediately diff --git a/plugins/db_size_api_plugin/src/db_size_api_plugin.cpp b/plugins/db_size_api_plugin/src/db_size_api_plugin.cpp index 65e24ab1ec..df886010b8 100644 --- a/plugins/db_size_api_plugin/src/db_size_api_plugin.cpp +++ b/plugins/db_size_api_plugin/src/db_size_api_plugin.cpp @@ -1,32 +1,18 @@ #include #include #include -#include +#include namespace sysio { using namespace sysio; -#define CALL_WITH_400(api_name, api_handle, call_name, INVOKE, http_response_code) \ -{std::string("/v1/" #api_name "/" #call_name), \ - api_category::db_size, \ - [api_handle](string&&, string&& body, url_response_callback&& cb) mutable { \ - try { \ - body = parse_params(body); \ - INVOKE \ - cb(http_response_code, fc::variant(result)); \ - } catch (...) { \ - http_plugin::handle_exception(#api_name, #call_name, body, cb); \ - } \ - }} - -#define INVOKE_R_V(api_handle, call_name) \ - auto result = api_handle->call_name(); - - void db_size_api_plugin::plugin_startup() { - app().get_plugin().add_api({ - CALL_WITH_400(db_size, this, get, INVOKE_R_V(this, get), 200), + auto& _http_plugin = app().get_plugin(); + _http_plugin.add_api_stream({ + bind_stream<&db_size_api_plugin::get, dispatch::sync>( + _http_plugin, this, "/v1/db_size/get", + api_category::db_size, http_params_types::no_params, 200), }, appbase::exec_queue::read_only); } @@ -46,7 +32,4 @@ db_size_stats db_size_api_plugin::get() { return ret; } -#undef INVOKE_R_V -#undef CALL - } diff --git a/plugins/http_plugin/include/sysio/http_plugin/beast_http_session.hpp b/plugins/http_plugin/include/sysio/http_plugin/beast_http_session.hpp index 1d3bb72808..1d3ddbf83a 100644 --- a/plugins/http_plugin/include/sysio/http_plugin/beast_http_session.hpp +++ b/plugins/http_plugin/include/sysio/http_plugin/beast_http_session.hpp @@ -105,6 +105,7 @@ class beast_http_session : public detail::abstract_conn, break; case http_content_type::json: + case http_content_type::json_raw: default: res_->set(http::field::content_type, "application/json"); } @@ -164,9 +165,26 @@ class beast_http_session : public detail::abstract_conn, remote_endpoint_, to_log_string(req) ); std::string resource = std::string(req.target()); - // look for the URL handler to handle this resource + // look for the URL handler to handle this resource - check the streaming map + // first since it's the smaller of the two during migration; both maps share + // the URL keyspace and registration asserts disjointness. + auto stream_itr = plugin_state_->url_handlers_stream.find(resource); auto handler_itr = plugin_state_->url_handlers.find(resource); - if(handler_itr != plugin_state_->url_handlers.end() && categories_.contains(handler_itr->second.category)) { + if(stream_itr != plugin_state_->url_handlers_stream.end() && categories_.contains(stream_itr->second.category)) { + fc_tlog(plugin_state_->get_logger(), "resource (stream): {}", resource); + std::string body = req.body(); + // Streaming responses are always application/json - the json_writer output + // is the response body verbatim, no plaintext / json_raw distinction. + set_content_type_header(http_content_type::json); + + if (plugin_state_->update_metrics) + plugin_state_->update_metrics({resource}); + + stream_itr->second.fn(this->shared_from_this(), + std::move(resource), + std::move(body), + make_http_stream_response_handler(*plugin_state_, this->shared_from_this())); + } else if(handler_itr != plugin_state_->url_handlers.end() && categories_.contains(handler_itr->second.category)) { fc_tlog(plugin_state_->get_logger(), "resource: {}", resource); std::string body = req.body(); auto content_type = handler_itr->second.content_type; @@ -185,6 +203,10 @@ class beast_http_session : public detail::abstract_conn, if (categories_.contains(handler.second.category)) result.apis.push_back(handler.first); } + for (const auto& handler : plugin_state_->url_handlers_stream) { + if (categories_.contains(handler.second.category)) + result.apis.push_back(handler.first); + } send_response(fc::json::to_string(fc::variant(result), fc::time_point::maximum()), 200); } else { fc_dlog( plugin_state_->get_logger(), "404 - not found: {}", resource ); diff --git a/plugins/http_plugin/include/sysio/http_plugin/bind_stream.hpp b/plugins/http_plugin/include/sysio/http_plugin/bind_stream.hpp new file mode 100644 index 0000000000..2871be68a8 --- /dev/null +++ b/plugins/http_plugin/include/sysio/http_plugin/bind_stream.hpp @@ -0,0 +1,491 @@ +#pragma once + +#include +#include +// controller.hpp must precede plugin_interface.hpp: the latter uses block_signal_params +// (declared in controller.hpp) without including it, so this header includes it to stay +// self-contained regardless of the caller's include order. +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace sysio { + +/** + * @brief Compile-time tag selecting the response control flow used by `bind_stream`. + * + * - `sync` : `R api.method(args...)` -- result is captured directly into the + * json_writer-emitting closure handed to the streaming cb. + * - `sync_void` : `void api.method(args...)` -- emits the convention `{"result":"ok"}` + * response, matching the producer_api / net_api void-method pattern. + * - `post` : `function()> api.method(args...)` -- Phase 1 + * runs on the calling queue and returns a closure (`http_fwd`) that + * will produce the typed result; Phase 2 runs `http_fwd` on the http + * thread pool and emits via `to_json_stream(result)`. + * - `post_direct` : `function()> api.method(args...)` -- like + * `post`, but Phase 2 hands the emit closure straight to the cb. + * Used when the api builds and emits its JSON without an + * intermediate reflected struct (eg per-row binary decode). + * - `async` : `void api.method(args..., next_function)` -- the api invokes + * the callback later with either a typed result, an exception, or a + * Phase-2 `http_fwd` closure (mirroring the variant-cb semantics). + */ +enum class dispatch { sync, sync_void, post, post_direct, async }; + +namespace http_detail { + + /// Standard `{"result":"ok"}` reply emitted by `dispatch::sync_void`. Reflected + /// here so each api plugin doesn't have to define its own ack struct. + struct ok_response { + std::string result = "ok"; + }; + + using emit_fn_t = std::function; + + /// @brief Pulls apart a member-function pointer into its pieces. + /// + /// Specialized for both non-const and const-qualified member functions so + /// `member_fn` works regardless of cv-qualification. + template struct member_fn; + + template + struct member_fn { + using class_t = Cls; + using ret_t = R; + template using arg_t = std::tuple_element_t>; + static constexpr size_t arity = sizeof...(A); + }; + + template + struct member_fn { + using class_t = Cls; + using ret_t = R; + template using arg_t = std::tuple_element_t>; + static constexpr size_t arity = sizeof...(A); + }; + + /// @brief Detects `function()>` (the Phase-2 closure shape). + template struct is_typed_fwd : std::false_type {}; + template + struct is_typed_fwd()>> : std::true_type { + using payload = T; + }; + + /// @brief Detects `chain::plugin_interface::next_function` (the async cb shape). + template struct is_next_fn : std::false_type {}; + template + struct is_next_fn> : std::true_type { + using payload = T; + }; + + /// @brief True iff T is `fc::time_point` after stripping cv/ref. + template + inline constexpr bool is_deadline_v = + std::is_same_v, fc::time_point>; + + /// Lazily evaluates the method's first argument bare-type when `HasArgs` is true. + /// Required because `std::conditional_t` eagerly instantiates both branches, so a + /// direct `mf::arg_t<0>` reference fails to compile for arity-0 methods. + template + struct first_arg_or_void { using type = std::monostate; }; + template + struct first_arg_or_void { + using type = std::remove_cvref_t>; + }; + + /// Compute the per-request deadline. Chain api handles expose `start()` which both + /// returns the deadline and registers the in-flight call; other handles fall back to + /// `http_plugin::get_max_response_time()`. Constraint requires `start()` return to + /// be convertible to fc::time_point so an unrelated `start()` does not match. + template + inline fc::time_point compute_deadline(Handle& handle, http_plugin& http) { + if constexpr (requires { { handle.start() } -> std::convertible_to; }) { + return handle.start(); + } else if constexpr (requires { { handle->start() } -> std::convertible_to; }) { + return handle->start(); + } else { + const fc::microseconds m = http.get_max_response_time(); + return m == fc::microseconds::maximum() ? fc::time_point::maximum() + : fc::time_point::now() + m; + } + } + + /// Runtime dispatch wrapper around the compile-time `parse_params` template. + /// Three instantiations per T -- compilers inline through the switch. + template + inline T parse_params_rt(const std::string& body, http_params_types pt) { + switch (pt) { + case http_params_types::no_params: + return parse_params(body); + case http_params_types::params_required: + return parse_params(body); + case http_params_types::possible_no_params: + return parse_params(body); + } + std::unreachable(); + } + + /// Parse a registration path of the form "/v1//" into the + /// (api_name, call_name) labels used by `handle_exception_stream`. Returns + /// owning std::string because the caller captures the labels into a long-lived + /// registration lambda. + inline std::pair split_path(std::string_view path) { + auto last_slash = path.find_last_of('/'); + assert(last_slash != std::string_view::npos && last_slash > 0); // expects "/api/call" shape + auto prev_slash = path.find_last_of('/', last_slash - 1); + assert(prev_slash != std::string_view::npos); + return { std::string(path.substr(prev_slash + 1, last_slash - prev_slash - 1)), + std::string(path.substr(last_slash + 1)) }; + } + + /// Invoke `MethodPtr` against `handle` with the right argument shape: + /// arity 0 -> () + /// arity 1 -> (params) + /// arity 2 + deadline-> (params, deadline) + /// arity 1 async -> (next_function) + /// arity 2 async -> (params, next_function) + /// Deadline-vs-not is selected by inspecting the last param type at compile time. + template + inline auto invoke_sync(Handle& handle, Params&& params, fc::time_point deadline) { + using mf = member_fn; + if constexpr (mf::arity == 0) { + return std::invoke(MethodPtr, handle); + } else if constexpr (mf::arity == 1) { + return std::invoke(MethodPtr, handle, std::forward(params)); + } else { + // arity == 2: (params, deadline). + static_assert(is_deadline_v>, + "second argument must be fc::time_point (deadline)"); + return std::invoke(MethodPtr, handle, std::forward(params), deadline); + } + } + + /// Async variant: last argument is `next_function`, called by api method later. + template + inline void invoke_async(Handle& handle, Params&& params, Next&& next) { + using mf = member_fn; + if constexpr (mf::arity == 1) { + // (next_function) -- no params arg, body must be empty + std::invoke(MethodPtr, handle, std::forward(next)); + } else { + // (params, next_function) + std::invoke(MethodPtr, handle, std::forward(params), std::forward(next)); + } + } + + /// Packed-size estimate of a typed response payload, used to reserve bytes-in-flight + /// (http_plugin::reserve_bytes_in_flight) for as long as the payload sits captured in a + /// stream_emitter on the http thread pool queue. fc::raw::pack_size is an + /// allocation-free byte count over the same data the variant path estimates via + /// detail::in_flight_sizeof (pack_size of the response variant tree). + /// + /// Payload types the generic reflected pack cannot traverse (eg results embedding + /// streamed_processed_trace, which is emitted but never wire-packed) provide an + /// ADL-visible `queued_payload_size(const T&)` overload instead -- checked first. + /// Anything neither customized, reflected, nor a variant (emit-closure payloads) + /// returns 0: those are produced on the http pool and handed to the response callback + /// inline, so no queue window exists to charge, and a std::function cannot be packed. + template + inline size_t source_size_estimate(const T& r) { + if constexpr (requires { { queued_payload_size(r) } -> std::convertible_to; }) { + return queued_payload_size(r); + } else if constexpr (fc::reflector::is_defined::value || std::is_same_v) { + try { + return fc::raw::pack_size(r); + } catch (...) {} + } + return 0; + } + + /// Batch results (eg push_transactions' vector of per-transaction results) size as the + /// sum of their elements' estimates. + template + inline size_t source_size_estimate(const std::vector& v) { + size_t sz = 0; + for (const auto& e : v) + sz += source_size_estimate(e); + return sz; + } + +} // namespace http_detail + + +/** + * @brief Build an `api_entry_stream` registration for an api method, dispatching the + * response per `Kind`. + * + * @tparam MethodPtr pointer-to-member of the api class implementing the endpoint + * @tparam Kind which response shape (`dispatch::sync` etc) the method uses; + * compile-time `static_assert`s pin Kind to the method signature + * so a refactor that changes the signature produces a build error, + * not a silent behavior change. + * @tparam Handle the api handle type (read_only, read_write, producer_plugin&, + * plugin*, ...). Captured by-value into the lambda; for handles + * that should be referenced across requests pass an explicit + * reference type or pointer. + * + * @param http the http_plugin instance, used for the http thread pool, max + * response time, and exception logging. + * @param handle the api handle, captured into the request lambda. + * @param path e.g. "/v1/chain/get_info"; used for url registration AND for + * deriving the (api_name, call_name) labels passed to + * `handle_exception_stream` on errors. + * @param cat api category for the registration. + * @param pt parameter parsing mode for the request body. + * @param resp_code HTTP status code on success. + * + * @return an `api_entry_stream` ready to be batched into + * `http_plugin::add_api_stream` / `add_async_api_stream`. + */ +template +api_entry_stream bind_stream(http_plugin& http, Handle handle, + std::string path, api_category cat, + http_params_types pt, uint16_t resp_code) { + using namespace http_detail; + using mf = member_fn; + using ret_t = typename mf::ret_t; + + // ----- Compile-time signature validation against `Kind` ------------------ + if constexpr (Kind == dispatch::sync) { + static_assert(!std::is_void_v, + "dispatch::sync expects T return; method returns void (use dispatch::sync_void)."); + static_assert(!is_typed_fwd::value, + "dispatch::sync method returns function()>; use dispatch::post or dispatch::post_direct."); + } else if constexpr (Kind == dispatch::sync_void) { + static_assert(std::is_void_v, + "dispatch::sync_void expects void return; method returns a value."); + } else if constexpr (Kind == dispatch::post) { + static_assert(is_typed_fwd::value, + "dispatch::post expects function()> return."); + static_assert(!std::is_same_v::payload, emit_fn_t>, + "dispatch::post payload is emit_fn_t; use dispatch::post_direct instead."); + } else if constexpr (Kind == dispatch::post_direct) { + static_assert(std::is_same_v()>>, + "dispatch::post_direct expects function()> return."); + } else { // dispatch::async + static_assert(std::is_void_v, + "dispatch::async expects void return."); + static_assert(mf::arity >= 1, "dispatch::async requires at least one arg (next_function)."); + static_assert( + is_next_fn>>::value, + "dispatch::async requires the last argument to be next_function."); + } + + // ----- Parameter type deduction ------------------------------------------ + // `params_t` is the *first* method arg's bare type (when present). + // For arity-0 sync methods we don't parse a body. + constexpr bool has_params = []() { + if constexpr (Kind == dispatch::async) { + // async signature: (next_function) or (params, next_function) + return mf::arity >= 2; + } else { + return mf::arity >= 1; + } + }(); + + using params_t = typename http_detail::first_arg_or_void::type; + + auto labels = split_path(path); + std::string api_name = std::move(labels.first); + std::string call_name = std::move(labels.second); + + return api_entry_stream { + std::move(path), cat, + [handle = std::move(handle), &http, pt, resp_code, + api_name = std::move(api_name), call_name = std::move(call_name)] + (std::string&&, std::string&& body, url_response_stream_callback&& cb) mutable { + // `http` is referenced by some Kinds (post/post_direct/async/sync via start()); + // taking its address unconditionally keeps the capture flagged as used so the + // compiler does not emit -Wunused-lambda-capture for sync_void instantiations. + (void)&http; + + // For sync paths that need a deadline, compute it before the try{} so that + // chain_api's start() side effect runs even if param parsing throws. + fc::time_point deadline; + if constexpr (Kind != dispatch::sync_void) { + deadline = compute_deadline(handle, http); + } + + try { + // ---- Parse params (if the method takes any) ---- + // Immediately-invoked initializer so `params` is constructed directly from the + // parse result (no default-construct-then-assign, and param structs need not be + // default-constructible). The no-params arm still validates the body per `pt` + // and yields the std::monostate placeholder. + [[maybe_unused]] params_t params = [&]() -> params_t { + if constexpr (has_params) { + return parse_params_rt(body, pt); + } else { + (void)parse_params_rt(body, pt); + return {}; + } + }(); + + // ---- Dispatch on Kind -------------------------------------- + if constexpr (Kind == dispatch::sync) { + auto result = invoke_sync(handle, std::move(params), deadline); + // Reserve the typed result's packed size for as long as it rides the emitter + // closure across the queue to the http pool -- the streaming analogue of the + // variant path's pre-dispatch in_flight_sizeof charge. Released when the + // emitter (and with it the captured result) is destroyed after emission. + auto reservation = http.reserve_bytes_in_flight(source_size_estimate(result)); + cb(resp_code, + [r = std::move(result), reservation = std::move(reservation)](fc::json_writer& w) mutable { + fc::to_json_stream(r, w); + }); + + } else if constexpr (Kind == dispatch::sync_void) { + if constexpr (mf::arity == 0) { + std::invoke(MethodPtr, handle); + } else { + std::invoke(MethodPtr, handle, std::move(params)); + } + cb(resp_code, [](fc::json_writer& w) { + ok_response r; + fc::to_json_stream(r, w); + }); + + } else if constexpr (Kind == dispatch::post) { + using payload_t = typename is_typed_fwd::payload; + using http_fwd_t = std::function()>; + http_fwd_t http_fwd = invoke_sync(handle, std::move(params), deadline); + http.post_http_thread_pool( + [resp_code, cb = std::move(cb), body = std::move(body), + api_name, call_name, + http_fwd = std::move(http_fwd)]() mutable { + try { + chain::t_or_exception result = http_fwd(); + if (std::holds_alternative(result)) { + // rethrow() (virtual) preserves the stored exception's dynamic type so + // classify_current_exception's specific-type catches map it to the right + // status code (tx_duplicate -> 409, unknown_block -> 400, ...). A plain + // `throw *ptr` would slice to fc::exception and turn them all into 500s. + try { std::get(result)->rethrow(); } + catch (...) { + http_plugin::handle_exception_stream( + api_name.c_str(), call_name.c_str(), body, cb); + } + } else { + cb(resp_code, + [r = std::get(std::move(result))] + (fc::json_writer& w) mutable { fc::to_json_stream(r, w); }); + } + } catch (...) { + http_plugin::handle_exception_stream( + api_name.c_str(), call_name.c_str(), body, cb); + } + }); + + } else if constexpr (Kind == dispatch::post_direct) { + using http_fwd_t = std::function()>; + http_fwd_t http_fwd = invoke_sync(handle, std::move(params), deadline); + http.post_http_thread_pool( + [resp_code, cb = std::move(cb), body = std::move(body), + api_name, call_name, + http_fwd = std::move(http_fwd)]() mutable { + try { + chain::t_or_exception result = http_fwd(); + if (std::holds_alternative(result)) { + try { std::get(result)->rethrow(); } + catch (...) { + http_plugin::handle_exception_stream( + api_name.c_str(), call_name.c_str(), body, cb); + } + } else { + cb(resp_code, std::get(std::move(result))); + } + } catch (...) { + http_plugin::handle_exception_stream( + api_name.c_str(), call_name.c_str(), body, cb); + } + }); + + } else { // dispatch::async + using payload_t = typename is_next_fn< + std::remove_cvref_t>>::payload; + using http_fwd_t = std::function()>; + // The next_function operator() is rvalue-only; move out of the variant into + // the response closure so the heavy payload (raw ABIs, traces) is moved, not + // copied, before the closure runs on the http pool. + static_assert(std::is_move_constructible_v, + "bind_stream::async requires the next_function payload to be move-constructible."); + + auto next = [&http, cb = std::move(cb), body = std::move(body), + api_name, call_name, resp_code] + (chain::plugin_interface::next_function_variant&& result) mutable { + if (std::holds_alternative(result)) { + try { std::get(result)->rethrow(); } + catch (...) { + http_plugin::handle_exception_stream( + api_name.c_str(), call_name.c_str(), body, cb); + } + } else if (std::holds_alternative(result)) { + // Same queued-payload reservation as dispatch::sync: the typed result + // is produced on whichever queue invoked the next_function and rides + // the emitter closure across to the http pool. + auto reservation = + http.reserve_bytes_in_flight(source_size_estimate(std::get(result))); + cb(resp_code, + [r = std::get(std::move(result)), reservation = std::move(reservation)] + (fc::json_writer& w) mutable { fc::to_json_stream(r, w); }); + } else { + http.post_http_thread_pool( + [resp_code, cb = std::move(cb), body = std::move(body), + api_name, call_name, + http_fwd = std::get(std::move(result))]() mutable { + try { + chain::t_or_exception r = http_fwd(); + if (std::holds_alternative(r)) { + try { std::get(r)->rethrow(); } + catch (...) { + http_plugin::handle_exception_stream( + api_name.c_str(), call_name.c_str(), body, cb); + } + } else { + cb(resp_code, + [v = std::get(std::move(r))] + (fc::json_writer& w) mutable { + fc::to_json_stream(v, w); + }); + } + } catch (...) { + http_plugin::handle_exception_stream( + api_name.c_str(), call_name.c_str(), body, cb); + } + }); + } + }; + + // Empty body -> default-constructed params for async (matches + // CALL_ASYNC_STREAM "if (body.empty()) body = "{}";" preflight). + if constexpr (has_params) { + invoke_async(handle, std::move(params), std::move(next)); + } else { + // Arity-1 async: no params slot, invoke the method directly. + std::invoke(MethodPtr, handle, std::move(next)); + } + } + } catch (...) { + http_plugin::handle_exception_stream( + api_name.c_str(), call_name.c_str(), body, cb); + } + } + }; +} + +} // namespace sysio + +FC_REFLECT(sysio::http_detail::ok_response, (result)); diff --git a/plugins/http_plugin/include/sysio/http_plugin/common.hpp b/plugins/http_plugin/include/sysio/http_plugin/common.hpp index f0c11a4c05..ace206208e 100644 --- a/plugins/http_plugin/include/sysio/http_plugin/common.hpp +++ b/plugins/http_plugin/include/sysio/http_plugin/common.hpp @@ -3,6 +3,7 @@ #include // for thread pool #include +#include #include #include #include @@ -94,6 +95,19 @@ struct internal_url_handler { api_category category; http_content_type content_type = http_content_type::json; }; + +/** +* Streaming counterpart of internal_url_handler. Endpoints registered via +* http_plugin::add_handler_stream / add_async_handler_stream live in the parallel +* url_handlers_stream map; the beast dispatch checks both maps and routes to whichever +* one owns the URL. No content_type slot - the streaming cb always emits +* application/json directly into the response buffer. +*/ +using internal_url_handler_stream_fn = std::function; +struct internal_url_handler_stream { + internal_url_handler_stream_fn fn; + api_category category; +}; /** * Helper method to calculate the "in flight" size of a fc::variant * This is an estimate based on fc::raw::pack if that process can be successfully executed @@ -123,11 +137,30 @@ static size_t in_flight_sizeof(const std::optional& o) { return 0; } +/** +* Thrown by the streaming-response growth guard when the incrementally-charged response +* buffer pushes bytes_in_flight past the configured budget (--http-max-bytes-in-flight-mb). +* +* Deliberately NOT derived from fc::exception or std::exception: mid-emission rollback +* handlers (eg abi_serializer's catch-rewind-hex-fallback path) catch and absorb serializer +* exceptions to keep emitting, and absorbing a budget abort would defeat the cap. A foreign +* type dodges their typed catches, and json_writer re-fires the guard on the next token +* after a throw so even a catch(...) absorber re-raises this until it unwinds out of the +* emitter (see json_writer's guarded-constructor contract). +*/ +struct stream_response_budget_exceeded { + std::string error; ///< busy-response text produced by verify_max_bytes_in_flight +}; + }// namespace detail // key -> priority, url_handler typedef map url_handlers_type; +// Streaming-cb counterpart of url_handlers_type. Disjoint URL-set from the variant +// path; an endpoint registers either form, never both. +typedef map url_handlers_stream_type; + struct http_plugin_state { string access_control_allow_origin; string access_control_allow_headers; @@ -147,6 +180,7 @@ struct http_plugin_state { string server_header; url_handlers_type url_handlers; + url_handlers_stream_type url_handlers_stream; bool keep_alive = false; uint16_t thread_pool_size = 2; @@ -189,7 +223,22 @@ inline auto make_http_response_handler(http_plugin_state& plugin_state, detail:: try { if (response.has_value()) { - std::string json = (content_type == http_content_type::plaintext) ? response->as_string() : fc::json::to_string(*response, fc::time_point::maximum()); + // plaintext path: the variant's string payload is the literal + // response body. + // json_raw path: the variant's string payload is pre-serialized + // JSON (built via fc::json_writer) and passed through as-is. + // json_raw still tolerates non-string variants (eg error + // responses produced by http_plugin::handle_exception) by + // falling through to fc::json::to_string; the content-type + // header stays "application/json" either way. + std::string json; + if (content_type == http_content_type::plaintext) { + json = response->as_string(); + } else if (content_type == http_content_type::json_raw && response->is_string()) { + json = response->as_string(); + } else { + json = fc::json::to_string(*response, fc::time_point::maximum()); + } if (auto error_str = session_ptr->verify_max_bytes_in_flight(json.size()); error_str.empty()) session_ptr->send_response(std::move(json), code); else @@ -205,6 +254,98 @@ inline auto make_http_response_handler(http_plugin_state& plugin_state, detail:: } +/** +* Construct a url_response_stream_callback that runs the emitter on the http_plugin +* thread pool and sends the produced JSON as the response body. +* +* The api lambda is responsible for capturing whatever it wants to serialize into +* the emitter closure (typically the typed result struct, by-move). All JSON +* serialization happens inside the dispatched lambda on the http thread pool, so +* the api thread (read_only / read_write queue) never pays the per-field +* allocation cost the variant tree built. +* +* Backpressure note: the response buffer is charged against bytes_in_flight +* incrementally WHILE the emitter runs -- json_writer's growth guard settles the +* buffer's size into the budget every json_writer_default_guard_stride bytes and +* throws stream_response_budget_exceeded (mapped to the busy response) once +* verify_max_bytes_in_flight rejects. Concurrent large streaming responses are +* therefore visible to --http-max-bytes-in-flight-mb during serialization, and +* over-budget emissions abort early instead of materializing in full; the variant +* path achieves the analogous mid-flight visibility by pre-charging its +* in_flight_sizeof estimate of the response variant tree. +* +* The captured source payload is accounted for separately: bind_stream reserves its +* packed-size estimate (http_plugin::reserve_bytes_in_flight) when it captures a typed +* result into the emitter, and that reservation rides the closure across this queue -- +* so results waiting here for a pool thread are already visible to the budget, exactly +* like the variant path's queued response variants. The dispatched lambda below +* re-checks the budget before emitting and drains over-budget work with the busy +* response without running the emitter at all. +*/ +inline auto make_http_stream_response_handler(http_plugin_state& plugin_state, detail::abstract_conn_ptr session_ptr) { + return url_response_stream_callback{ + [&plugin_state, session_ptr{std::move(session_ptr)}] + (int code, stream_emitter emitter) mutable { + + // Note on threading: boost::asio::dispatch on a thread_pool executor runs + // the function inline when called from a thread already inside the pool + // (impl/thread_pool.hpp `do_execute`: "Invoke immediately if the + // blocking.possibly property is enabled and we are already inside the + // thread pool"). So when this cb is fired from a `post_http_thread_pool` + // task -- e.g. the Phase 2 closure of a `dispatch::post_direct` registration + // -- the dispatch below is a function-call short-circuit, not a real post. + // For sync / sync_void / async-immediate paths (cb invoked on read-only + // or read-write queue), it correctly bounces JSON serialization + socket + // I/O onto the http thread pool. + boost::asio::dispatch(plugin_state.thread_pool.get_executor(), + [&plugin_state, session_ptr{std::move(session_ptr)}, code, emitter{std::move(emitter)}]() mutable { + // `charged` mirrors what this response has contributed to the shared + // bytes_in_flight budget; settle_to trues the contribution up against the + // buffer's actual size at guard checkpoints, after emission completes, and + // (via the scoped_exit) on every exit path -- including rewinds that SHRANK + // the buffer below an earlier checkpoint, hence the signed reconciliation. + size_t charged = 0; + auto settle_to = [&plugin_state, &charged](size_t actual) { + if (actual >= charged) + plugin_state.bytes_in_flight += actual - charged; + else + plugin_state.bytes_in_flight -= charged - actual; + charged = actual; + }; + auto on_exit = fc::make_scoped_exit([&settle_to]() { settle_to(0); }); + try { + // Queued-payload reservations (bind_stream's source_size_estimate, still + // held by the emitter) and every other request's charges are visible here: + // reject over-budget work before emitting anything, mirroring the variant + // path's pre-serialization verify. + if (auto error_str = session_ptr->verify_max_bytes_in_flight(0); !error_str.empty()) { + session_ptr->send_busy_response(std::move(error_str)); + return; + } + std::string body; + { + fc::json_writer w(body, [&settle_to, &session_ptr](size_t buffer_size) { + settle_to(buffer_size); + if (auto error_str = session_ptr->verify_max_bytes_in_flight(0); !error_str.empty()) + throw detail::stream_response_budget_exceeded{std::move(error_str)}; + }); + emitter(w); + } + settle_to(body.size()); + if (auto error_str = session_ptr->verify_max_bytes_in_flight(0); !error_str.empty()) + session_ptr->send_busy_response(std::move(error_str)); + else + session_ptr->send_response(std::move(body), code); + } catch (detail::stream_response_budget_exceeded& e) { + session_ptr->send_busy_response(std::move(e.error)); + } catch (...) { + session_ptr->handle_exception(); + } + }); + } + }; +} + inline bool host_is_valid(const http_plugin_state& plugin_state, const std::string& header_host_port, const asio::ip::address& addr) { diff --git a/plugins/http_plugin/include/sysio/http_plugin/http_plugin.hpp b/plugins/http_plugin/include/sysio/http_plugin/http_plugin.hpp index 347fc37304..334c49f3c4 100644 --- a/plugins/http_plugin/include/sysio/http_plugin/http_plugin.hpp +++ b/plugins/http_plugin/include/sysio/http_plugin/http_plugin.hpp @@ -5,8 +5,13 @@ #include #include #include +#include #include #include +#include + +#include + namespace sysio { using namespace appbase; @@ -33,6 +38,29 @@ namespace sysio { **/ using url_handler = std::function; + /** + * @brief Streaming response callback path. + * + * Parallel to url_response_callback / url_handler. Endpoints that opt into the + * streaming path build no fc::variant on the response - instead they hand cb() + * a closure that emits the response body directly into a fc::json_writer. The + * closure runs on the http_plugin thread pool, deferring the JSON emission off + * whatever thread the api lambda runs on (api thread, read-only pool, ...). + * + * Migration shape: + * variant cb: cb(code, fc::variant(result)); + * stream cb: cb(code, [r = std::move(result)](fc::json_writer& w) mutable { + * fc::to_json_stream(r, w); + * }); + * + * stream_emitter is move_only_function so the captured result struct does not + * have to satisfy CopyConstructible; pass the typed struct directly without + * round-tripping through fc::variant. + */ + using stream_emitter = fc::move_only_function; + using url_response_stream_callback = fc::move_only_function; + using url_handler_stream = std::function; + /** * @brief An API, containing URLs and handlers * @@ -48,9 +76,30 @@ namespace sysio { using api_description = std::vector; + /** + * api_entry_stream is the streaming-cb counterpart of api_entry. Same + * registration shape but the handler invokes a url_response_stream_callback + * rather than url_response_callback. Endpoints register either form via + * http_plugin::add_handler / add_handler_stream (or the _api batch + * registration helpers). + */ + struct api_entry_stream { + string path; + api_category category; + url_handler_stream handler; + }; + + using api_description_stream = std::vector; + enum class http_content_type { json = 1, - plaintext = 2 + plaintext = 2, + // Handler returns a fc::variant whose string payload is already a JSON document. + // Skips fc::json::to_string on the response path and sends the raw bytes with + // Content-Type: application/json. Used by endpoints that build their response via + // fc::json_writer instead of fc::mutable_variant_object to avoid the + // variant-tree -> JSON-string round trip. + json_raw = 3 }; struct http_plugin_defaults { @@ -111,6 +160,28 @@ namespace sysio { add_async_handler(std::move(call), content_type); } + // ---- Streaming-cb registration -------------------------------------------------- + // Endpoints registered via add_handler_stream / add_async_handler_stream invoke + // url_response_stream_callback instead of url_response_callback. The cb's emitter + // closure runs on the http_plugin thread pool, so the api/main thread does no + // JSON serialization work; it just hands a (typically by-move) result struct off + // through the closure capture. + // + // content_type is implicitly application/json - the streaming path never produces + // anything else by design. No json_raw distinction is needed; the json_writer + // output is the response body verbatim. + void add_handler_stream(api_entry_stream&& entry, appbase::exec_queue q, int priority = appbase::priority::medium_low); + void add_api_stream(api_description_stream&& api, appbase::exec_queue q, int priority = appbase::priority::medium_low) { + for (auto& call : api) + add_handler_stream(std::move(call), q, priority); + } + + void add_async_handler_stream(api_entry_stream&& entry); + void add_async_api_stream(api_description_stream&& api) { + for (auto& call : api) + add_async_handler_stream(std::move(call)); + } + /// Register a raw handler that receives the abstract_conn directly. /// Use for endpoints that need to send binary/file responses. void add_raw_handler(string path, api_category category, raw_url_handler handler); @@ -118,7 +189,18 @@ namespace sysio { // standard exception handling for api handlers static void handle_exception( const char *api_name, const char *call_name, const string& body, const url_response_callback& cb ); - void post_http_thread_pool(std::function f); + /// Streaming counterpart to handle_exception: emits an error_results object as + /// JSON via the reflector path instead of building a fc::variant tree. Pass cb + /// by mutable lvalue ref since url_response_stream_callback is move-only; + /// invoking it does not move from cb, so the api lambda may still call cb + /// elsewhere on a different code path (in practice exactly one path runs per + /// request). + static void handle_exception_stream( const char *api_name, const char *call_name, const string& body, url_response_stream_callback& cb ); + + // Accepts a move-only callable so streaming-cb macros can capture + // url_response_stream_callback (move_only_function) into the closure. + // std::function values implicitly convert and so are still accepted. + void post_http_thread_pool(fc::move_only_function f); bool is_on_loopback(api_category category) const; @@ -146,6 +228,18 @@ namespace sysio { size_t bytes_in_flight() const; + /** + * @brief RAII charge against --http-max-bytes-in-flight-mb for a payload captured + * into a queued response closure. + * + * The charge is taken immediately and released when the returned token is + * destroyed -- so a typed result captured into a stream_emitter is accounted for + * exactly as long as it is alive, including the time it sits on the http thread + * pool queue before emission (the streaming analogue of the variant path's + * pre-dispatch in_flight_sizeof charge). Returns an empty token for size 0. + */ + std::shared_ptr reserve_bytes_in_flight(size_t size); + std::atomic& listening(); private: std::shared_ptr my; diff --git a/plugins/http_plugin/include/sysio/http_plugin/macros.hpp b/plugins/http_plugin/include/sysio/http_plugin/macros.hpp index ec6872f5bf..1570deffd6 100644 --- a/plugins/http_plugin/include/sysio/http_plugin/macros.hpp +++ b/plugins/http_plugin/include/sysio/http_plugin/macros.hpp @@ -1,5 +1,9 @@ #pragma once +// Variant-cb async-call macro retained for endpoints that emit a non-JSON content +// type (e.g. prometheus_plugin returning text/plain via fc::variant string payload). +// Streaming JSON endpoints use sysio::bind_stream from . + #define CALL_ASYNC_WITH_400(api_name, category, api_handle, api_namespace, call_name, call_result, http_resp_code, params_type) \ { std::string("/v1/" #api_name "/" #call_name), \ api_category::category, \ @@ -43,40 +47,3 @@ } \ } \ } - - -// call an API which returns either fc::exception_ptr, or a function to be posted on the http thread pool -// for execution (typically doing the final serialization) -// ------------------------------------------------------------------------------------------------------ -#define CALL_WITH_400_POST(api_name, category, api_handle, api_namespace, call_name, call_result, http_resp_code, params_type) \ -{std::string("/v1/" #api_name "/" #call_name), \ - api_category::category, \ - [api_handle, &_http_plugin](string&&, string&& body, url_response_callback&& cb) { \ - auto deadline = api_handle.start(); \ - try { \ - auto params = parse_params(body); \ - using http_fwd_t = std::function()>; \ - /* called on main application thread */ \ - http_fwd_t http_fwd(api_handle.call_name(std::move(params), deadline)); \ - _http_plugin.post_http_thread_pool([resp_code=http_resp_code, cb=std::move(cb), \ - body=std::move(body), \ - http_fwd = std::move(http_fwd)]() { \ - try { \ - chain::t_or_exception result = http_fwd(); \ - if (std::holds_alternative(result)) { \ - try { \ - std::get(result)->rethrow(); \ - } catch (...) { \ - http_plugin::handle_exception(#api_name, #call_name, body, cb); \ - } \ - } else { \ - cb(resp_code, fc::variant(std::get(std::move(result)))); \ - } \ - } catch (...) { \ - http_plugin::handle_exception(#api_name, #call_name, body, cb); \ - } \ - }); \ - } catch (...) { \ - http_plugin::handle_exception(#api_name, #call_name, body, cb); \ - } \ - }} diff --git a/plugins/http_plugin/src/http_plugin.cpp b/plugins/http_plugin/src/http_plugin.cpp index f60b7b9ffd..109fae9bf5 100644 --- a/plugins/http_plugin/src/http_plugin.cpp +++ b/plugins/http_plugin/src/http_plugin.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -227,6 +228,62 @@ namespace sysio { }; return handler; } + + /** + * Streaming-cb counterpart of make_app_thread_url_handler. Posts the api + * lambda onto the requested executor queue (read_only / read_write / etc) + * and forwards a streaming-cb that the api lambda invokes with a closure + * which the http_plugin thread pool will run later. + */ + static detail::internal_url_handler_stream make_app_thread_url_handler_stream(api_entry_stream&& entry, appbase::exec_queue to_queue, int priority) { + detail::internal_url_handler_stream handler; + handler.category = entry.category; + auto next_ptr = std::make_shared(std::move(entry.handler)); + handler.fn = [priority, to_queue, next_ptr=std::move(next_ptr)] + ( detail::abstract_conn_ptr conn, string&& r, string&& b, url_response_stream_callback&& then ) { + if (auto error_str = conn->verify_max_bytes_in_flight(b.size()); !error_str.empty()) { + conn->send_busy_response(std::move(error_str)); + return; + } + + // Mirror the variant-cb handler above: the body is about to sit in the (unbounded) + // app-thread queue, so reserve its bytes against bytes_in_flight until the posted + // work is destroyed (run, thrown, or discarded). Without this, queued streaming + // request bodies accumulate uncounted and evade verify_max_bytes_in_flight. + auto body_in_flight_guard = std::make_shared(conn, b.size()); + + app().executor().post( priority, to_queue, + [next_ptr, conn=std::move(conn), r=std::move(r), b=std::move(b), then=std::move(then), + body_in_flight_guard=std::move(body_in_flight_guard)]() mutable { + try { + if( app().is_quiting() ) return; + (*next_ptr)( std::move(r), std::move(b), std::move(then) ); + } catch( ... ) { + conn->handle_exception(); + } + }); + }; + return handler; + } + + /** + * Streaming-cb counterpart of make_http_thread_url_handler. Runs the api + * lambda directly on the http_plugin thread (the streaming cb's emitter + * runs on the same pool, so the api lambda doing minimal work and + * immediately handing off to the cb is fine). + */ + static detail::internal_url_handler_stream make_http_thread_url_handler_stream(api_entry_stream&& entry) { + detail::internal_url_handler_stream handler; + handler.category = entry.category; + handler.fn = [next=std::move(entry.handler)]( const detail::abstract_conn_ptr& conn, string&& r, string&& b, url_response_stream_callback&& then ) mutable { + try { + next(std::move(r), std::move(b), std::move(then)); + } catch( ... ) { + conn->handle_exception(); + } + }; + return handler; + } bool is_unix_socket_address(const std::string& address) const { return address.starts_with("/") || address.starts_with("./") || address.starts_with("../"); @@ -565,6 +622,11 @@ namespace sysio { void http_plugin::add_handler(api_entry&& entry, appbase::exec_queue q, int priority, http_content_type content_type) { log_add_handler(my.get(), entry); std::string path = entry.path; + // Symmetric with add_handler_stream: a path registered on the streaming map + // would silently shadow this variant registration since lookup checks the + // streaming map first. + SYS_ASSERT( !my->plugin_state->url_handlers_stream.contains(path), chain::plugin_config_exception, + "http url {} is not unique - already registered on the streaming cb path", path ); auto p = my->plugin_state->url_handlers.emplace(path, my->make_app_thread_url_handler(std::move(entry), q, priority, content_type)); SYS_ASSERT( p.second, chain::plugin_config_exception, "http url {} is not unique", path ); } @@ -572,15 +634,53 @@ namespace sysio { void http_plugin::add_async_handler(api_entry&& entry, http_content_type content_type) { log_add_handler(my.get(), entry); std::string path = entry.path; + SYS_ASSERT( !my->plugin_state->url_handlers_stream.contains(path), chain::plugin_config_exception, + "http url {} is not unique - already registered on the streaming cb path", path ); auto p = my->plugin_state->url_handlers.emplace(path, my->make_http_thread_url_handler(std::move(entry), content_type)); SYS_ASSERT( p.second, chain::plugin_config_exception, "http url {} is not unique", path ); } + namespace { + void log_add_handler(http_plugin_impl* my, api_category category, const std::string& path) { + auto addrs = my->addresses_for_category(category); + if (addrs.size()) + addrs = "on " + addrs; + else + addrs = "disabled for category address not configured"; + fc_ilog(logger(), "add {} api url: {} {}", + from_category(category), path, addrs); + } + } + + void http_plugin::add_handler_stream(api_entry_stream&& entry, appbase::exec_queue q, int priority) { + log_add_handler(my.get(), entry.category, entry.path); + std::string path = entry.path; + // The two handler maps are keyed identically; assert disjointness so a stale + // duplicate registration on the variant path doesn't silently shadow the + // streaming registration (or vice versa). + auto& vmap = my->plugin_state->url_handlers; + auto& smap = my->plugin_state->url_handlers_stream; + SYS_ASSERT( !vmap.contains(path), chain::plugin_config_exception, + "http url {} is not unique - already registered on the variant cb path", path ); + SYS_ASSERT( !smap.contains(path), chain::plugin_config_exception, + "http url {} is not unique - already registered on the streaming cb path", path ); + smap.emplace(path, my->make_app_thread_url_handler_stream(std::move(entry), q, priority)); + } + + void http_plugin::add_async_handler_stream(api_entry_stream&& entry) { + log_add_handler(my.get(), entry.category, entry.path); + std::string path = entry.path; + auto& vmap = my->plugin_state->url_handlers; + auto& smap = my->plugin_state->url_handlers_stream; + SYS_ASSERT( !vmap.contains(path), chain::plugin_config_exception, + "http url {} is not unique - already registered on the variant cb path", path ); + SYS_ASSERT( !smap.contains(path), chain::plugin_config_exception, + "http url {} is not unique - already registered on the streaming cb path", path ); + smap.emplace(path, my->make_http_thread_url_handler_stream(std::move(entry))); + } + void http_plugin::add_raw_handler(string path, api_category category, raw_url_handler handler) { - fc_ilog(logger(), "add {} api url: {} {}", - from_category(category), path, - my->addresses_for_category(category).empty() ? "disabled for category address not configured" - : "on " + my->addresses_for_category(category)); + log_add_handler(my.get(), category, path); detail::internal_url_handler internal_handler; internal_handler.category = category; internal_handler.content_type = http_content_type::json; // default, though raw handlers manage their own responses @@ -592,71 +692,97 @@ namespace sysio { conn->handle_exception(); } }; + SYS_ASSERT( !my->plugin_state->url_handlers_stream.contains(path), chain::plugin_config_exception, + "http url {} is not unique - already registered on the streaming cb path", path ); auto p = my->plugin_state->url_handlers.emplace(std::move(path), std::move(internal_handler)); SYS_ASSERT(p.second, chain::plugin_config_exception, "http url {} is not unique", p.first->first); } - void http_plugin::post_http_thread_pool(std::function f) { + void http_plugin::post_http_thread_pool(fc::move_only_function f) { if( f ) - boost::asio::post( my->plugin_state->thread_pool.get_executor(), f ); + boost::asio::post( my->plugin_state->thread_pool.get_executor(), std::move(f) ); } - void http_plugin::handle_exception( const char* api_name, const char* call_name, const string& body, const url_response_callback& cb) { - try { + namespace { + // Shared catch-chain for handle_exception / handle_exception_stream. Logs the + // in-flight exception and invokes `emit` with (http_code, error_results); the + // caller decides how to deliver the result via its specific cb shape (variant + // path or streaming closure). + template + void classify_current_exception(const char* api_name, const char* call_name, + const std::string& body, Emit&& emit) { try { - throw; - } catch (chain::unknown_block_exception& e) { - fc_dlog( logger(), "Unknown block while processing {}.{}: {}", - api_name, call_name, e.to_detail_string() ); - error_results results{400, "Unknown Block", error_results::error_info(e, verbose_http_errors)}; - cb( 400, fc::variant( results )); - } catch (chain::invalid_http_request& e) { - fc_dlog( logger(), "Invalid http request while processing {}.{}: {}", - api_name, call_name, e.to_detail_string() ); - error_results results{400, "Invalid Request", error_results::error_info(e, verbose_http_errors)}; - cb( 400, fc::variant( results )); - } catch (chain::account_query_exception& e) { - fc_dlog( logger(), "Account query exception while processing {}.{}: {}", - api_name, call_name, e.to_detail_string() ); - error_results results{400, "Account lookup", error_results::error_info(e, verbose_http_errors)}; - cb( 400, fc::variant( results )); - } catch (chain::unsatisfied_authorization& e) { - fc_dlog( logger(), "Auth error while processing {}.{}: {}", - api_name, call_name, e.to_detail_string() ); - error_results results{401, "UnAuthorized", error_results::error_info(e, verbose_http_errors)}; - cb( 401, fc::variant( results )); - } catch (chain::tx_duplicate& e) { - fc_dlog( logger(), "Duplicate trx while processing {}.{}: {}", - api_name, call_name, e.to_detail_string() ); - error_results results{409, "Conflict", error_results::error_info(e, verbose_http_errors)}; - cb( 409, fc::variant( results )); - } catch (fc::eof_exception& e) { - fc_elog( logger(), "Unable to parse arguments to {}.{}", api_name, call_name ); - fc_dlog( logger(), "Bad arguments: {}", body ); - error_results results{422, "Unprocessable Entity", error_results::error_info(e, verbose_http_errors)}; - cb( 422, fc::variant( results )); - } catch (fc::exception& e) { - fc_dlog( logger(), "Exception while processing {}.{}: {}", - api_name, call_name, e.to_detail_string() ); - error_results results{500, "Internal Service Error", error_results::error_info(e, verbose_http_errors)}; - cb( 500, fc::variant( results )); - } catch (std::exception& e) { - fc_dlog( logger(), "STD Exception encountered while processing {}.{}: {}", - api_name, call_name, e.what() ); - error_results results{500, "Internal Service Error", error_results::error_info(fc::exception( FC_LOG_MESSAGE( error, "{}", e.what())), verbose_http_errors)}; - cb( 500, fc::variant( results )); + try { + throw; + } catch (chain::unknown_block_exception& e) { + fc_dlog( logger(), "Unknown block while processing {}.{}: {}", + api_name, call_name, e.to_detail_string() ); + emit(400, error_results{400, "Unknown Block", error_results::error_info(e, verbose_http_errors)}); + } catch (chain::invalid_http_request& e) { + fc_dlog( logger(), "Invalid http request while processing {}.{}: {}", + api_name, call_name, e.to_detail_string() ); + emit(400, error_results{400, "Invalid Request", error_results::error_info(e, verbose_http_errors)}); + } catch (chain::account_query_exception& e) { + fc_dlog( logger(), "Account query exception while processing {}.{}: {}", + api_name, call_name, e.to_detail_string() ); + emit(400, error_results{400, "Account lookup", error_results::error_info(e, verbose_http_errors)}); + } catch (chain::unsatisfied_authorization& e) { + fc_dlog( logger(), "Auth error while processing {}.{}: {}", + api_name, call_name, e.to_detail_string() ); + emit(401, error_results{401, "UnAuthorized", error_results::error_info(e, verbose_http_errors)}); + } catch (chain::tx_duplicate& e) { + fc_dlog( logger(), "Duplicate trx while processing {}.{}: {}", + api_name, call_name, e.to_detail_string() ); + emit(409, error_results{409, "Conflict", error_results::error_info(e, verbose_http_errors)}); + } catch (fc::eof_exception& e) { + fc_elog( logger(), "Unable to parse arguments to {}.{}", api_name, call_name ); + fc_dlog( logger(), "Bad arguments: {}", body ); + emit(422, error_results{422, "Unprocessable Entity", error_results::error_info(e, verbose_http_errors)}); + } catch (fc::exception& e) { + fc_dlog( logger(), "Exception while processing {}.{}: {}", + api_name, call_name, e.to_detail_string() ); + emit(500, error_results{500, "Internal Service Error", error_results::error_info(e, verbose_http_errors)}); + } catch (std::exception& e) { + fc_dlog( logger(), "STD Exception encountered while processing {}.{}: {}", + api_name, call_name, e.what() ); + emit(500, error_results{500, "Internal Service Error", + error_results::error_info(fc::exception( FC_LOG_MESSAGE( error, "{}", e.what())), verbose_http_errors)}); + } catch (...) { + fc_elog( logger(), "Unknown Exception encountered while processing {}.{}", + api_name, call_name ); + emit(500, error_results{500, "Internal Service Error", + error_results::error_info(fc::exception( FC_LOG_MESSAGE( error, "Unknown Exception" )), verbose_http_errors)}); + } } catch (...) { - fc_elog( logger(), "Unknown Exception encountered while processing {}.{}", - api_name, call_name ); - error_results results{500, "Internal Service Error", - error_results::error_info(fc::exception( FC_LOG_MESSAGE( error, "Unknown Exception" )), verbose_http_errors)}; - cb( 500, fc::variant( results )); + std::cerr << "Exception attempting to handle exception for " << api_name << "." << call_name << std::endl; } - } catch (...) { - std::cerr << "Exception attempting to handle exception for " << api_name << "." << call_name << std::endl; } } + void http_plugin::handle_exception( const char* api_name, const char* call_name, const string& body, const url_response_callback& cb) { + classify_current_exception(api_name, call_name, body, [&cb](int code, error_results r) { + cb(code, fc::variant(std::move(r))); + }); + } + + void http_plugin::handle_exception_stream( const char* api_name, const char* call_name, const string& body, url_response_stream_callback& cb) { + // bind_stream's outer catch can land here AFTER cb has been moved into a + // downstream lambda (eg if post_http_thread_pool throws after the move). + // Calling an empty move_only_function would throw bad_function_call on top + // of the original exception; classify_current_exception would log THAT one, + // and the client would never get a response. Log the original exception + // and return so the original failure is at least visible. + if (!cb) { + classify_current_exception(api_name, call_name, body, [](int, error_results){}); + return; + } + classify_current_exception(api_name, call_name, body, [&cb](int code, error_results r) { + cb(code, [r = std::move(r)](fc::json_writer& w) mutable { + fc::to_json_stream(r, w); + }); + }); + } + bool http_plugin::is_on_loopback(api_category category) const { return std::all_of(my->categories_by_address.begin(), my->categories_by_address.end(), [&category, this](const auto& entry) { @@ -698,6 +824,17 @@ namespace sysio { return my->plugin_state->bytes_in_flight; } + std::shared_ptr http_plugin::reserve_bytes_in_flight(size_t size) { + if (size == 0) + return {}; + my->plugin_state->bytes_in_flight += size; + // Deleter-only token: the null pointer carries no object, but the deleter still runs + // exactly once when the last copy is destroyed, releasing the charge. Capturing the + // plugin_state shared_ptr keeps the counter alive past plugin shutdown. + return std::shared_ptr(static_cast(nullptr), + [state = my->plugin_state, size](void*) { state->bytes_in_flight -= size; }); + } + std::atomic& http_plugin::listening() { return my->listening; } diff --git a/plugins/http_plugin/test/CMakeLists.txt b/plugins/http_plugin/test/CMakeLists.txt index 478dbd7a4d..97c8e17f9f 100644 --- a/plugins/http_plugin/test/CMakeLists.txt +++ b/plugins/http_plugin/test/CMakeLists.txt @@ -4,11 +4,21 @@ add_executable( http_plugin_unit_tests ${UNIT_TESTS}) target_link_libraries( http_plugin_unit_tests PRIVATE appbase - PRIVATE http_plugin + PRIVATE http_plugin ${CMAKE_DL_LIBS} ${PLATFORM_SPECIFIC_LIBS} ) +# Header-only chain dependencies for the stored-exception status-code test: +# bind_stream.hpp needs sysio/chain/plugin_interface.hpp (chain_interface has no +# CMake target -- see producer_plugin's CMakeLists for the same pattern) plus the +# chain exception / t_or_exception headers. Everything used is inline, so include +# dirs suffice; linking sysio_chain would drag fc's PUBLIC Boost::unit_test_framework +# onto the link line and collide with this test's header-only Boost.Test. target_include_directories( http_plugin_unit_tests PRIVATE ${CMAKE_SOURCE_DIR}/plugins/http_plugin/include + ${CMAKE_SOURCE_DIR}/plugins/chain_interface/include + ${CMAKE_SOURCE_DIR}/libraries/chain/include + ${CMAKE_BINARY_DIR}/libraries/chain/include + ${CMAKE_SOURCE_DIR}/libraries/chaindb/include ${CMAKE_SOURCE_DIR}/tests ) add_np_test( NAME http_plugin_unit_tests COMMAND http_plugin_unit_tests ) diff --git a/plugins/http_plugin/test/unit_tests.cpp b/plugins/http_plugin/test/unit_tests.cpp index cc8eb49202..51571b48da 100644 --- a/plugins/http_plugin/test/unit_tests.cpp +++ b/plugins/http_plugin/test/unit_tests.cpp @@ -1,6 +1,10 @@ #include #include #include +#include +#include + +#include #include #include @@ -41,6 +45,12 @@ constexpr uint32_t bytes_in_flight_index = 3; constexpr uint32_t requests_in_flight_index = 4; constexpr uint32_t request_body_bytes_in_flight_index = 5; constexpr uint32_t category_uw_index = 6; +constexpr uint32_t stream_status_codes_index = 7; +constexpr uint32_t stream_request_body_index = 8; +constexpr uint32_t stream_response_bytes_index = 9; +constexpr uint32_t stream_over_budget_index = 10; +constexpr uint32_t stream_single_token_index = 11; +constexpr uint32_t stream_queued_payload_index = 12; // Keeps unsharded IPv6 probe coverage on the historical port 9999. constexpr uint32_t ipv6_probe_index = 2; @@ -916,5 +926,451 @@ BOOST_FIXTURE_TEST_CASE(requests_in_flight, http_plugin_test_fixture) { wait_for_no_requests_in_flight(); } +namespace { + /// Minimal api handle for driving bind_stream's stored-exception delivery paths end-to-end. + /// Both methods deliver a DERIVED exception through the stored fc::exception_ptr alternative + /// (never a live throw), which is the path where a `throw *ptr` would slice the dynamic type + /// and defeat classify_current_exception's specific-type -> status-code mapping. + struct stored_exception_api { + /// dispatch::async -- stored tx_duplicate must classify as 409 Conflict. + void duplicate_trx(sysio::chain::plugin_interface::next_function next) { + next(sysio::chain::plugin_interface::next_function_variant{ + fc::exception_ptr{std::make_shared( + FC_LOG_MESSAGE(error, "duplicate transaction"))}}); + } + + /// dispatch::post -- Phase-2 closure returning a stored unknown_block_exception must + /// classify as 400 Unknown Block. + std::function()> missing_block() { + return []() -> sysio::chain::t_or_exception { + return fc::exception_ptr{std::make_shared( + FC_LOG_MESSAGE(error, "no such block"))}; + }; + } + }; +} + +// Stored exceptions delivered through bind_stream (async next_function and post http_fwd +// alternatives) must reach classify_current_exception with their dynamic type intact so the +// specific-type catches map them to the right HTTP status. A regression to `throw *ptr` +// (slicing to fc::exception) turns both of these into 500 Internal Service Error. +BOOST_FIXTURE_TEST_CASE(stream_stored_exception_status_codes, http_plugin_test_fixture) { + const std::string endpoint = test_http_endpoint("127.0.0.1", stream_status_codes_index); + const std::string server_address = "--http-server-address=" + endpoint; + const std::string port = test_http_port(stream_status_codes_index); + + http_plugin* http_plugin = init({"--plugin=sysio::http_plugin", server_address.c_str()}); + BOOST_REQUIRE(http_plugin); + + http_plugin->add_api_stream({ + bind_stream<&stored_exception_api::duplicate_trx, dispatch::async>( + *http_plugin, stored_exception_api{}, "/v1/test/duplicate_trx", + api_category::node, http_params_types::no_params, 200), + bind_stream<&stored_exception_api::missing_block, dispatch::post>( + *http_plugin, stored_exception_api{}, "/v1/test/missing_block", + api_category::node, http_params_types::no_params, 200), + }, appbase::exec_queue::read_write); + + boost::asio::io_context ctx; + boost::asio::ip::tcp::resolver resolver(ctx); + + auto get_status = [&](const char* path) { + boost::asio::ip::tcp::socket s(ctx, boost::asio::ip::tcp::v4()); + boost::asio::connect(s, resolver.resolve("127.0.0.1", port)); + boost::beast::http::request req(boost::beast::http::verb::get, path, 11); + req.set(http::field::host, endpoint); + boost::beast::http::write(s, req); + + boost::beast::http::response resp; + boost::beast::flat_buffer buffer; + boost::beast::http::read(s, buffer, resp); + return resp.result(); + }; + + BOOST_CHECK(get_status("/v1/test/duplicate_trx") == boost::beast::http::status::conflict); // 409 + BOOST_CHECK(get_status("/v1/test/missing_block") == boost::beast::http::status::bad_request); // 400 +} + +// Streaming-path twin of request_body_bytes_in_flight: request bodies queued to the app thread +// through the streaming-cb registration path (make_app_thread_url_handler_stream) must take the +// same bytes_in_flight reservation the variant path takes. Without it the sampled value below +// reads 0 (queued streaming bodies uncounted -- the WSA-176 vector); with it, >= body size. The +// budget must also drain to zero afterwards, proving the reservation releases exactly once. +BOOST_FIXTURE_TEST_CASE(request_body_bytes_in_flight_stream, http_plugin_test_fixture) { + const std::string endpoint = test_http_endpoint("127.0.0.1", stream_request_body_index); + const std::string server_address = "--http-server-address=" + endpoint; + const std::string port = test_http_port(stream_request_body_index); + + http_plugin* http_plugin = init({"--plugin=sysio::http_plugin", + server_address.c_str(), + "--http-max-bytes-in-flight-mb=64"}); + BOOST_REQUIRE(http_plugin); + + std::atomic observed_body_size{0}; + std::atomic observed_bytes_in_flight{0}; + http_plugin->add_api_stream({{std::string("/echo_body_stream"), api_category::node, + [&](string&&, string&& body, url_response_stream_callback&& cb) { + // Sampled on the app thread while the request body is still in + // flight and before any response bytes are accounted, so it + // reflects only the request-body reservation. + observed_body_size.store(body.size()); + observed_bytes_in_flight.store(http_plugin->bytes_in_flight()); + cb(200, [](fc::json_writer& w) { w.value_string("ok"); }); + }}}, appbase::exec_queue::read_write); + + boost::asio::io_context ctx; + boost::asio::ip::tcp::resolver resolver(ctx); + + // 1 MiB body: comfortably under the 2 MiB max-body-size and the 64 MiB in-flight budget. + const size_t body_size = 1024 * 1024; + + auto post_body = [&]() { + boost::asio::ip::tcp::socket s(ctx, boost::asio::ip::tcp::v4()); + boost::asio::connect(s, resolver.resolve("127.0.0.1", port)); + boost::beast::http::request req(boost::beast::http::verb::post, "/echo_body_stream", 11); + req.keep_alive(true); + req.set(http::field::host, endpoint); + req.body() = std::string(body_size, 'x'); + req.prepare_payload(); + boost::beast::http::write(s, req); + + boost::beast::http::response resp; + boost::beast::flat_buffer buffer; + boost::beast::http::read(s, buffer, resp); + return resp.result(); + }; + + auto wait_for_no_bytes_in_flight = [&]() { + uint16_t max = std::numeric_limits::max(); + while (http_plugin->requests_in_flight() > 0 && --max) + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + BOOST_CHECK(max > 0); + BOOST_CHECK_EQUAL(http_plugin->bytes_in_flight(), 0u); + }; + + BOOST_REQUIRE(post_body() == boost::beast::http::status::ok); + BOOST_CHECK_EQUAL(observed_body_size.load(), body_size); + // Regression assertion: 0 without the reservation in make_app_thread_url_handler_stream. + BOOST_CHECK_GE(observed_bytes_in_flight.load(), body_size); + wait_for_no_bytes_in_flight(); + + // A second request must be admitted and counted the same way: no leaked or double-counted + // reservation. + observed_bytes_in_flight.store(0); + BOOST_REQUIRE(post_body() == boost::beast::http::status::ok); + BOOST_CHECK_GE(observed_bytes_in_flight.load(), body_size); + wait_for_no_bytes_in_flight(); +} + +// Streaming responses must be charged against bytes_in_flight WHILE the emitter runs, not only +// after the body is fully materialized: json_writer's growth guard settles the buffer into the +// budget every stride. The emitter below samples the budget mid-emission; without incremental +// accounting the sampled value reads 0 (the response was invisible to the budget until complete). +BOOST_FIXTURE_TEST_CASE(stream_response_bytes_in_flight, http_plugin_test_fixture) { + const std::string endpoint = test_http_endpoint("127.0.0.1", stream_response_bytes_index); + const std::string server_address = "--http-server-address=" + endpoint; + const std::string port = test_http_port(stream_response_bytes_index); + + http_plugin* http_plugin = init({"--plugin=sysio::http_plugin", + server_address.c_str(), + "--http-max-bytes-in-flight-mb=64"}); + BOOST_REQUIRE(http_plugin); + + // 32 x 64KiB chunks ~= 2 MiB emitted; sample the budget halfway through emission. + constexpr size_t chunk_size = 64 * 1024; + constexpr size_t chunk_count = 32; + std::atomic observed_bytes_in_flight{0}; + http_plugin->add_api_stream({{std::string("/2megabyte_stream"), api_category::node, + [&](string&&, string&&, url_response_stream_callback&& cb) { + cb(200, [&](fc::json_writer& w) { + w.begin_array(); + for (size_t i = 0; i < chunk_count; ++i) { + w.value_string(std::string(chunk_size, 'x')); + if (i == chunk_count / 2) + observed_bytes_in_flight.store(http_plugin->bytes_in_flight()); + } + w.end_array(); + }); + }}}, appbase::exec_queue::read_write); + + boost::asio::io_context ctx; + boost::asio::ip::tcp::resolver resolver(ctx); + + auto get_response = [&]() { + boost::asio::ip::tcp::socket s(ctx, boost::asio::ip::tcp::v4()); + boost::asio::connect(s, resolver.resolve("127.0.0.1", port)); + boost::beast::http::request req(boost::beast::http::verb::get, "/2megabyte_stream", 11); + req.set(http::field::host, endpoint); + boost::beast::http::write(s, req); + + boost::beast::http::response resp; + boost::beast::flat_buffer buffer; + boost::beast::http::read(s, buffer, resp); + return resp; + }; + + auto resp = get_response(); + BOOST_REQUIRE(resp.result() == boost::beast::http::status::ok); + BOOST_CHECK_GT(resp.body().size(), chunk_size * chunk_count); + // Regression assertion: 0 without the growth guard's incremental settle. At the halfway + // sample ~1 MiB has been emitted; charging lags by at most one stride, so half that is a + // safely conservative floor. + BOOST_CHECK_GE(observed_bytes_in_flight.load(), chunk_size * chunk_count / 4); + + uint16_t max = std::numeric_limits::max(); + while (http_plugin->requests_in_flight() > 0 && --max) + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + BOOST_CHECK(max > 0); + BOOST_CHECK_EQUAL(http_plugin->bytes_in_flight(), 0u); +} + +// Over-budget streaming responses must abort DURING emission (503 busy) instead of +// materializing in full and only then being rejected: the growth guard throws once +// verify_max_bytes_in_flight rejects the incrementally-charged buffer. The chunk counter +// proves early abort; the follow-up request proves the budget released and the plugin +// still serves. +BOOST_FIXTURE_TEST_CASE(stream_response_over_budget_aborts_early, http_plugin_test_fixture) { + const std::string endpoint = test_http_endpoint("127.0.0.1", stream_over_budget_index); + const std::string server_address = "--http-server-address=" + endpoint; + const std::string port = test_http_port(stream_over_budget_index); + + http_plugin* http_plugin = init({"--plugin=sysio::http_plugin", + server_address.c_str(), + "--http-max-bytes-in-flight-mb=1"}); + BOOST_REQUIRE(http_plugin); + + // Attempts 32 MiB against a 1 MiB budget; the guard must stop it within a stride or two + // past the budget. + constexpr size_t chunk_size = 64 * 1024; + constexpr size_t chunk_count = 512; + std::atomic chunks_emitted{0}; + http_plugin->add_api_stream({{std::string("/32megabyte_stream"), api_category::node, + [&](string&&, string&&, url_response_stream_callback&& cb) { + cb(200, [&](fc::json_writer& w) { + w.begin_array(); + for (size_t i = 0; i < chunk_count; ++i) { + w.value_string(std::string(chunk_size, 'x')); + chunks_emitted.fetch_add(1); + } + w.end_array(); + }); + }, + }, + {std::string("/small_stream"), api_category::node, + [&](string&&, string&&, url_response_stream_callback&& cb) { + cb(200, [](fc::json_writer& w) { w.value_string("ok"); }); + }}}, appbase::exec_queue::read_write); + + boost::asio::io_context ctx; + boost::asio::ip::tcp::resolver resolver(ctx); + + auto get_status = [&](const char* path) { + boost::asio::ip::tcp::socket s(ctx, boost::asio::ip::tcp::v4()); + boost::asio::connect(s, resolver.resolve("127.0.0.1", port)); + boost::beast::http::request req(boost::beast::http::verb::get, path, 11); + req.set(http::field::host, endpoint); + boost::beast::http::write(s, req); + + boost::beast::http::response resp; + boost::beast::flat_buffer buffer; + boost::beast::http::read(s, buffer, resp); + return resp.result(); + }; + + BOOST_CHECK(get_status("/32megabyte_stream") == boost::beast::http::status::service_unavailable); + // Early abort: the guard threw within a couple of strides past the 1 MiB budget, far + // short of the 512 chunks the emitter would produce unguarded. + BOOST_CHECK_LT(chunks_emitted.load(), 64u); + BOOST_CHECK_GT(chunks_emitted.load(), 0u); + + uint16_t max = std::numeric_limits::max(); + while (http_plugin->requests_in_flight() > 0 && --max) + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + BOOST_CHECK(max > 0); + BOOST_CHECK_EQUAL(http_plugin->bytes_in_flight(), 0u); + + // The budget drained: a small streaming response is admitted and served normally. + BOOST_CHECK(get_status("/small_stream") == boost::beast::http::status::ok); +} + +// One oversized token -- a single 32 MiB string value -- must abort mid-token against the +// budget: the growth guard fires inside the escape loop, so the emitter's post-token +// statement never runs. Distinguishes intra-token enforcement from token-boundary-only +// checks, which would materialize the full token first and only then reject (same 503, but +// token_completed would read true). +BOOST_FIXTURE_TEST_CASE(stream_response_single_token_over_budget, http_plugin_test_fixture) { + const std::string endpoint = test_http_endpoint("127.0.0.1", stream_single_token_index); + const std::string server_address = "--http-server-address=" + endpoint; + const std::string port = test_http_port(stream_single_token_index); + + http_plugin* http_plugin = init({"--plugin=sysio::http_plugin", + server_address.c_str(), + "--http-max-bytes-in-flight-mb=1"}); + BOOST_REQUIRE(http_plugin); + + std::atomic token_completed{false}; + http_plugin->add_api_stream({{std::string("/one_giant_token"), api_category::node, + [&](string&&, string&&, url_response_stream_callback&& cb) { + cb(200, [&](fc::json_writer& w) { + w.value_string(std::string(32u * 1024 * 1024, 'x')); + token_completed.store(true); + }); + }}}, appbase::exec_queue::read_write); + + boost::asio::io_context ctx; + boost::asio::ip::tcp::resolver resolver(ctx); + + auto get_status = [&](const char* path) { + boost::asio::ip::tcp::socket s(ctx, boost::asio::ip::tcp::v4()); + boost::asio::connect(s, resolver.resolve("127.0.0.1", port)); + boost::beast::http::request req(boost::beast::http::verb::get, path, 11); + req.set(http::field::host, endpoint); + boost::beast::http::write(s, req); + + boost::beast::http::response resp; + boost::beast::flat_buffer buffer; + boost::beast::http::read(s, buffer, resp); + return resp.result(); + }; + + BOOST_CHECK(get_status("/one_giant_token") == boost::beast::http::status::service_unavailable); + BOOST_CHECK(!token_completed.load()); + + uint16_t max = std::numeric_limits::max(); + while (http_plugin->requests_in_flight() > 0 && --max) + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + BOOST_CHECK(max > 0); + BOOST_CHECK_EQUAL(http_plugin->bytes_in_flight(), 0u); +} + +namespace { + /// FC_REFLECT'd payload big enough that its packed-size reservation is unmistakable in + /// the bytes_in_flight sample (and comfortably below the test budget). + struct queued_payload_result { + std::vector data; + }; + + /// Minimal api handle for driving bind_stream's dispatch::sync queued-payload + /// reservation end-to-end. The method blocks (on the app thread) until the test has + /// stalled the single http pool thread, so the emitter -- and the typed result it + /// captures -- must sit queued behind the stall the moment it is produced. + struct queued_payload_api { + std::atomic* entered; + std::atomic* proceed; + size_t payload_size; + + queued_payload_result make_big() { + entered->store(true); + while (!proceed->load()) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + return queued_payload_result{std::vector(payload_size, 'q')}; + } + }; +} +FC_REFLECT(queued_payload_result, (data)) + +// Typed results captured into a stream_emitter and queued to the http pool must be charged +// against bytes_in_flight for the time they sit in the queue -- the variant path charges its +// in_flight_sizeof estimate before dispatch, and the streaming path must not be blinder. +// With --http-threads=1, /stall_stream parks the only pool thread inside a latched emitter +// while the bind_stream endpoint's result is produced on the app thread; the peak sampled +// below reads ~0 without the source-size reservation and >= the payload size with it. The +// budget must then drain to zero, proving the reservation releases exactly once. +BOOST_FIXTURE_TEST_CASE(stream_queued_payload_reserved, http_plugin_test_fixture) { + const std::string endpoint = test_http_endpoint("127.0.0.1", stream_queued_payload_index); + const std::string server_address = "--http-server-address=" + endpoint; + const std::string port = test_http_port(stream_queued_payload_index); + + http_plugin* http_plugin = init({"--plugin=sysio::http_plugin", + server_address.c_str(), + "--http-threads=1", + "--http-max-bytes-in-flight-mb=64"}); + BOOST_REQUIRE(http_plugin); + + constexpr size_t payload_size = 1024 * 1024; + std::atomic entered{false}; + std::atomic proceed{false}; + std::atomic stalled{false}; + std::atomic release{false}; + + http_plugin->add_api_stream({ + bind_stream<&queued_payload_api::make_big, dispatch::sync>( + *http_plugin, queued_payload_api{&entered, &proceed, payload_size}, + "/v1/test/queued_payload", api_category::node, http_params_types::no_params, 200), + }, appbase::exec_queue::read_write); + + // Async streaming handlers run inline on the pool thread that read the request, and the + // response dispatch short-circuits on that same thread -- so this emitter's latch parks + // the pool's only thread. + http_plugin->add_async_api_stream({{std::string("/stall_stream"), api_category::node, + [&](string&&, string&&, url_response_stream_callback&& cb) { + cb(200, [&](fc::json_writer& w) { + stalled.store(true); + while (!release.load()) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + w.value_string("ok"); + }); + }}}); + + boost::asio::io_context ctx; + boost::asio::ip::tcp::resolver resolver(ctx); + + auto send_request = [&](const char* path) { + auto s = std::make_shared(ctx, boost::asio::ip::tcp::v4()); + boost::asio::connect(*s, resolver.resolve("127.0.0.1", port)); + boost::beast::http::request req(boost::beast::http::verb::get, path, 11); + req.set(http::field::host, endpoint); + boost::beast::http::write(*s, req); + return s; + }; + auto read_response = [&](boost::asio::ip::tcp::socket& s) { + boost::beast::http::response_parser parser; + parser.body_limit(16 * 1024 * 1024); // hex-encoded payload is ~2x its byte size + boost::beast::flat_buffer buffer; + boost::beast::http::read(s, buffer, parser); + return parser.release(); + }; + auto wait_for = [](auto&& pred) { // ~10s cap: pool scheduling is slow under sanitizers + for (int i = 0; i < 10000; ++i) { + if (pred()) + return true; + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + return pred(); + }; + + auto payload_socket = send_request("/v1/test/queued_payload"); + BOOST_REQUIRE(wait_for([&] { return entered.load(); })); // request read; app thread inside make_big + auto stall_socket = send_request("/stall_stream"); + BOOST_REQUIRE(wait_for([&] { return stalled.load(); })); // pool's only thread now parked + proceed.store(true); // result produced -> emitter queued behind the stall + + // Regression sample: reads ~0 without the reservation riding the queued emitter closure. + size_t peak = 0; + wait_for([&] { + peak = std::max(peak, http_plugin->bytes_in_flight()); + return peak >= payload_size; + }); + BOOST_CHECK_GE(peak, payload_size); + + release.store(true); + auto stall_resp = read_response(*stall_socket); + BOOST_CHECK(stall_resp.result() == boost::beast::http::status::ok); + auto payload_resp = read_response(*payload_socket); + BOOST_CHECK(payload_resp.result() == boost::beast::http::status::ok); + BOOST_CHECK_GT(payload_resp.body().size(), 2 * payload_size); // hex-encoded vector + + // requests_in_flight tracks session lifetime, so the keep-alive sockets must close + // before the drain below can reach zero. + stall_socket.reset(); + payload_socket.reset(); + + uint16_t max = std::numeric_limits::max(); + while (http_plugin->requests_in_flight() > 0 && --max) + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + BOOST_CHECK(max > 0); + BOOST_CHECK_EQUAL(http_plugin->bytes_in_flight(), 0u); +} + //A warning for future tests: destruction of http_plugin_test_fixture sometimes does not destroy http_plugin's listeners. Tests // added in the future should avoid reusing ports of other tests in http_plugin_unit_tests. diff --git a/plugins/net_api_plugin/src/net_api_plugin.cpp b/plugins/net_api_plugin/src/net_api_plugin.cpp index 4599ccf5f3..b7963063ee 100644 --- a/plugins/net_api_plugin/src/net_api_plugin.cpp +++ b/plugins/net_api_plugin/src/net_api_plugin.cpp @@ -1,69 +1,38 @@ #include #include #include +#include #include #include #include -namespace sysio { namespace detail { - struct net_api_plugin_empty {}; -}} - -FC_REFLECT(sysio::detail::net_api_plugin_empty, ); - namespace sysio { using namespace sysio; -#define CALL_WITH_400(api_name, category, api_handle, call_name, INVOKE, http_response_code) \ -{std::string("/v1/" #api_name "/" #call_name), \ - api_category::category, \ - [&api_handle](string&&, string&& body, url_response_callback&& cb) mutable { \ - try { \ - INVOKE \ - cb(http_response_code, fc::variant(result)); \ - } catch (...) { \ - http_plugin::handle_exception(#api_name, #call_name, body, cb); \ - } \ - }} - -#define INVOKE_R_R(api_handle, call_name, in_param) \ - auto params = parse_params(body);\ - fc::variant result( api_handle.call_name( std::move(params) ) ); - -#define INVOKE_R_V(api_handle, call_name) \ - body = parse_params(body); \ - auto result = api_handle.call_name(); - -#define INVOKE_V_R(api_handle, call_name, in_param) \ - auto params = parse_params(body);\ - api_handle.call_name( std::move(params) ); \ - sysio::detail::net_api_plugin_empty result; - -#define INVOKE_V_V(api_handle, call_name) \ - body = parse_params(body); \ - api_handle.call_name(); \ - sysio::detail::net_api_plugin_empty result; - - void net_api_plugin::plugin_startup() { dlog("starting net_api_plugin"); // lifetime of plugin is lifetime of application auto& net_mgr = app().get_plugin(); - app().get_plugin().add_async_api({ - CALL_WITH_400(net, net_rw, net_mgr, connect, - INVOKE_R_R(net_mgr, connect, std::string), 201), - CALL_WITH_400(net, net_rw, net_mgr, disconnect, - INVOKE_R_R(net_mgr, disconnect, std::string), 201), - CALL_WITH_400(net, net_ro, net_mgr, status, - INVOKE_R_R(net_mgr, status, std::string), 201), - CALL_WITH_400(net, net_ro, net_mgr, connections, - INVOKE_R_V(net_mgr, connections), 201), - CALL_WITH_400(net, net_ro, net_mgr, bp_gossip_peers, - INVOKE_R_V(net_mgr, bp_gossip_peers), 201), - } ); + auto& _http_plugin = app().get_plugin(); + + using cat = api_category; + using pt = http_params_types; + + _http_plugin.add_async_api_stream({ + bind_stream<&net_plugin::connect, dispatch::sync>( + _http_plugin, std::ref(net_mgr), "/v1/net/connect", cat::net_rw, pt::params_required, 201), + bind_stream<&net_plugin::disconnect, dispatch::sync>( + _http_plugin, std::ref(net_mgr), "/v1/net/disconnect", cat::net_rw, pt::params_required, 201), + bind_stream<&net_plugin::status, dispatch::sync>( + _http_plugin, std::ref(net_mgr), "/v1/net/status", cat::net_ro, pt::params_required, 201), + bind_stream<&net_plugin::connections, dispatch::sync>( + _http_plugin, std::ref(net_mgr), "/v1/net/connections", cat::net_ro, pt::no_params, 201), + bind_stream<&net_plugin::bp_gossip_peers, dispatch::sync>( + _http_plugin, std::ref(net_mgr), "/v1/net/bp_gossip_peers", cat::net_ro, pt::no_params, 201), + }); } void net_api_plugin::plugin_initialize(const variables_map& options) { @@ -82,11 +51,4 @@ void net_api_plugin::plugin_initialize(const variables_map& options) { } FC_LOG_AND_RETHROW() } - -#undef INVOKE_R_R -#undef INVOKE_R_V -#undef INVOKE_V_R -#undef INVOKE_V_V -#undef CALL - } diff --git a/plugins/producer_api_plugin/src/producer_api_plugin.cpp b/plugins/producer_api_plugin/src/producer_api_plugin.cpp index cc01532681..098d5bc7ee 100644 --- a/plugins/producer_api_plugin/src/producer_api_plugin.cpp +++ b/plugins/producer_api_plugin/src/producer_api_plugin.cpp @@ -1,142 +1,83 @@ #include #include +#include #include #include #include -namespace sysio { namespace detail { - struct producer_api_plugin_response { - std::string result; - }; -}} - -FC_REFLECT(sysio::detail::producer_api_plugin_response, (result)); - namespace sysio { using namespace sysio; -#define CALL_WITH_400(api_name, category, api_handle, call_name, INVOKE, http_response_code) \ -{std::string("/v1/" #api_name "/" #call_name), \ - api_category::category, \ - [&producer](string&&, string&& body, url_response_callback&& cb) mutable { \ - try { \ - INVOKE \ - cb(http_response_code, fc::variant(result)); \ - } catch (...) { \ - http_plugin::handle_exception(#api_name, #call_name, body, cb); \ - } \ - }} - -#define CALL_ASYNC(api_name, category, api_handle, call_name, call_result, INVOKE, http_response_code) \ -{std::string("/v1/" #api_name "/" #call_name), \ - api_category::category, \ - [&api_handle](string&&, string&& body, url_response_callback&& cb) mutable { \ - if (body.empty()) body = "{}"; \ - auto next = [cb=std::move(cb), body=std::move(body)](const chain::next_function_variant& result){ \ - if (std::holds_alternative(result)) {\ - try {\ - std::get(result)->rethrow();\ - } catch (...) {\ - http_plugin::handle_exception(#api_name, #call_name, body, cb);\ - }\ - } else if (std::holds_alternative(result)) { \ - cb(http_response_code, fc::variant(std::get(result)));\ - } else { \ - assert(0); \ - } \ - };\ - INVOKE\ - }\ -} - -#define INVOKE_R_R(api_handle, call_name, in_param) \ - auto params = parse_params(body);\ - auto result = api_handle.call_name(std::move(params)); - -#define INVOKE_R_R_II(api_handle, call_name, in_param) \ - auto params = parse_params(body);\ - auto result = api_handle.call_name(std::move(params)); - -#define INVOKE_R_R_D(api_handle, call_name, in_param) \ - auto& http = app().get_plugin(); \ - const fc::microseconds http_max_response_time = http.get_max_response_time(); \ - auto deadline = http_max_response_time == fc::microseconds::maximum() ? fc::time_point::maximum() \ - : fc::time_point::now() + http_max_response_time; \ - auto params = parse_params(body);\ - auto result = api_handle.call_name(std::move(params), deadline); - -#define INVOKE_R_V(api_handle, call_name) \ - body = parse_params(body); \ - auto result = api_handle.call_name(); - -#define INVOKE_R_V_ASYNC(api_handle, call_name)\ - api_handle.call_name(next); - -#define INVOKE_V_R(api_handle, call_name, in_param) \ - auto params = parse_params(body);\ - api_handle.call_name(std::move(params)); \ - sysio::detail::producer_api_plugin_response result{"ok"}; - -#define INVOKE_V_V(api_handle, call_name) \ - body = parse_params(body); \ - api_handle.call_name(); \ - sysio::detail::producer_api_plugin_response result{"ok"}; - - void producer_api_plugin::plugin_startup() { dlog("starting producer_api_plugin"); // lifetime of plugin is lifetime of application auto& producer = app().get_plugin(); - - app().get_plugin().add_api({ - CALL_WITH_400(producer, producer_ro, producer, paused, - INVOKE_R_V(producer, paused), 201), - CALL_WITH_400(producer, producer_ro, producer, get_runtime_options, - INVOKE_R_V(producer, get_runtime_options), 201), - CALL_WITH_400(producer, producer_ro, producer, get_greylist, - INVOKE_R_V(producer, get_greylist), 201), - CALL_WITH_400(producer, producer_ro, producer, get_whitelist_blacklist, - INVOKE_R_V(producer, get_whitelist_blacklist), 201), - CALL_WITH_400(producer, producer_ro, producer, get_scheduled_protocol_feature_activations, - INVOKE_R_V(producer, get_scheduled_protocol_feature_activations), 201), - CALL_WITH_400(producer, producer_ro, producer, get_supported_protocol_features, - INVOKE_R_R_II(producer, get_supported_protocol_features, - producer_plugin::get_supported_protocol_features_params), 201), - CALL_WITH_400(producer, producer_ro, producer, get_unapplied_transactions, - INVOKE_R_R_D(producer, get_unapplied_transactions, producer_plugin::get_unapplied_transactions_params), 200), - CALL_WITH_400(producer, producer_ro, producer, get_snapshot_requests, - INVOKE_R_V(producer, get_snapshot_requests), 201), + auto& _http_plugin = app().get_plugin(); + + using pp = producer_plugin; + using cat = api_category; + using pt = http_params_types; + + _http_plugin.add_api_stream({ + bind_stream<&pp::paused, dispatch::sync>( + _http_plugin, std::ref(producer), "/v1/producer/paused", cat::producer_ro, pt::no_params, 201), + bind_stream<&pp::get_runtime_options, dispatch::sync>( + _http_plugin, std::ref(producer), "/v1/producer/get_runtime_options", cat::producer_ro, pt::no_params, 201), + bind_stream<&pp::get_greylist, dispatch::sync>( + _http_plugin, std::ref(producer), "/v1/producer/get_greylist", cat::producer_ro, pt::no_params, 201), + bind_stream<&pp::get_whitelist_blacklist, dispatch::sync>( + _http_plugin, std::ref(producer), "/v1/producer/get_whitelist_blacklist", cat::producer_ro, pt::no_params, 201), + bind_stream<&pp::get_scheduled_protocol_feature_activations, dispatch::sync>( + _http_plugin, std::ref(producer), "/v1/producer/get_scheduled_protocol_feature_activations", + cat::producer_ro, pt::no_params, 201), + bind_stream<&pp::get_supported_protocol_features, dispatch::sync>( + _http_plugin, std::ref(producer), "/v1/producer/get_supported_protocol_features", + cat::producer_ro, pt::possible_no_params, 201), + bind_stream<&pp::get_unapplied_transactions, dispatch::sync>( + _http_plugin, std::ref(producer), "/v1/producer/get_unapplied_transactions", + cat::producer_ro, pt::possible_no_params, 200), + bind_stream<&pp::get_snapshot_requests, dispatch::sync>( + _http_plugin, std::ref(producer), "/v1/producer/get_snapshot_requests", + cat::producer_ro, pt::no_params, 201), }, appbase::exec_queue::read_only, appbase::priority::medium_high); // Not safe to run in parallel - app().get_plugin().add_api({ - CALL_WITH_400(producer, producer_rw, producer, pause, - INVOKE_V_V(producer, pause), 201), - CALL_WITH_400(producer, producer_rw, producer, pause_at_block, - INVOKE_V_R(producer, pause_at_block, producer_plugin::pause_at_block_params), 201), - CALL_WITH_400(producer, producer_rw, producer, resume, - INVOKE_V_V(producer, resume), 201), - CALL_WITH_400(producer, producer_rw, producer, update_runtime_options, - INVOKE_V_R(producer, update_runtime_options, producer_plugin::runtime_options), 201), - CALL_WITH_400(producer, producer_rw, producer, add_greylist_accounts, - INVOKE_V_R(producer, add_greylist_accounts, producer_plugin::greylist_params), 201), - CALL_WITH_400(producer, producer_rw, producer, remove_greylist_accounts, - INVOKE_V_R(producer, remove_greylist_accounts, producer_plugin::greylist_params), 201), - CALL_WITH_400(producer, producer_rw, producer, set_whitelist_blacklist, - INVOKE_V_R(producer, set_whitelist_blacklist, producer_plugin::whitelist_blacklist), 201), - CALL_ASYNC(producer, snapshot, producer, create_snapshot, chain::snapshot_scheduler::snapshot_information, - INVOKE_R_V_ASYNC(producer, create_snapshot), 201), - CALL_WITH_400(producer, snapshot, producer, schedule_snapshot, - INVOKE_R_R_II(producer, schedule_snapshot, chain::snapshot_scheduler::snapshot_request_params), 201), - CALL_WITH_400(producer, snapshot, producer, unschedule_snapshot, - INVOKE_R_R(producer, unschedule_snapshot, chain::snapshot_scheduler::snapshot_request_id_information), 201), - CALL_WITH_400(producer, producer_rw, producer, get_integrity_hash, - INVOKE_R_V(producer, get_integrity_hash), 201), - CALL_WITH_400(producer, producer_rw, producer, schedule_protocol_feature_activations, - INVOKE_V_R(producer, schedule_protocol_feature_activations, producer_plugin::scheduled_protocol_feature_activations), 201), + _http_plugin.add_api_stream({ + bind_stream<&pp::pause, dispatch::sync_void>( + _http_plugin, std::ref(producer), "/v1/producer/pause", cat::producer_rw, pt::no_params, 201), + bind_stream<&pp::pause_at_block, dispatch::sync_void>( + _http_plugin, std::ref(producer), "/v1/producer/pause_at_block", cat::producer_rw, pt::params_required, 201), + bind_stream<&pp::resume, dispatch::sync_void>( + _http_plugin, std::ref(producer), "/v1/producer/resume", cat::producer_rw, pt::no_params, 201), + bind_stream<&pp::update_runtime_options, dispatch::sync_void>( + _http_plugin, std::ref(producer), "/v1/producer/update_runtime_options", + cat::producer_rw, pt::params_required, 201), + bind_stream<&pp::add_greylist_accounts, dispatch::sync_void>( + _http_plugin, std::ref(producer), "/v1/producer/add_greylist_accounts", + cat::producer_rw, pt::params_required, 201), + bind_stream<&pp::remove_greylist_accounts, dispatch::sync_void>( + _http_plugin, std::ref(producer), "/v1/producer/remove_greylist_accounts", + cat::producer_rw, pt::params_required, 201), + bind_stream<&pp::set_whitelist_blacklist, dispatch::sync_void>( + _http_plugin, std::ref(producer), "/v1/producer/set_whitelist_blacklist", + cat::producer_rw, pt::params_required, 201), + bind_stream<&pp::create_snapshot, dispatch::async>( + _http_plugin, std::ref(producer), "/v1/producer/create_snapshot", cat::snapshot, pt::no_params, 201), + bind_stream<&pp::schedule_snapshot, dispatch::sync>( + _http_plugin, std::ref(producer), "/v1/producer/schedule_snapshot", + cat::snapshot, pt::possible_no_params, 201), + bind_stream<&pp::unschedule_snapshot, dispatch::sync>( + _http_plugin, std::ref(producer), "/v1/producer/unschedule_snapshot", + cat::snapshot, pt::params_required, 201), + bind_stream<&pp::get_integrity_hash, dispatch::sync>( + _http_plugin, std::ref(producer), "/v1/producer/get_integrity_hash", + cat::producer_rw, pt::no_params, 201), + bind_stream<&pp::schedule_protocol_feature_activations, dispatch::sync_void>( + _http_plugin, std::ref(producer), "/v1/producer/schedule_protocol_feature_activations", + cat::producer_rw, pt::params_required, 201), }, appbase::exec_queue::read_write, appbase::priority::medium_high); } @@ -168,11 +109,4 @@ void producer_api_plugin::plugin_initialize(const variables_map& options) { } FC_LOG_AND_RETHROW() } - -#undef INVOKE_R_R -#undef INVOKE_R_V -#undef INVOKE_V_R -#undef INVOKE_V_V -#undef CALL - } diff --git a/plugins/trace_api_plugin/include/sysio/trace_api/abi_data_handler.hpp b/plugins/trace_api_plugin/include/sysio/trace_api/abi_data_handler.hpp index 3a8d43b337..f0a7af1f26 100644 --- a/plugins/trace_api_plugin/include/sysio/trace_api/abi_data_handler.hpp +++ b/plugins/trace_api_plugin/include/sysio/trace_api/abi_data_handler.hpp @@ -13,6 +13,8 @@ #include #include +namespace fc { class json_writer; } + namespace sysio { namespace chain { struct abi_serializer; @@ -97,6 +99,24 @@ namespace sysio { */ std::tuple> serialize_to_variant(const std::variant& action); + /** + * Streaming counterpart of serialize_to_variant: emits the ABI-decoded "params" and (when applicable) + * "return_data" key/value pairs directly into `w` via fc::json_writer, without ever materialising an + * fc::variant tree. Pre-conditions: `w` is positioned inside the action's enclosing JSON object such that + * additional `key(...) value(...)` pairs are valid (i.e. caller has already written global_sequence, receiver, + * etc.). No-op if the action's account has no registered ABI or the action is not in the ABI. + * + * Fields decode independently: a parse failure on one is logged via `except_handler` and rolled back via + * `w.rewind()` to its own per-field checkpoint, but does NOT prevent the other from being attempted. The + * lone short-circuit case is `chain::abi_recursion_depth_exception` thrown during the params decode -- the + * ABI is structurally bad, so retrying the other field with the same ABI cannot succeed in a useful way + * and the function returns early with neither field emitted. serialize_to_variant uses the same rule. + * + * @param action - trace of the action including metadata necessary for finding the ABI + * @param w - writer to emit into; on return, zero, one, or both of "params" / "return_data" have been added + */ + void serialize_to_json_stream(const std::variant& action, fc::json_writer& w); + /** * Utility class that allows multiple request_handlers to share the same abi_data_handler */ @@ -114,6 +134,10 @@ namespace sysio { return handler->serialize_to_variant(action); } + void serialize_to_json_stream( const std::variant& action, fc::json_writer& w ) { + handler->serialize_to_json_stream(action, w); + } + std::shared_ptr handler; }; diff --git a/plugins/trace_api_plugin/include/sysio/trace_api/request_handler.hpp b/plugins/trace_api_plugin/include/sysio/trace_api/request_handler.hpp index e5a8132745..b2ee3b3f13 100644 --- a/plugins/trace_api_plugin/include/sysio/trace_api/request_handler.hpp +++ b/plugins/trace_api_plugin/include/sysio/trace_api/request_handler.hpp @@ -1,11 +1,13 @@ #pragma once #include +#include #include #include #include #include #include +#include #include #include #include @@ -16,10 +18,43 @@ namespace sysio::trace_api { using data_handler_function = std::function>( const std::variant& action)>; + // Streaming counterpart of data_handler_function: instead of returning a variant tuple that the caller then + // re-serializes to JSON, the callback emits the action's "params" and (when applicable) "return_data" + // key/value pairs directly into the writer. Decoded bytes flow straight from abi_serializer through + // binary_to_json_stream into the output buffer with no intermediate fc::variant allocation. + using stream_data_handler_function = std::function& action, fc::json_writer& w)>; + namespace detail { class response_formatter { public: static fc::variant process_block( const data_log_entry& trace, bool irreversible, const data_handler_function& data_handler ); + + // Streaming counterpart: emit the same JSON payload as process_block directly into the + // caller-provided fc::json_writer, skipping the fc::mutable_variant_object tree entirely. + // Output is byte-for-byte identical to fc::json::to_string(process_block(...)). The HTTP + // path passes the guarded writer the streaming callback provides, so the memory budget + // observes the response while it serializes. + static void process_block_to_stream( const data_log_entry& trace, bool irreversible, const stream_data_handler_function& data_handler, fc::json_writer& w ); + + // Buffer-building wrapper over process_block_to_stream (unguarded writer into a local + // string). Used by tests and the trace_api_json benchmark. + static std::string process_block_to_json( const data_log_entry& trace, bool irreversible, const stream_data_handler_function& data_handler ); + + // True iff the block trace contains a transaction with the given id. Lets the HTTP layer + // resolve its 404 before committing to a 200 emitter -- a streaming response cannot signal + // "not found" after emission has begun. + static bool contains_transaction( const data_log_entry& trace, const chain::transaction_id_type& trxid ); + + // Streaming counterpart for a single transaction inside a block trace. Walks the block's + // transaction list and, for the first entry whose id matches trxid, emits just that + // transaction's JSON object (no surrounding block wrapper) into the caller-provided writer. + // Returns false (and emits nothing) if no transaction in the block matches. Output for a + // matching trx is byte-for-byte identical to fc::json::to_string(get_transaction_trace(...)). + static bool process_transaction_to_stream( const data_log_entry& trace, const chain::transaction_id_type& trxid, const stream_data_handler_function& data_handler, fc::json_writer& w ); + + // Buffer-building wrapper over process_transaction_to_stream; returns an empty string on a + // miss. Used by tests. + static std::string process_transaction_to_json( const data_log_entry& trace, const chain::transaction_id_type& trxid, const stream_data_handler_function& data_handler ); }; } @@ -195,6 +230,50 @@ namespace sysio::trace_api { return detail::response_formatter::process_block(std::get<0>(*data), std::get<1>(*data), data_handler); } + /** + * Streaming variant of get_block_trace for the HTTP streaming-callback path. + * + * Resolves block existence up front so the caller can produce its 404 before + * committing to a 200 emitter; the returned closure (owning the block's decoded + * trace data) emits the response body into the caller-provided fc::json_writer. + * The HTTP layer passes its guarded writer, so the memory budget observes -- and + * can abort -- the response while it serializes instead of after it is allocated. + * + * @param block_height - the height of the block whose trace is requested + * @return an emitter for the block's trace, or std::nullopt if the block does not exist + * @throws bad_data_exception when there are issues with the underlying data preventing processing. + */ + std::optional> get_block_trace_emitter( uint32_t block_height ) { + auto data = logfile_provider.get_block(block_height); + if (!data) { + _log("No block found at block height " + std::to_string(block_height) ); + return std::nullopt; + } + + return [this, data = std::move(*data)](fc::json_writer& w) { + detail::response_formatter::process_block_to_stream(std::get<0>(data), std::get<1>(data), + make_stream_data_handler(), w); + }; + } + + /** + * Buffer-building wrapper over get_block_trace_emitter: emits into a local string via + * an unguarded writer. Returned string is empty if the block does not exist. Kept + * for direct (non-HTTP) callers and tests; the HTTP path uses the emitter form so the + * guarded writer's budget applies. + */ + std::string get_block_trace_json( uint32_t block_height ) { + auto emit = get_block_trace_emitter(block_height); + if (!emit) + return {}; + std::string out; + { + fc::json_writer w(out); + (*emit)(w); + } + return out; + } + /** * Fetch the trace for a given transaction id and convert it to a fc::variant for conversion to a final format * (eg JSON) @@ -263,6 +342,57 @@ namespace sysio::trace_api { return result; } + /** + * Streaming variant of get_transaction_trace for the HTTP streaming-callback path. + * + * Resolves both misses up front -- block absent, or trxid not present in the block + * (via response_formatter::contains_transaction) -- so the caller can produce its + * 404 before committing to a 200 emitter; the returned closure emits just the + * matching transaction's JSON into the caller-provided (guarded) fc::json_writer. + * + * @param trxid - the transaction id whose trace is requested + * @param block_height - the height of the block whose trace contains the transaction + * @return an emitter for the transaction's trace, or std::nullopt on either miss + * @throws bad_data_exception when there are issues with the underlying data preventing processing. + */ + std::optional> get_transaction_trace_emitter( const chain::transaction_id_type& trxid, uint32_t block_height ) { + _log("get_transaction_trace_emitter called"); + auto data = logfile_provider.get_block(block_height); + if (!data) { + _log("No block found at block height " + std::to_string(block_height)); + return std::nullopt; + } + if (!detail::response_formatter::contains_transaction(std::get<0>(*data), trxid)) { + _log("No transaction with id " + trxid.str() + " found in block " + std::to_string(block_height)); + return std::nullopt; + } + + return [this, data = std::move(*data), trxid](fc::json_writer& w) { + [[maybe_unused]] const bool found = detail::response_formatter::process_transaction_to_stream( + std::get<0>(data), trxid, make_stream_data_handler(), w); + assert(found); // contains_transaction gated emitter creation + }; + } + + /** + * Buffer-building wrapper over get_transaction_trace_emitter: emits into a local + * string via an unguarded writer. Returned string is empty if the block does not + * exist or the trxid is not present in the block. Kept for direct (non-HTTP) + * callers and tests; the HTTP path uses the emitter form so the guarded writer's + * budget applies. + */ + std::string get_transaction_trace_json( const chain::transaction_id_type& trxid, uint32_t block_height ) { + auto emit = get_transaction_trace_emitter(trxid, block_height); + if (!emit) + return {}; + std::string out; + { + fc::json_writer w(out); + (*emit)(w); + } + return out; + } + /** * Scan a block range for action traces matching the given filter. * @@ -288,6 +418,17 @@ namespace sysio::trace_api { } private: + /// Adapter from the provider's per-action streaming serializer to the + /// stream_data_handler_function shape the response_formatter consumes. Shared by + /// both emitter factories; binds `this`, so emitters must not outlive the handler. + stream_data_handler_function make_stream_data_handler() { + return [this](const std::variant& action, fc::json_writer& w) { + std::visit([&](const auto& a) { + data_handler_provider.serialize_to_json_stream(a, w); + }, action); + }; + } + actions_result get_actions_impl(const action_query& query, variant_shape shape) { actions_result result; diff --git a/plugins/trace_api_plugin/src/abi_data_handler.cpp b/plugins/trace_api_plugin/src/abi_data_handler.cpp index 9ab3025993..f420edb39c 100644 --- a/plugins/trace_api_plugin/src/abi_data_handler.cpp +++ b/plugins/trace_api_plugin/src/abi_data_handler.cpp @@ -1,9 +1,22 @@ #include #include +#include #include namespace sysio::trace_api { + namespace { + // ABIs are user-provided; the yield function only enforces the recursion-depth cap and + // intentionally has no wall-clock deadline. Shared between the variant and streaming paths + // so they stay in lock-step on what counts as "too deep". + auto make_abi_yield() { + return [](size_t recursion_depth) { + SYS_ASSERT( recursion_depth < chain::abi_serializer::max_recursion_depth, chain::abi_recursion_depth_exception, + "exceeded max_recursion_depth {}", chain::abi_serializer::max_recursion_depth ); + }; + } + } + std::shared_ptr abi_data_handler::get_serializer(chain::name account, uint64_t action_global_seq) { if (!_abi_seq_resolver) return nullptr; @@ -73,12 +86,7 @@ namespace sysio::trace_api { auto type_name = serializer->get_action_type(a.action); if (type_name.empty()) return result; - // abis are user provided, do not use a deadline - auto abi_yield = [](size_t recursion_depth) { - SYS_ASSERT( recursion_depth < chain::abi_serializer::max_recursion_depth, - chain::abi_recursion_depth_exception, - "exceeded max_recursion_depth {}", chain::abi_serializer::max_recursion_depth ); - }; + auto abi_yield = make_abi_yield(); // Separate try blocks so that a failure decoding return_value does not // discard successfully-decoded params (and vice versa). @@ -128,4 +136,43 @@ namespace sysio::trace_api { return {}; }, action); } + + void abi_data_handler::serialize_to_json_stream(const std::variant& action, fc::json_writer& w) { + std::visit([&](const auto& a) { + auto serializer = get_serializer(a.account, a.global_sequence); + if (!serializer) return; + auto type_name = serializer->get_action_type(a.action); + if (type_name.empty()) return; + + auto abi_yield = make_abi_yield(); + + // Kept in lock-step with decode() + serialize_to_variant: the get_block / + // get_transaction_trace pipeline emits params/return_data all-or-nothing. Any decode + // failure rewinds every byte this function wrote so the streamed shape matches the + // variant path's legacy empty shape (no params, no return_data). + auto cp = w.checkpoint(); + try { + w.key("params"); + serializer->binary_to_json_stream(type_name, a.data, w, abi_yield); + } catch (...) { + w.rewind(cp); + except_handler(MAKE_EXCEPTION_WITH_CONTEXT(std::current_exception())); + return; + } + + if (a.return_value.empty()) return; + auto return_type_name = serializer->get_action_result_type(a.action); + if (return_type_name.empty()) return; + + try { + w.key("return_data"); + serializer->binary_to_json_stream(return_type_name, a.return_value, w, abi_yield); + } catch (...) { + // Roll back params as well: the variant path returns the legacy empty shape when + // return_value fails to decode, and the two paths must emit identical JSON. + w.rewind(cp); + except_handler(MAKE_EXCEPTION_WITH_CONTEXT(std::current_exception())); + } + }, action); + } } diff --git a/plugins/trace_api_plugin/src/request_handler.cpp b/plugins/trace_api_plugin/src/request_handler.cpp index f28afa3595..6bafad0aac 100644 --- a/plugins/trace_api_plugin/src/request_handler.cpp +++ b/plugins/trace_api_plugin/src/request_handler.cpp @@ -1,7 +1,12 @@ #include #include +#include +#include +#include +#include +#include #include namespace sysio::trace_api { @@ -126,3 +131,177 @@ namespace sysio::trace_api::detail { }, trace); } } + +// ============================================================================ +// Streaming counterpart: write JSON directly to std::string via fc::json_writer +// ---------------------------------------------------------------------------- +// Emits byte-identical JSON to process_block + fc::json::to_string, without +// constructing any fc::mutable_variant_object / fc::variant nodes on the +// response path. Profiling showed the variant-tree build was the dominant +// allocator on trace_api HTTP queries (~41% of per-test allocations on a +// validator serving get_block). This path skips that tree entirely. +// ============================================================================ +namespace { + using namespace sysio::trace_api; + + void write_authorizations(fc::json_writer& w, + const std::vector& auths) { + w.begin_array(); + for (const auto& a : auths) { + w.begin_object(); + w.set("actor", a.actor.to_string()) + .set("permission", a.permission.to_string()); + w.end_object(); + } + w.end_array(); + } + + void write_actions(fc::json_writer& w, + const std::vector& actions, + const stream_data_handler_function& data_handler) { + // Iteration order matches process_actions: sort indices by global_sequence so + // clients see execution order (notifications run before the inline actions that + // queued them, so action-slot order is not global_sequence order). + std::vector indices(actions.size()); + std::iota(indices.begin(), indices.end(), 0); + std::sort(indices.begin(), indices.end(), [&actions](int l, int r) { + return actions.at(l).global_sequence < actions.at(r).global_sequence; + }); + + w.begin_array(); + for (int idx : indices) { + const auto& a = actions.at(idx); + w.begin_object(); + // Field set and order mirror build_action_variant's variant_shape::full shape -- + // the streaming and variant paths must emit identical JSON (pinned by the + // streaming_vs_variant_* parity tests). + w.set("global_sequence", a.global_sequence) + .set("receiver", a.receiver.to_string()) + .set("account", a.account.to_string()) + .set("name", a.action.to_string()); + w.key("authorization"); write_authorizations(w, a.authorization); + w.key("data"); w.value_hex(a.data.data(), a.data.size()); + w.key("return_value"); w.value_hex(a.return_value.data(), a.return_value.size()); + w.set("action_ordinal", a.action_ordinal.value) + .set("creator_action_ordinal", a.creator_action_ordinal.value) + .set("closest_unnotified_ancestor_action_ordinal", a.closest_unnotified_ancestor_action_ordinal.value) + .set("recv_sequence", a.recv_sequence); + // fc/container/flat.hpp co-locates to_json_stream(flat_map) with its to_variant + // sibling, so the streaming and variant paths emit the same map shape by design. + w.key("auth_sequence"); + fc::to_json_stream(a.auth_sequence, w); + w.set("code_sequence", a.code_sequence.value) + .set("abi_sequence", a.abi_sequence.value); + w.key("account_ram_deltas"); + w.begin_array(); + for (const auto& d : a.account_ram_deltas) { + w.begin_object(); + w.set("account", d.account.to_string()) + .set("delta", d.delta); + w.end_object(); + } + w.end_array(); + if (a.cpu_usage_us.has_value()) + w.set("cpu_usage_us", a.cpu_usage_us->value); + if (a.net_usage.has_value()) + w.set("net_usage", a.net_usage->value); + // ABI-decoded "params" / "return_data" are emitted directly into w by the streaming + // data_handler -- no fc::variant tree, no fc::json::to_string splice. The handler + // is responsible for emitting zero, one, or both keys depending on what's available + // and whether the decode succeeds; on failure it rolls back via json_writer::rewind + // so the action object's preceding fields stay intact. + data_handler(a, w); + w.end_object(); + } + w.end_array(); + } + + void write_transaction(fc::json_writer& w, + const transaction_trace_v0& t, + const stream_data_handler_function& data_handler) { + w.begin_object(); + w.set("id", t.id) + .set("block_num", t.block_num) + // FC_SERIALIZE_AS_STRING-reflected; emits the ISO date string. + .set("block_time", t.block_time) + .set("producer_block_id", t.producer_block_id); + w.key("actions"); write_actions(w, t.actions, data_handler); + w.set("cpu_usage_us", t.cpu_usage_us) + .set("net_usage_words", t.net_usage_words.value) + .set("signatures", t.signatures); + // transaction_header is a reflected struct composed of fc::time_point_sec, + // uint16/uint32 and unsigned_ints. No native to_json_stream path yet so + // fall back via the variant bridge for just that field. + w.key("transaction_header"); + fc::to_json_stream_via_variant(t.trx_header, w); + w.end_object(); + } + + void write_transactions(fc::json_writer& w, + const std::vector& transactions, + const stream_data_handler_function& data_handler) { + w.begin_array(); + for (const auto& t : transactions) { + write_transaction(w, t, data_handler); + } + w.end_array(); + } +} + +namespace sysio::trace_api::detail { + void response_formatter::process_block_to_stream( const data_log_entry& trace, bool irreversible, const stream_data_handler_function& data_handler, fc::json_writer& w ) { + std::visit([&](auto&& bt) { + w.begin_object(); + w.set("id", bt.id) + .set("number", bt.number) + .set("previous_id", bt.previous_id) + .set("status", irreversible ? "irreversible" : "pending") + // to_iso8601_datetime appends a 'Z' suffix that process_block also emits. + .set("timestamp", to_iso8601_datetime(bt.timestamp)) + .set("producer", bt.producer.to_string()) + .set("transaction_mroot", bt.transaction_mroot) + .set("finality_mroot", bt.finality_mroot); + w.key("transactions"); write_transactions(w, bt.transactions, data_handler); + w.end_object(); + }, trace); + } + + std::string response_formatter::process_block_to_json( const data_log_entry& trace, bool irreversible, const stream_data_handler_function& data_handler ) { + std::string out; + fc::json_writer w(out); + process_block_to_stream(trace, irreversible, data_handler, w); + return out; + } + + bool response_formatter::contains_transaction( const data_log_entry& trace, const chain::transaction_id_type& trxid ) { + return std::visit([&](auto&& bt) { + for (const auto& t : bt.transactions) { + if (t.id == trxid) + return true; + } + return false; + }, trace); + } + + bool response_formatter::process_transaction_to_stream( const data_log_entry& trace, const chain::transaction_id_type& trxid, const stream_data_handler_function& data_handler, fc::json_writer& w ) { + return std::visit([&](auto&& bt) { + for (const auto& t : bt.transactions) { + if (t.id == trxid) { + write_transaction(w, t, data_handler); + return true; + } + } + return false; + }, trace); + } + + std::string response_formatter::process_transaction_to_json( const data_log_entry& trace, const chain::transaction_id_type& trxid, const stream_data_handler_function& data_handler ) { + std::string out; + { + fc::json_writer w(out); + if (!process_transaction_to_stream(trace, trxid, data_handler, w)) + return {}; // miss: preserve the empty-string contract (out holds only the writer's reserve) + } + return out; + } +} diff --git a/plugins/trace_api_plugin/src/trace_api_plugin.cpp b/plugins/trace_api_plugin/src/trace_api_plugin.cpp index 0ee4e3d5f0..154a58363c 100644 --- a/plugins/trace_api_plugin/src/trace_api_plugin.cpp +++ b/plugins/trace_api_plugin/src/trace_api_plugin.cpp @@ -10,6 +10,8 @@ #include +#include + #include #include @@ -142,6 +144,16 @@ namespace { std::shared_ptr store; }; + + /// Build a stream_emitter for an error_results payload -- the streaming twin of the + /// old `cb(code, fc::variant(error_results{...}))` error responses. Emits through the + /// same reflector path http_plugin::handle_exception_stream uses, so the error body + /// shape matches the variant path byte-for-byte. + sysio::stream_emitter make_error_results_emitter(sysio::error_results results) { + return [results = std::move(results)](fc::json_writer& w) mutable { + fc::to_json_stream(results, w); + }; + } } namespace sysio { @@ -283,9 +295,15 @@ struct trace_api_rpc_plugin_impl : public std::enable_shared_from_this(); - http.add_async_handler({"/v1/trace_api/get_block", + // Streaming-cb registration: the emitter writes the response body directly into the + // guarded fc::json_writer the http layer provides, so --http-max-bytes-in-flight-mb + // observes -- and can abort -- large trace responses WHILE they serialize, instead + // of only after the full body has been allocated (the old json_raw pass-through). + // Misses resolve to their 404 before an emitter is committed; error responses emit + // tiny error_results objects through the same streaming path. + http.add_async_api_stream({{std::string("/v1/trace_api/get_block"), api_category::trace_api, - [self = shared_from_this()](std::string, std::string body, url_response_callback cb) + [self = shared_from_this()](std::string&&, std::string&& body, url_response_stream_callback&& cb) { auto block_number = ([&body]() -> std::optional { if (body.empty()) { @@ -305,29 +323,28 @@ struct trace_api_rpc_plugin_impl : public std::enable_shared_from_thisreq_handler->get_block_trace(*block_number); - if (resp.is_null()) { - error_results results{404, "Trace API: block trace missing"}; - cb( 404, fc::variant( results )); + auto emitter = self->req_handler->get_block_trace_emitter(*block_number); + if (!emitter) { + cb( 404, make_error_results_emitter({404, "Trace API: block trace missing"}) ); } else { - cb( 200, std::move(resp) ); + // self keeps req_handler (bound as `this` inside the emitter) alive until + // emission completes on the http thread pool. + cb( 200, [self, emit = std::move(*emitter)](fc::json_writer& w) { emit(w); } ); } } catch (...) { - http_plugin::handle_exception("trace_api", "get_block", body, cb); + http_plugin::handle_exception_stream("trace_api", "get_block", body, cb); } - }}); + }}}); - http.add_async_handler({"/v1/trace_api/get_transaction_trace", + http.add_async_api_stream({{std::string("/v1/trace_api/get_transaction_trace"), api_category::trace_api, - [self = shared_from_this()](std::string, std::string body, url_response_callback cb) + [self = shared_from_this()](std::string&&, std::string&& body, url_response_stream_callback&& cb) { auto trx_id = ([&body]() -> std::optional { if (body.empty()) { @@ -346,8 +363,7 @@ struct trace_api_rpc_plugin_impl : public std::enable_shared_from_thiscommon->store->get_trx_block_number(*trx_id); if (!blk_num.has_value()){ - error_results results{404, "Trace API: transaction id missing in the transaction id log files"}; - cb( 404, fc::variant( results )); + cb( 404, make_error_results_emitter({404, "Trace API: transaction id missing in the transaction id log files"}) ); } else { - auto resp = self->req_handler->get_transaction_trace(*trx_id, *blk_num); - if (resp.is_null()) { - error_results results{404, "Trace API: transaction trace missing"}; - cb( 404, fc::variant( results )); + auto emitter = self->req_handler->get_transaction_trace_emitter(*trx_id, *blk_num); + if (!emitter) { + cb( 404, make_error_results_emitter({404, "Trace API: transaction trace missing"}) ); } else { - cb( 200, std::move(resp) ); + // self keeps req_handler (bound as `this` inside the emitter) alive until + // emission completes on the http thread pool, mirroring get_block. + cb( 200, [self, emit = std::move(*emitter)](fc::json_writer& w) { emit(w); } ); } } } catch (...) { - http_plugin::handle_exception("trace_api", "get_transaction", body, cb); + http_plugin::handle_exception_stream("trace_api", "get_transaction", body, cb); } - }}); + }}}); http.add_async_handler({"/v1/trace_api/get_actions", api_category::trace_api, diff --git a/plugins/trace_api_plugin/test/test_data_handlers.cpp b/plugins/trace_api_plugin/test/test_data_handlers.cpp index f462b2657a..9dfb3913c7 100644 --- a/plugins/trace_api_plugin/test/test_data_handlers.cpp +++ b/plugins/trace_api_plugin/test/test_data_handlers.cpp @@ -1,5 +1,8 @@ #include +#include +#include + #include #include @@ -281,5 +284,217 @@ BOOST_AUTO_TEST_SUITE(abi_data_handler_tests) BOOST_REQUIRE(!std::get<1>(actual)); } + namespace { + // Shared two-type ABI for the streaming/variant lock-step cases below: action type "foo" + // (four varuint32 fields) with action_result type "foor" (three varuint32 fields). + chain::abi_def make_foo_abi_with_result() { + auto abi = chain::abi_def ( {}, + { + { "foo", "", { {"a", "varuint32"}, {"b", "varuint32"}, {"c", "varuint32"}, {"d", "varuint32"} } }, + { "foor", "", { {"e", "varuint32"}, {"f", "varuint32"}, {"g", "varuint32"} } } + }, + { + { "foo"_n, "foo", ""} + }, + {}, {}, {} + ); + abi.version = "sysio::abi/1."; + abi.action_results = { std::vector{ chain::action_result_def{ "foo"_n, "foor"} } }; + return abi; + } + + action_trace_v0 make_foo_action(chain::bytes data, chain::bytes return_value) { + action_trace_v0 action; + action.global_sequence = 0; + action.receiver = "alice"_n; + action.account = "alice"_n; + action.action = "foo"_n; + action.data = std::move(data); + action.return_value = std::move(return_value); + return action; + } + } + + // Streaming serialize_to_json_stream must emit byte-equivalent JSON to the variant path + // when the same ABI is supplied. Drive the streamer into a wrapped object so the result + // can be parsed back as a variant and compared field-by-field. + BOOST_AUTO_TEST_CASE(streaming_basic_abi_with_return_value_parity) + { + std::variant action_trace_t = + make_foo_action({0x00, 0x01, 0x02, 0x03}, {0x04, 0x05, 0x06}); + + abi_data_handler handler(exception_handler{}, make_resolver("alice"_n), + make_fetcher("alice"_n, pack_abi(make_foo_abi_with_result()))); + + auto [params_v, return_v] = handler.serialize_to_variant(action_trace_t); + + std::string out; + { + fc::json_writer w(out); + w.begin_object(); + handler.serialize_to_json_stream(action_trace_t, w); + w.end_object(); + BOOST_REQUIRE(w.balanced()); + } + + fc::variant streamed = fc::json::from_string(out); + const auto& streamed_obj = streamed.get_object(); + BOOST_REQUIRE(streamed_obj.contains("params")); + BOOST_REQUIRE(streamed_obj.contains("return_data")); + + BOOST_TEST(to_kv(params_v) == to_kv(streamed_obj["params"]), boost::test_tools::per_element()); + BOOST_REQUIRE(return_v.has_value()); + BOOST_TEST(to_kv(*return_v) == to_kv(streamed_obj["return_data"]), boost::test_tools::per_element()); + } + + // No ABI resolvable: the streaming path must emit zero key/value pairs and leave the writer + // in a balanced, key/value-pair-empty state so the surrounding object closes cleanly. + BOOST_AUTO_TEST_CASE(streaming_no_abi_emits_nothing) + { + std::variant action_trace_t = + make_foo_action({0x00, 0x01, 0x02, 0x03}, {0x04, 0x05, 0x06}); + + abi_data_handler handler(exception_handler{}); + + std::string out; + { + fc::json_writer w(out); + w.begin_object(); + handler.serialize_to_json_stream(action_trace_t, w); + w.end_object(); + BOOST_REQUIRE(w.balanced()); + } + + BOOST_TEST(out == "{}"); + } + + // Variant-pipeline shape on params decode failure: decode() short-circuits with + // decode_status::failed, and serialize_to_variant maps anything but ok to the legacy empty + // tuple. The failure is still surfaced through except_handler. + BOOST_AUTO_TEST_CASE(variant_basic_abi_params_throws_yields_empty) + { + std::variant action_trace_t = + make_foo_action({0x00, 0x01, 0x02} /* truncated */, {0x04, 0x05, 0x06}); + + bool log_called = false; + abi_data_handler handler([&log_called](const exception_with_context&){ log_called = true; }, + make_resolver("alice"_n), + make_fetcher("alice"_n, pack_abi(make_foo_abi_with_result()))); + + auto actual = handler.serialize_to_variant(action_trace_t); + + BOOST_TEST(log_called); + BOOST_TEST(std::get<0>(actual).is_null()); + BOOST_REQUIRE(!std::get<1>(actual).has_value()); + } + + // Variant-pipeline shape on return_value decode failure: decode() keeps the decoded params in + // decode_result (the get_actions path emits them alongside decode_error), but the legacy + // tuple wrapper drops the whole action on any failure -- both fields come back empty. + BOOST_AUTO_TEST_CASE(variant_basic_abi_return_data_throws_yields_empty) + { + std::variant action_trace_t = + make_foo_action({0x00, 0x01, 0x02, 0x03}, {0x04, 0x05} /* truncated */); + + bool log_called = false; + abi_data_handler handler([&log_called](const exception_with_context&){ log_called = true; }, + make_resolver("alice"_n), + make_fetcher("alice"_n, pack_abi(make_foo_abi_with_result()))); + + auto actual = handler.serialize_to_variant(action_trace_t); + + BOOST_TEST(log_called); + BOOST_TEST(std::get<0>(actual).is_null()); + BOOST_REQUIRE(!std::get<1>(actual).has_value()); + } + + // Streaming lock-step with the variant pipeline on params decode failure: truncated params + // bytes throw after "params" is half-written; the handler rewinds every byte it wrote and + // logs, emitting nothing -- the same empty shape serialize_to_variant produces. + BOOST_AUTO_TEST_CASE(streaming_basic_abi_params_throws_emits_nothing) + { + std::variant action_trace_t = + make_foo_action({0x00, 0x01, 0x02} /* truncated */, {0x04, 0x05, 0x06}); + + bool log_called = false; + abi_data_handler handler([&log_called](const exception_with_context&){ log_called = true; }, + make_resolver("alice"_n), + make_fetcher("alice"_n, pack_abi(make_foo_abi_with_result()))); + + std::string out; + { + fc::json_writer w(out); + w.begin_object(); + handler.serialize_to_json_stream(action_trace_t, w); + w.end_object(); + BOOST_REQUIRE(w.balanced()); + } + + BOOST_TEST(log_called); + BOOST_TEST(out == "{}"); + } + + // Streaming lock-step with the variant pipeline on return_value decode failure: params were + // already emitted when the return_data decode throws, so the rewind must roll back params as + // well -- the variant path returns the legacy empty shape and the two paths must match. + BOOST_AUTO_TEST_CASE(streaming_basic_abi_return_data_throws_emits_nothing) + { + std::variant action_trace_t = + make_foo_action({0x00, 0x01, 0x02, 0x03}, {0x04, 0x05} /* truncated */); + + bool log_called = false; + abi_data_handler handler([&log_called](const exception_with_context&){ log_called = true; }, + make_resolver("alice"_n), + make_fetcher("alice"_n, pack_abi(make_foo_abi_with_result()))); + + std::string out; + { + fc::json_writer w(out); + w.begin_object(); + handler.serialize_to_json_stream(action_trace_t, w); + w.end_object(); + BOOST_REQUIRE(w.balanced()); + } + + BOOST_TEST(log_called); + BOOST_TEST(out == "{}"); + } + + // Truncated action data triggers an exception inside binary_to_json_stream after "params" has + // already been written. The handler must rewind, log via except_handler, and leave the writer + // observably empty (no half-written "params": fragment) so the surrounding object closes cleanly. + BOOST_AUTO_TEST_CASE(streaming_basic_abi_insufficient_data_rolls_back) + { + std::variant action_trace_t = + make_foo_action({0x00, 0x01, 0x02} /* truncated */, {}); + + auto abi = chain::abi_def ( {}, + { + { "foo", "", { {"a", "varuint32"}, {"b", "varuint32"}, {"c", "varuint32"}, {"d", "varuint32"} } } + }, + { + { "foo"_n, "foo", ""} + }, + {}, {}, {} + ); + abi.version = "sysio::abi/1."; + + bool log_called = false; + abi_data_handler handler([&log_called](const exception_with_context&){ log_called = true; }, + make_resolver("alice"_n), make_fetcher("alice"_n, pack_abi(abi))); + + std::string out; + { + fc::json_writer w(out); + w.begin_object(); + handler.serialize_to_json_stream(action_trace_t, w); + w.end_object(); + BOOST_REQUIRE(w.balanced()); + } + + BOOST_TEST(log_called); + BOOST_TEST(out == "{}"); + } + BOOST_AUTO_TEST_SUITE_END() diff --git a/plugins/trace_api_plugin/test/test_responses.cpp b/plugins/trace_api_plugin/test/test_responses.cpp index 20a37f51e1..d45164ff82 100644 --- a/plugins/trace_api_plugin/test/test_responses.cpp +++ b/plugins/trace_api_plugin/test/test_responses.cpp @@ -1,5 +1,7 @@ #include +#include +#include #include #include @@ -48,6 +50,22 @@ struct response_test_fixture { return fixture.mock_data_handler(action); } + // Streaming peer for the mock: delegates to the same mock_data_handler that the variant path uses, + // then walks the resulting variant via fc::to_json_stream so the streaming output matches the + // variant output byte-for-byte. Production uses abi_serializer::binary_to_json_stream end-to-end; + // here we don't have a real ABI, so we fake parity by re-using the variant tuple. + void serialize_to_json_stream(const action_trace_v0& action, fc::json_writer& w) { + auto [params, return_data] = fixture.mock_data_handler(action); + if (!params.is_null()) { + w.key("params"); + fc::to_json_stream(params, w); + } + if (return_data.has_value()) { + w.key("return_data"); + fc::to_json_stream(*return_data, w); + } + } + response_test_fixture& fixture; }; @@ -64,6 +82,26 @@ struct response_test_fixture { return response_impl.get_block_trace( block_height ); } + std::string get_block_trace_json( uint32_t block_height ) { + return response_impl.get_block_trace_json( block_height ); + } + + fc::variant get_transaction_trace( const chain::transaction_id_type& trxid, uint32_t block_height ) { + return response_impl.get_transaction_trace( trxid, block_height ); + } + + std::string get_transaction_trace_json( const chain::transaction_id_type& trxid, uint32_t block_height ) { + return response_impl.get_transaction_trace_json( trxid, block_height ); + } + + std::optional> get_block_trace_emitter( uint32_t block_height ) { + return response_impl.get_block_trace_emitter( block_height ); + } + + std::optional> get_transaction_trace_emitter( const chain::transaction_id_type& trxid, uint32_t block_height ) { + return response_impl.get_transaction_trace_emitter( trxid, block_height ); + } + // fixture data and methods std::function mock_get_block; std::function>(const action_trace_v0&)> mock_data_handler = default_mock_data_handler; @@ -524,4 +562,287 @@ BOOST_AUTO_TEST_SUITE(trace_responses) BOOST_TEST(null_response.is_null()); } -BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file + // The streaming JSON response (process_block_to_json) must produce the same + // observable shape as the variant response (process_block) for every field. + // Round-trip the streaming bytes through fc::json::from_string and compare + // key-by-key with the variant path. Catches silent shape regressions like + // status: "executed" -> 0 or block_time: {"slot":N} -> N. + BOOST_FIXTURE_TEST_CASE(streaming_vs_variant_block_response_parity, response_test_fixture) + { + action_trace_v0 action_trace{}; + action_trace.global_sequence = 0; + action_trace.receiver = "receiver"_n; + action_trace.account = "contract"_n; + action_trace.action = "action"_n; + action_trace.authorization = {{ "alice"_n, "active"_n }}; + action_trace.data = { 0x00, 0x01, 0x02, 0x03 }; + action_trace.return_value = { 0x04, 0x05, 0x06, 0x07 }; + + auto block_trace = block_trace_v0 { + "b000000000000000000000000000000000000000000000000000000000000001"_h, + 1, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + chain::block_timestamp_type(0), + "bp.one"_n, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + std::vector { + { + "0000000000000000000000000000000000000000000000000000000000000001"_h, + std::vector { action_trace }, + 10, + 5, + std::vector{ chain::signature_type() }, + { chain::time_point_sec(), 1, 0, 100, 50, 0 }, + 1, + chain::block_timestamp_type(0), + std::optional{} + } + } + }; + + mock_get_block = [&block_trace]( uint32_t height ) -> get_block_t { + return std::make_tuple(data_log_entry(block_trace), false); + }; + + fc::variant variant_response = get_block_trace( 1 ); + std::string streamed_response = get_block_trace_json( 1 ); + fc::variant streamed_as_variant = fc::json::from_string( streamed_response ); + + BOOST_TEST(to_kv(variant_response) == to_kv(streamed_as_variant), boost::test_tools::per_element()); + } + + // The streaming JSON response (process_transaction_to_json) must produce the same observable shape as the variant + // response (get_transaction_trace) for a matching trxid. Round-trip the streaming bytes through fc::json::from_string + // and compare key-by-key with the variant path. Catches silent shape regressions in the per-transaction emitter that + // the block-level parity test would not detect because they are masked by the surrounding block wrapper. + BOOST_FIXTURE_TEST_CASE(streaming_vs_variant_transaction_response_parity, response_test_fixture) + { + auto trx_id_a = "0000000000000000000000000000000000000000000000000000000000000001"_h; + auto trx_id_b = "0000000000000000000000000000000000000000000000000000000000000002"_h; + + action_trace_v0 action_a{}; + action_a.global_sequence = 0; + action_a.receiver = "receiver"_n; + action_a.account = "contract"_n; + action_a.action = "action"_n; + action_a.authorization = {{ "alice"_n, "active"_n }}; + action_a.data = { 0x00, 0x01, 0x02, 0x03 }; + action_a.return_value = { 0x04, 0x05, 0x06, 0x07 }; + + // Populate every optional / container field the full variant shape emits, so the + // parity assertion also covers the ordinal, auth_sequence, ram-delta and usage paths. + action_trace_v0 action_b{}; + action_b.action_ordinal = 2; + action_b.creator_action_ordinal = 1; + action_b.closest_unnotified_ancestor_action_ordinal = 1; + action_b.global_sequence = 1; + action_b.recv_sequence = 7; + action_b.auth_sequence = {{ "bob"_n, 3 }}; + action_b.code_sequence = 2; + action_b.abi_sequence = 4; + action_b.receiver = "receiver"_n; + action_b.account = "contract"_n; + action_b.action = "action"_n; + action_b.authorization = {{ "bob"_n, "active"_n }}; + action_b.data = { 0x10, 0x11 }; + action_b.account_ram_deltas = {{ "bob"_n, 240 }}; + action_b.cpu_usage_us = 11; + action_b.net_usage = 8; + + auto block_trace = block_trace_v0 { + "b000000000000000000000000000000000000000000000000000000000000001"_h, + 1, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + chain::block_timestamp_type(0), + "bp.one"_n, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + std::vector { + { + trx_id_a, + std::vector { action_a }, + 10, + 5, + std::vector{ chain::signature_type() }, + { chain::time_point_sec(), 1, 0, 100, 50, 0 }, + 1, + chain::block_timestamp_type(0), + std::optional{} + }, + { + trx_id_b, + std::vector { action_b }, + 20, + 6, + std::vector{ chain::signature_type() }, + { chain::time_point_sec(), 2, 0, 200, 60, 0 }, + 1, + chain::block_timestamp_type(0), + std::optional{} + } + } + }; + + mock_get_block = [&block_trace]( uint32_t height ) -> get_block_t { + return std::make_tuple(data_log_entry(block_trace), false); + }; + + // Hit case: trxid present in the block. Both paths must agree on every field. + { + fc::variant variant_response = get_transaction_trace( trx_id_b, 1 ); + std::string streamed_response = get_transaction_trace_json( trx_id_b, 1 ); + BOOST_TEST(!variant_response.is_null()); + BOOST_TEST(!streamed_response.empty()); + fc::variant streamed_as_variant = fc::json::from_string( streamed_response ); + BOOST_TEST(to_kv(variant_response) == to_kv(streamed_as_variant), boost::test_tools::per_element()); + } + + // Miss case: trxid not in the block. Variant path returns null variant; streaming path returns empty string. + { + auto missing_id = "00000000000000000000000000000000000000000000000000000000000000ff"_h; + fc::variant variant_response = get_transaction_trace( missing_id, 1 ); + std::string streamed_response = get_transaction_trace_json( missing_id, 1 ); + BOOST_TEST(variant_response.is_null()); + BOOST_TEST(streamed_response.empty()); + } + } + + // The HTTP layer commits to a 200 before the emitter runs, so both emitter factories must + // resolve every miss (absent block, absent trxid) to nullopt up front -- and an engaged + // emitter must produce exactly the bytes the buffer-building wrappers produce. + BOOST_FIXTURE_TEST_CASE(streaming_emitter_miss_resolution_and_parity, response_test_fixture) + { + auto trx_id = "0000000000000000000000000000000000000000000000000000000000000001"_h; + + action_trace_v0 action_trace{}; + action_trace.global_sequence = 0; + action_trace.receiver = "receiver"_n; + action_trace.account = "contract"_n; + action_trace.action = "action"_n; + action_trace.authorization = {{ "alice"_n, "active"_n }}; + action_trace.data = { 0x00, 0x01, 0x02, 0x03 }; + + auto block_trace = block_trace_v0 { + "b000000000000000000000000000000000000000000000000000000000000001"_h, + 1, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + chain::block_timestamp_type(0), + "bp.one"_n, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + std::vector { + { + trx_id, + std::vector { action_trace }, + 10, + 5, + std::vector{ chain::signature_type() }, + { chain::time_point_sec(), 1, 0, 100, 50, 0 }, + 1, + chain::block_timestamp_type(0), + std::optional{} + } + } + }; + + mock_get_block = [&block_trace]( uint32_t height ) -> get_block_t { + if (height != 1) { + return {}; + } + return std::make_tuple(data_log_entry(block_trace), false); + }; + + auto drive = [](const std::function& emit) { + std::string out; + { + fc::json_writer w(out); + emit(w); + } + return out; + }; + + // Engaged emitters emit byte-identically to the wrappers. + auto block_emitter = get_block_trace_emitter( 1 ); + BOOST_REQUIRE(block_emitter.has_value()); + BOOST_CHECK_EQUAL(drive(*block_emitter), get_block_trace_json( 1 )); + + auto trx_emitter = get_transaction_trace_emitter( trx_id, 1 ); + BOOST_REQUIRE(trx_emitter.has_value()); + BOOST_CHECK_EQUAL(drive(*trx_emitter), get_transaction_trace_json( trx_id, 1 )); + + // Misses resolve before any emitter exists: absent block, and absent trxid in a + // present block (contains_transaction gating). + BOOST_CHECK(!get_block_trace_emitter( 2 ).has_value()); + auto missing_id = "00000000000000000000000000000000000000000000000000000000000000ff"_h; + BOOST_CHECK(!get_transaction_trace_emitter( missing_id, 1 ).has_value()); + BOOST_CHECK(!get_transaction_trace_emitter( trx_id, 2 ).has_value()); + } + + // Trace responses must be abortable by the http layer's memory budget WHILE they + // serialize: driving the block emitter into a small-threshold guarded writer (the shape + // make_http_stream_response_handler provides) must throw the guard's abort partway + // through emission, with the buffer far below the full body size -- the old json_raw + // path materialized the entire body before the budget ever saw a byte. + BOOST_FIXTURE_TEST_CASE(streaming_block_response_budget_abort, response_test_fixture) + { + // 32 transactions x 4 KiB action payload: full body far exceeds the 8 KiB threshold + // below (each payload alone hex-expands past it). + constexpr size_t payload_bytes = 4 * 1024; + constexpr size_t transaction_count = 32; + std::vector transactions; + transactions.reserve(transaction_count); + for (size_t i = 0; i < transaction_count; ++i) { + action_trace_v0 action_trace{}; + action_trace.global_sequence = i; + action_trace.receiver = "receiver"_n; + action_trace.account = "contract"_n; + action_trace.action = "action"_n; + action_trace.authorization = {{ "alice"_n, "active"_n }}; + action_trace.data.assign(payload_bytes, static_cast(i)); + transactions.push_back(transaction_trace_v0{ + fc::sha256::hash(static_cast(i)), + std::vector { std::move(action_trace) }, + 10, + 5, + std::vector{ chain::signature_type() }, + { chain::time_point_sec(), 1, 0, 100, 50, 0 }, + 1, + chain::block_timestamp_type(0), + std::optional{} + }); + } + + auto block_trace = block_trace_v0 { + "b000000000000000000000000000000000000000000000000000000000000001"_h, + 1, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + chain::block_timestamp_type(0), + "bp.one"_n, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + std::move(transactions) + }; + + mock_get_block = [&block_trace]( uint32_t height ) -> get_block_t { + return std::make_tuple(data_log_entry(block_trace), false); + }; + + const std::string full_body = get_block_trace_json( 1 ); + + /// Foreign (non-fc, non-std) exception type modeling the http layer's budget abort + /// (sysio::detail::stream_response_budget_exceeded). + struct budget_abort {}; + constexpr size_t threshold = 8 * 1024; + + auto emitter = get_block_trace_emitter( 1 ); + BOOST_REQUIRE(emitter.has_value()); + std::string out; + fc::json_writer w(out, [](size_t sz) { if (sz > threshold) throw budget_abort{}; }, 256); + BOOST_CHECK_THROW((*emitter)(w), budget_abort); + // Aborted within a token or two of the threshold; nowhere near the full body. + BOOST_CHECK_LT(out.size(), 2 * threshold); + BOOST_CHECK_GT(full_body.size(), 8 * threshold); + } + +BOOST_AUTO_TEST_SUITE_END() diff --git a/tests/test_chain_plugin.cpp b/tests/test_chain_plugin.cpp index 92a5f0e12b..29fab3b666 100644 --- a/tests/test_chain_plugin.cpp +++ b/tests/test_chain_plugin.cpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include #include #include @@ -17,7 +19,6 @@ using namespace sysio; using namespace sysio::chain; using namespace sysio::chain_apis; using namespace sysio::testing; -using namespace sysio::chain_apis; using namespace fc; using mvo = fc::mutable_variant_object; @@ -137,4 +138,88 @@ BOOST_FIXTURE_TEST_CASE(account_results_total_resources_test, chain_plugin_teste } FC_LOG_AND_RETHROW() } +// Byte-identical compat between the variant-cb path (fc::variant + +// fc::json::to_string) and the streaming-cb path (fc::to_json_stream via +// reflector dispatch + per-type overrides for chain::name / asset / symbol). +// Pins the parity that get_account's add_api_stream registration depends on. +BOOST_FIXTURE_TEST_CASE(get_account_streaming_vs_variant_byte_identical, chain_plugin_tester) { try { + produce_blocks(10); + setup_system_accounts(); + produce_blocks(); + create_account("alice1111111"_n, config::system_account_name); + transfer(name("sysio"), name("alice1111111"), core_from_string("650000000.0000"), name("sysio")); + + read_only::get_account_results results = get_account_info(name("alice1111111")); + + const std::string variant_path = + fc::json::to_string(fc::variant(results), fc::time_point::maximum()); + const std::string stream_path = fc::to_json_string(results); + + BOOST_CHECK_EQUAL(variant_path, stream_path); +} FC_LOG_AND_RETHROW() } + +// Byte-identical compat for the new get_block_stream direct path. Builds a +// populated block (account creation + a transfer; both decode through the +// resolver-located sysio.token / system ABI), then compares +// `to_string(convert_block(block))` against the streaming +// `convert_block_stream(block, json_writer)` output. Drives the +// abi_serializer::to_json_stream template -- any drift in the +// ABI-aware stream path's token order, quoting, or action data form would +// surface here. +BOOST_FIXTURE_TEST_CASE(get_block_streaming_vs_variant_byte_identical, chain_plugin_tester) { try { + produce_blocks(10); + setup_system_accounts(); + produce_blocks(); + create_account("alice1111111"_n, config::system_account_name); + transfer(name("sysio"), name("alice1111111"), core_from_string("100.0000"), name("sysio")); + produce_block(); + + auto signed_block = control->fetch_block_by_number(control->head().block_num()); + BOOST_REQUIRE(signed_block); + + std::optional _tracked_votes; + read_only ro(*control, {}, {}, _tracked_votes, + fc::microseconds::maximum(), fc::microseconds::maximum(), {}); + + auto resolver_v = ro.get_block_serializers(signed_block, fc::microseconds::maximum()); + auto resolver_s = ro.get_block_serializers(signed_block, fc::microseconds::maximum()); + + const fc::variant variant_block = ro.convert_block(signed_block, resolver_v); + const std::string variant_path = fc::json::to_string(variant_block, fc::time_point::maximum()); + + std::string stream_path; + { + fc::json_writer w(stream_path); + ro.convert_block_stream(signed_block, resolver_s, w); + BOOST_REQUIRE(w.balanced()); + } + + BOOST_CHECK_EQUAL(variant_path, stream_path); +} FC_LOG_AND_RETHROW() } + +// get_finalizer_info absent-policy schema: both policy keys are ALWAYS present and emit an +// explicit JSON null when the policy does not exist (fc::nullable fields). A fresh chain +// has an active policy (Savanna activates at genesis) and no pending policy, so this pins +// the disengaged shape end-to-end plus byte parity between the variant and streaming paths. +// A reflected std::optional would omit the pending key entirely -- the regression this +// guards against. +BOOST_FIXTURE_TEST_CASE(get_finalizer_info_absent_policy_emits_null, chain_plugin_tester) { try { + produce_blocks(2); + + std::optional _tracked_votes; + read_only ro(*control, {}, {}, _tracked_votes, + fc::microseconds::maximum(), fc::microseconds::maximum(), {}); + + const read_only::get_finalizer_info_result result = ro.get_finalizer_info({}, fc::time_point::maximum()); + BOOST_REQUIRE(result.active_finalizer_policy.has_value()); + BOOST_REQUIRE(!result.pending_finalizer_policy.has_value()); + + const std::string variant_path = fc::json::to_string(fc::variant(result), fc::time_point::maximum()); + const std::string stream_path = fc::to_json_string(result); + + BOOST_CHECK_EQUAL(variant_path, stream_path); + BOOST_CHECK(stream_path.find("\"pending_finalizer_policy\":null") != std::string::npos); + BOOST_CHECK(stream_path.find("\"active_finalizer_policy\":{") != std::string::npos); +} FC_LOG_AND_RETHROW() } + BOOST_AUTO_TEST_SUITE_END() diff --git a/tests/test_get_table_rows_page.cpp b/tests/test_get_table_rows_page.cpp index 56a480a12f..c97472f5c3 100644 --- a/tests/test_get_table_rows_page.cpp +++ b/tests/test_get_table_rows_page.cpp @@ -23,6 +23,8 @@ #include +#include +#include #include using namespace sysio; @@ -311,4 +313,119 @@ BOOST_FIXTURE_TEST_CASE(find_with_index_returns_single_match_even_when_all_rows, BOOST_CHECK_EQUAL(res.rows[0].get_object()["sec64"].as_uint64(), 3u); } FC_LOG_AND_RETHROW() +// Byte-identical compat between the variant-cb path (fc::variant + fc::json::to_string) +// and the streaming-cb path (fc::to_json_stream via reflector dispatch). The HTTP +// /v1/chain/get_table_rows endpoint flips between these via add_api / add_api_stream; +// any drift in the streaming path's emitted JSON would break clients that depend on +// the exact byte sequence (eg keypair-string ordering, integer formatting). This case +// pins parity for a representative populated result. +BOOST_FIXTURE_TEST_CASE(streaming_vs_variant_byte_identical, validating_tester) try { + deploy_contract(*this); + populate_numobjs(*this, 4); + + auto p = numobjs_params(); + p.all_rows = true; + p.show_payer = true; + + auto ro = make_read_only(*this); + auto res = run(ro, p); + + const std::string variant_path = + fc::json::to_string(fc::variant(res), fc::time_point::maximum()); + const std::string stream_path = fc::to_json_string(res); + + BOOST_CHECK_EQUAL(variant_path, stream_path); +} FC_LOG_AND_RETHROW() + +namespace { + /// Run the new direct-streaming `get_table_rows_stream` to completion and return the + /// emitted JSON. Fails the test if the RPC produced an exception_ptr. + std::string run_stream(read_only& ro, const read_only::get_table_rows_params& p) { + auto outer = ro.get_table_rows_stream(p, fc::time_point::maximum())(); + BOOST_REQUIRE(!std::holds_alternative(outer)); + auto emit = std::get(std::move(outer)); + std::string out; + { + fc::json_writer w(out); + emit(w); + BOOST_REQUIRE(w.balanced()); + } + return out; + } +} + +// Byte-identical compat for the new direct-streaming path. `get_table_rows_stream` +// bypasses the per-row fc::variant assembly step; its emitted JSON must match +// `fc::json::to_string(fc::variant(get_table_rows().result))` for the same params. +// Any drift here would break HTTP clients that hit /v1/chain/get_table_rows after the +// chain_api_plugin migration to CHAIN_RO_CALL_STREAM_POST_DIRECT. +BOOST_FIXTURE_TEST_CASE(stream_direct_vs_variant_byte_identical, validating_tester) try { + deploy_contract(*this); + populate_numobjs(*this, 4); + + auto ro = make_read_only(*this); + + // Three shapes worth pinning: full wrapper + payer, all_rows escape hatch, and + // values_only stripped form. Each exercises a distinct branch in the stream + // emit closure (wrapped object vs bare value, more/next_key suppression). + for (auto setup : { +[](read_only::get_table_rows_params& q) { q.all_rows = true; q.show_payer = true; }, + +[](read_only::get_table_rows_params& q) { q.show_payer = true; q.limit = 2; }, + +[](read_only::get_table_rows_params& q) { q.values_only = true; q.all_rows = true; } }) { + auto p = numobjs_params(); + setup(p); + + auto via_variant = run(ro, p); + const std::string variant_json = + fc::json::to_string(fc::variant(via_variant), fc::time_point::maximum()); + + const std::string stream_json = run_stream(ro, p); + + BOOST_CHECK_EQUAL(variant_json, stream_json); + } +} FC_LOG_AND_RETHROW() + +// A row whose value fails ABI decode mid-emission must fall back to a hex string without +// corrupting the response. The streaming path writes tokens as it walks the ABI, so a decode +// failure after some fields were already emitted has to rewind before the hex fallback -- a +// regression here produces structurally invalid JSON on the wire (a hex token appended after a +// half-written value object). Force the failure by re-setting the contract ABI with an extra +// trailing field on `numobj`: stored rows then under-run the declared struct and throw +// "stream unexpectedly ended" after the five real fields were emitted. +BOOST_FIXTURE_TEST_CASE(stream_decode_failure_falls_back_to_hex_valid_json, validating_tester) try { + deploy_contract(*this); + populate_numobjs(*this, 3); + + const std::string abi_json = test_contracts::get_table_test_abi(); + abi_def abi = fc::json::from_string(abi_json).as(); + for (auto& st : abi.structs) { + if (st.name == "numobj") { + st.fields.push_back({"extra_field", "uint64"}); + } + } + set_abi("test"_n, fc::json::to_string(fc::variant(abi), fc::time_point::maximum())); + produce_block(); + + auto ro = make_read_only(*this); + auto p = numobjs_params(); + p.show_payer = true; + + // The streamed body must be parseable JSON even though every row's value decode threw. + const std::string stream_json = run_stream(ro, p); + fc::variant parsed; + BOOST_REQUIRE_NO_THROW(parsed = fc::json::from_string(stream_json)); + + // Every row fell back to the bare hex-string form (not a half-decoded object). + const auto& rows = parsed.get_object()["rows"].get_array(); + BOOST_REQUIRE_EQUAL(rows.size(), 3u); + for (const auto& row : rows) { + BOOST_CHECK(row.get_object()["value"].is_string()); + } + + // And the fallback shape stays byte-identical to the variant path's fallback. + auto via_variant = run(ro, p); + const std::string variant_json = + fc::json::to_string(fc::variant(via_variant), fc::time_point::maximum()); + BOOST_CHECK_EQUAL(variant_json, stream_json); +} FC_LOG_AND_RETHROW() + BOOST_AUTO_TEST_SUITE_END() diff --git a/unittests/abi_tests.cpp b/unittests/abi_tests.cpp index df0c559a7f..663e5b7b47 100644 --- a/unittests/abi_tests.cpp +++ b/unittests/abi_tests.cpp @@ -1,12 +1,16 @@ #include +#include +#include #include #include #include +#include #include #include #include +#include #include #include #include @@ -52,6 +56,21 @@ static fc::time_point get_deadline() { static auto yield_fn() { return abi_serializer::create_yield_function( max_serialization_time ); } +// Verify the streaming `binary_to_json_stream` path emits byte-identical JSON to +// `to_string(binary_to_variant(...))` for the same packed bytes. Called from the +// round-trip helpers below so every type already exercised by abi_tests gets a +// parity check on the streaming path for free. +void verify_stream_matches_variant( const abi_serializer& abis, const type_name& type, const bytes& packed, const std::string& variant_json ) +{ + std::string stream_json; + { + fc::json_writer w(stream_json); + abis.binary_to_json_stream(type, packed, w, yield_fn()); + BOOST_REQUIRE( w.balanced() ); + } + BOOST_TEST( stream_json == variant_json ); +} + // verify that round trip conversion, via bytes, reproduces the exact same data fc::variant verify_byte_round_trip_conversion( const abi_serializer& abis, const type_name& type, const fc::variant& var ) { @@ -66,6 +85,8 @@ fc::variant verify_byte_round_trip_conversion( const abi_serializer& abis, const std::string r3 = fc::json::to_string(var3, get_deadline()); BOOST_TEST( r2 == r3 ); + verify_stream_matches_variant( abis, type, bytes, r2 ); + auto bytes2 = abis.variant_to_binary(type, var2, yield_fn()); auto bytes3 = abis.variant_to_binary(type, var3, max_serialization_time); BOOST_TEST( bytes2 == bytes3 ); @@ -86,6 +107,7 @@ void verify_round_trip_conversion( const abi_serializer& abis, const type_name& BOOST_REQUIRE_EQUAL(fc::json::to_string(var2, get_deadline()), expected_json); auto var3 = abis.binary_to_variant(type, b, max_serialization_time ); BOOST_REQUIRE_EQUAL(fc::json::to_string(var3, get_deadline()), expected_json); + verify_stream_matches_variant(abis, type, bytes, expected_json); auto bytes2 = abis.variant_to_binary(type, var2, yield_fn()); BOOST_REQUIRE_EQUAL(fc::to_hex(bytes2), hex); auto b2 = abis.variant_to_binary(type, var3, max_serialization_time); @@ -128,6 +150,8 @@ fc::variant verify_type_round_trip_conversion( const abi_serializer& abis, const std::string r3 = fc::json::to_string(var3, get_deadline()); BOOST_TEST( r2 == r3 ); + verify_stream_matches_variant( abis, type, bytes, r2 ); + auto bytes2 = abis.variant_to_binary(type, var2, yield_fn()); auto b3 = abis.variant_to_binary(type, var3, max_serialization_time); BOOST_TEST( bytes2 == b3 ); @@ -731,15 +755,142 @@ BOOST_AUTO_TEST_CASE(uint_types) } FC_LOG_AND_RETHROW() } +// int64/uint64 quoting boundaries through the ABI paths: values with magnitude past +// fc::json_integer_quote_magnitude (0xffffffff) emit as quoted JSON strings, values at the +// boundary stay bare -- and binary_to_json_stream must agree byte-for-byte with +// binary_to_variant + json::to_string (verify_byte_round_trip_conversion runs +// verify_stream_matches_variant on every case). +BOOST_AUTO_TEST_CASE(large_int_quoting_boundaries) +{ try { -BOOST_AUTO_TEST_CASE(general) + const char* test_abi = R"=====( + { + "version": "sysio::abi/1.0", + "types": [], + "structs": [{ + "name": "int_boundaries", + "base": "", + "fields": [{ + "name": "i64_bare_max", + "type": "int64" + },{ + "name": "i64_quoted", + "type": "int64" + },{ + "name": "i64_neg_bare", + "type": "int64" + },{ + "name": "i64_neg_quoted", + "type": "int64" + },{ + "name": "u64_bare", + "type": "uint64" + },{ + "name": "u64_quoted", + "type": "uint64" + }] + }], + "actions": [], + "tables": [], + "ricardian_clauses": [] + } + )====="; + + abi_serializer abis(fc::json::from_string(test_abi).as(), yield_fn()); + + const char* test_data = R"=====( + { + "i64_bare_max" : 4294967295, + "i64_quoted" : "4294967296", + "i64_neg_bare" : -4294967295, + "i64_neg_quoted" : "-4294967296", + "u64_bare" : 4294967295, + "u64_quoted" : "18446744073709551615" + } + )====="; + + auto var2 = verify_byte_round_trip_conversion(abis, "int_boundaries", fc::json::from_string(test_data)); + + // Pin the emitted shapes: at the quote magnitude stays bare, one past it emits quoted. + const std::string json = fc::json::to_string(var2, get_deadline()); + BOOST_CHECK(json.find("\"i64_bare_max\":4294967295") != std::string::npos); + BOOST_CHECK(json.find("\"i64_quoted\":\"4294967296\"") != std::string::npos); + BOOST_CHECK(json.find("\"i64_neg_bare\":-4294967295") != std::string::npos); + BOOST_CHECK(json.find("\"i64_neg_quoted\":\"-4294967296\"") != std::string::npos); + BOOST_CHECK(json.find("\"u64_bare\":4294967295") != std::string::npos); + BOOST_CHECK(json.find("\"u64_quoted\":\"18446744073709551615\"") != std::string::npos); + +} FC_LOG_AND_RETHROW() } + +// Non-finite float32/float64 bit patterns through the ABI paths: the variant path +// (s_fc_to_string) formats non-finite doubles explicitly as "nan" / "-nan" / "inf" / +// "-inf" -- platform-independent, unlike the C library formatting it bypasses (glibc +// prints "-nan" where Apple prints "nan") -- and binary_to_json_stream must agree +// byte-for-byte. NaN never compares, so its sign only survives via signbit -- and +// since no JSON input can produce a NaN, the packed bytes are built directly from raw +// bit patterns, exactly as an arbitrary on-chain payload could. +BOOST_AUTO_TEST_CASE(non_finite_float_sign_parity) { try { - auto abi = sysio_contract_abi(fc::json::from_string(my_abi).as()); + const char* test_abi = R"=====( + { + "version": "sysio::abi/1.0", + "types": [], + "structs": [{ + "name": "floats", + "base": "", + "fields": [{ + "name": "f64_pos_nan", + "type": "float64" + },{ + "name": "f64_neg_nan", + "type": "float64" + },{ + "name": "f64_neg_inf", + "type": "float64" + },{ + "name": "f32_neg_nan", + "type": "float32" + }] + }], + "actions": [], + "tables": [], + "ricardian_clauses": [] + } + )====="; - abi_serializer abis(abi_def(abi), yield_fn()); + abi_serializer abis(fc::json::from_string(test_abi).as(), yield_fn()); + + const double f64_pos_nan = std::bit_cast(uint64_t{0x7FF8000000000000ULL}); + const double f64_neg_nan = std::bit_cast(uint64_t{0xFFF8000000000000ULL}); + const double f64_neg_inf = -std::numeric_limits::infinity(); + const float f32_neg_nan = std::bit_cast(uint32_t{0xFFC00000u}); + + bytes packed(3 * sizeof(double) + sizeof(float)); + fc::datastream ds(packed.data(), packed.size()); + fc::raw::pack(ds, f64_pos_nan); + fc::raw::pack(ds, f64_neg_nan); + fc::raw::pack(ds, f64_neg_inf); + fc::raw::pack(ds, f32_neg_nan); + + const std::string variant_json = + fc::json::to_string(abis.binary_to_variant("floats", packed, yield_fn()), get_deadline()); + + // Pin the legacy shapes, sign included. + BOOST_CHECK(variant_json.find("\"f64_pos_nan\":\"nan\"") != std::string::npos); + BOOST_CHECK(variant_json.find("\"f64_neg_nan\":\"-nan\"") != std::string::npos); + BOOST_CHECK(variant_json.find("\"f64_neg_inf\":\"-inf\"") != std::string::npos); + BOOST_CHECK(variant_json.find("\"f32_neg_nan\":\"-nan\"") != std::string::npos); + + verify_stream_matches_variant(abis, "floats", packed, variant_json); + +} FC_LOG_AND_RETHROW() } - const char *my_other = R"=====( + +// Shared fixture for `general` and the streaming-parity test. Top-level fields +// cover every built-in plus inherited base structs, large-int quoting boundaries, +// optional/array containers, and the ABI-described meta-types. +static const char* my_other_json = R"=====( { "publickey" : "SYS6MRyAjQq8ud7hVNYcfnVPJqcVpscN5So8BhtHuGYqET5GDW5CV", "publickey_arr" : ["SYS6MRyAjQq8ud7hVNYcfnVPJqcVpscN5So8BhtHuGYqET5GDW5CV","SYS6MRyAjQq8ud7hVNYcfnVPJqcVpscN5So8BhtHuGYqET5GDW5CV","SYS6MRyAjQq8ud7hVNYcfnVPJqcVpscN5So8BhtHuGYqET5GDW5CV"], @@ -932,7 +1083,14 @@ BOOST_AUTO_TEST_CASE(general) } )====="; - auto var = fc::json::from_string(my_other); +BOOST_AUTO_TEST_CASE(general) +{ try { + + auto abi = sysio_contract_abi(fc::json::from_string(my_abi).as()); + + abi_serializer abis(abi_def(abi), yield_fn()); + + auto var = fc::json::from_string(my_other_json); verify_byte_round_trip_conversion(abi_serializer{std::move(abi), yield_fn()}, "A", var); } FC_LOG_AND_RETHROW() } @@ -3393,17 +3551,19 @@ BOOST_AUTO_TEST_CASE(abi_to_variant__add_action__good_return_value) abi_serializer abis(abi_def(abidef), yield_fn()); { + fc::variant trace_var; + abi_serializer::to_variant(at, trace_var, get_resolver(abidef), yield_fn()); mutable_variant_object mvo; - sysio::chain::impl::abi_traverse_context ctx(yield_fn(), fc::microseconds{}); - sysio::chain::impl::abi_to_variant::add(mvo, "action_traces", at, get_resolver(abidef), ctx); + mvo("action_traces", std::move(trace_var)); std::string res = fc::json::to_string(mvo, get_deadline()); BOOST_CHECK_EQUAL(res, expected_json); } { + fc::variant trace_var; + abi_serializer::to_variant(at, trace_var, get_resolver(abidef), max_serialization_time); mutable_variant_object mvo; - sysio::chain::impl::abi_traverse_context ctx(abi_serializer::create_depth_yield_function(), max_serialization_time); - sysio::chain::impl::abi_to_variant::add(mvo, "action_traces", at, get_resolver(abidef), ctx); + mvo("action_traces", std::move(trace_var)); std::string res = fc::json::to_string(mvo, get_deadline()); BOOST_CHECK_EQUAL(res, expected_json); @@ -3428,17 +3588,19 @@ BOOST_AUTO_TEST_CASE(abi_to_variant__add_action__bad_return_value) abi_serializer abis(abi_def(abidef), yield_fn()); { + fc::variant trace_var; + abi_serializer::to_variant(at, trace_var, get_resolver(abidef), yield_fn()); mutable_variant_object mvo; - sysio::chain::impl::abi_traverse_context ctx(yield_fn(), fc::microseconds{}); - sysio::chain::impl::abi_to_variant::add(mvo, "action_traces", at, get_resolver(abidef), ctx); + mvo("action_traces", std::move(trace_var)); std::string res = fc::json::to_string(mvo, get_deadline()); BOOST_CHECK_EQUAL(res, expected_json); } { + fc::variant trace_var; + abi_serializer::to_variant(at, trace_var, get_resolver(abidef), max_serialization_time); mutable_variant_object mvo; - sysio::chain::impl::abi_traverse_context ctx(abi_serializer::create_depth_yield_function(), max_serialization_time); - sysio::chain::impl::abi_to_variant::add(mvo, "action_traces", at, get_resolver(abidef), ctx); + mvo("action_traces", std::move(trace_var)); std::string res = fc::json::to_string(mvo, get_deadline()); BOOST_CHECK_EQUAL(res, expected_json); @@ -3473,23 +3635,127 @@ BOOST_AUTO_TEST_CASE(abi_to_variant__add_action__no_return_value) abi_serializer abis(abi_def(abidef), yield_fn()); { + fc::variant trace_var; + abi_serializer::to_variant(at, trace_var, get_resolver(abidef), yield_fn()); mutable_variant_object mvo; - sysio::chain::impl::abi_traverse_context ctx(yield_fn(), fc::microseconds{}); - sysio::chain::impl::abi_to_variant::add(mvo, "action_traces", at, get_resolver(abidef), ctx); + mvo("action_traces", std::move(trace_var)); std::string res = fc::json::to_string(mvo, get_deadline()); BOOST_CHECK_EQUAL(res, expected_json); } { + fc::variant trace_var; + abi_serializer::to_variant(at, trace_var, get_resolver(abidef), max_serialization_time); mutable_variant_object mvo; - sysio::chain::impl::abi_traverse_context ctx(abi_serializer::create_depth_yield_function(), max_serialization_time); - sysio::chain::impl::abi_to_variant::add(mvo, "action_traces", at, get_resolver(abidef), ctx); + mvo("action_traces", std::move(trace_var)); std::string res = fc::json::to_string(mvo, get_deadline()); BOOST_CHECK_EQUAL(res, expected_json); } } +// When the action_result decode fails partway, the field is silently omitted +// from the JSON (existing variant-path behavior). The streaming sink had a +// bug where sink.key("return_value_data") was emitted BEFORE the inner unpack, +// so a mid-stream throw left a dangling key + orphan frame and end_object() +// closed the wrong scope. Stream + variant must produce identical JSON. +BOOST_AUTO_TEST_CASE(abi_to_variant__add_action_trace__return_value_unpack_throws_silently_omitted) +{ + action_trace at; + std::string expected_json; + // Set up a return_value whose declared type is "string" (length-prefix + body) + // but the bytes are a single 0x09 length-prefix with no body -> unpack throws + // partway. generate_action_trace's parsable=false branch suppresses the + // return_value_data field in expected_json, which is what we want. + std::tie(at, expected_json) = generate_action_trace(std::optional{0x09}, "09", false); + + auto abi = R"({ + "version": "sysio::abi/1.0", + "structs": [ + {"name": "acttest", "base": "", "fields": [ + {"name": "str", "type": "string"} + ]} + ], + "action_results": [ + {"name": "acttest", "result_type": "string"} + ] + })"; + auto abidef = fc::json::from_string(abi).as(); + + // Variant path: known good, builds expected_json (return_value_data omitted). + fc::variant variant_v; + abi_serializer::to_variant(at, variant_v, get_resolver(abidef), yield_fn()); + mutable_variant_object variant_mvo; + variant_mvo("action_traces", std::move(variant_v)); + const std::string variant_json = fc::json::to_string(variant_mvo, get_deadline()); + BOOST_CHECK_EQUAL(variant_json, expected_json); + + // Streaming path: must emit byte-identical JSON, with no orphan frame. + std::string stream_json; + { + fc::json_writer w(stream_json); + w.begin_object(); + w.key("action_traces"); + abi_serializer::to_json_stream(at, w, get_resolver(abidef), yield_fn()); + w.end_object(); + BOOST_REQUIRE(w.balanced()); + } + BOOST_CHECK_EQUAL(stream_json, expected_json); + BOOST_CHECK_NO_THROW(fc::json::from_string(stream_json)); +} + +// When action data fails ABI decode partway through (truncated bytes, missing +// fields, etc.), both the variant path and the streaming path must fall back +// to the hex-only form. The streaming sink had a bug where sink.key("data") +// was emitted BEFORE the inner unpack, so a mid-stream throw left the writer +// in a half-written state and the catch-side hex fallback emitted a duplicate +// "data" key plus an orphan inner object frame, producing invalid JSON. +BOOST_AUTO_TEST_CASE(abi_to_variant__add_action__data_unpack_throws_falls_back_to_hex) +{ + auto abi = R"({ + "version": "sysio::abi/1.0", + "structs": [ + {"name": "acttest", "base": "", "fields": [ + {"name": "str", "type": "string"} + ]} + ], + "actions": [ + {"name": "acttest", "type": "acttest", "ricardian_contract": ""} + ] + })"; + auto abidef = fc::json::from_string(abi).as(); + + // String length prefix says 9 bytes but no body follows -> unpack throws partway. + action act( + std::vector{ {account_name{"acctest"}, permission_name{"active"}} }, + account_name{"acctest"}, + action_name{"acttest"}, + bytes{0x09}); + + const std::string expected_hex = "09"; + const std::string expected_json = + "{\"account\":\"acctest\",\"name\":\"acttest\"," + "\"authorization\":[{\"actor\":\"acctest\",\"permission\":\"active\"}]," + "\"data\":\"" + expected_hex + "\"," + "\"hex_data\":\"" + expected_hex + "\"}"; + + { + fc::variant v; + abi_serializer::to_variant(act, v, get_resolver(abidef), yield_fn()); + const std::string s = fc::json::to_string(v, get_deadline()); + BOOST_CHECK_EQUAL(s, expected_json); + } + { + std::string s; + fc::json_writer w(s); + abi_serializer::to_json_stream(act, w, get_resolver(abidef), yield_fn()); + BOOST_REQUIRE(w.balanced()); + BOOST_CHECK_EQUAL(s, expected_json); + // Round-trip parses cleanly (catches duplicate-key / orphan-frame regressions). + BOOST_CHECK_NO_THROW(fc::json::from_string(s)); + } +} + BOOST_AUTO_TEST_CASE(enum_types) { using sysio::testing::fc_exception_message_starts_with;