From a698ea2242e33765f7aa0529b32b8a1f9a80a581 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Sat, 18 Apr 2026 21:04:20 -0500 Subject: [PATCH 01/71] libfc: add fc::json_writer + fc::to_json_stream streaming JSON emission Adds a streaming counterpart to the existing fc::to_variant + fc::json::to_string serialization path. Callers drive fc::json_writer with explicit begin_object / key / value_* calls that append bytes directly to an output std::string, skipping the mutable_variant_object -> fc::variant -> fc::json::to_string pipeline that dominates allocations in chain_plugin/trace_api HTTP responses. fc::to_json_stream mirrors fc::to_variant: a reflector-based primary template that dispatches via fc::reflector, plus scalar / container overloads. Any type marked with FC_REFLECT(...) or FC_REFLECT_ENUM(...) gains a JSON serializer with no further code change, and optional fields are omitted rather than emitted as explicit null to match the established to_variant_visitor semantics. Provided overloads: - bool, {int,uint}{8,16,32,64}_t, float, double - std::string, std::string_view, const char* - std::vector, std::array, std::map, std::unordered_map - std::optional - FC_REFLECT'd structs and FC_REFLECT_ENUM'd enums json_writer::raw_value splices a preformatted JSON fragment at a value position; to_json_stream_via_variant is an explicit escape hatch for incremental adoption of types that only have a to_variant overload so far. Scope: this is infrastructure only. Downstream HTTP endpoints (trace_api get_block / get_actions, chain_plugin get_block / get_account / get_table_rows) will migrate in follow-up PRs as each hot path is profiled and converted. Tests: libraries/libfc/test/io/test_json_stream.cpp covers primitives, escaping, optional omission, reflected structs + nesting, enum emission, std::map object emission, and raw_value splicing. 11 cases, all pass. --- libraries/libfc/include/fc/io/json_stream.hpp | 174 +++++++++++++++ .../libfc/include/fc/reflect/json_stream.hpp | 199 ++++++++++++++++++ libraries/libfc/test/CMakeLists.txt | 1 + libraries/libfc/test/io/test_json_stream.cpp | 149 +++++++++++++ 4 files changed, 523 insertions(+) create mode 100644 libraries/libfc/include/fc/io/json_stream.hpp create mode 100644 libraries/libfc/include/fc/reflect/json_stream.hpp create mode 100644 libraries/libfc/test/io/test_json_stream.cpp diff --git a/libraries/libfc/include/fc/io/json_stream.hpp b/libraries/libfc/include/fc/io/json_stream.hpp new file mode 100644 index 0000000000..7ad9561be7 --- /dev/null +++ b/libraries/libfc/include/fc/io/json_stream.hpp @@ -0,0 +1,174 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include + +namespace fc { + +/** + * Streaming JSON writer that emits tokens directly into an output std::string. + * + * Designed as a faster alternative to the fc::mutable_variant_object -> fc::variant -> + * fc::json::to_string pipeline used throughout the HTTP endpoints. Instead of allocating + * per-field variant_object entries and per-string fc::variant wrappers, callers drive the + * writer with explicit begin_object / key / value_* calls that append bytes to the output + * buffer in place. + * + * Contract: + * - Values must appear in value-position only (top-level, after a key(), or inside an + * array). Nesting is validated with asserts in debug builds. + * - end_object / end_array must balance begin_object / begin_array. + * - String content is JSON-escaped via fc::escape_string; control characters become + * \\uXXXX, surrogate pairs are produced when needed. + * - raw_value appends an already-serialized JSON fragment verbatim; callers are + * responsible for its validity (used e.g. to embed fc::variant output from legacy + * code paths). + */ +class json_writer { +public: + explicit json_writer(std::string& out) + : out_(out) + { + // Enough room for a reasonable response without reallocation; callers that know + // the expected size can reserve() themselves before constructing the writer. + out_.reserve(out_.size() + 4096); + } + + void begin_object() { + value_prefix(); + out_.push_back('{'); + stack_.push_back(frame{context::object, false}); + } + + void end_object() { + assert(!stack_.empty() && stack_.back().ctx == context::object); + out_.push_back('}'); + stack_.pop_back(); + } + + void begin_array() { + value_prefix(); + out_.push_back('['); + stack_.push_back(frame{context::array, false}); + } + + void end_array() { + assert(!stack_.empty() && stack_.back().ctx == context::array); + out_.push_back(']'); + stack_.pop_back(); + } + + void key(std::string_view k) { + assert(!stack_.empty() && stack_.back().ctx == context::object); + if (stack_.back().has_item) { + out_.push_back(','); + } else { + stack_.back().has_item = true; + } + out_.push_back('"'); + // key strings must also be escaped (eg field names containing special chars are rare, + // but correctness matters on untrusted input echoed into error messages). + fc::escape_string(k, out_, json::yield_function_t(), true); + out_.push_back('"'); + out_.push_back(':'); + // A key mandates a value follows; suppress the comma from value_prefix for that value. + awaiting_value_ = true; + } + + void value_string(std::string_view s) { + value_prefix(); + out_.push_back('"'); + fc::escape_string(s, out_, json::yield_function_t(), true); + out_.push_back('"'); + } + + void value_int64(int64_t n) { value_prefix(); append_signed(n); } + void value_uint64(uint64_t n) { value_prefix(); append_unsigned(n); } + void value_int32(int32_t n) { value_int64(n); } + void value_uint32(uint32_t n) { value_uint64(n); } + void value_int16(int16_t n) { value_int64(n); } + void value_uint16(uint16_t n) { value_uint64(n); } + void value_int8(int8_t n) { value_int64(n); } + void value_uint8(uint8_t n) { value_uint64(n); } + + void value_double(double d) { + value_prefix(); + char buf[32]; + // "%.17g" preserves round-trip precision for IEEE 754 doubles. Callers that need + // bit-exact formatting should emit via raw_value(). + int n = std::snprintf(buf, sizeof(buf), "%.17g", d); + if (n > 0) out_.append(buf, static_cast(n)); + else out_.append("null", 4); // NaN/inf are not valid JSON; emit null defensively. + } + + void value_bool(bool b) { + value_prefix(); + if (b) out_.append("true", 4); + else out_.append("false", 5); + } + + void value_null() { + value_prefix(); + out_.append("null", 4); + } + + /// Append a preformatted JSON fragment at a value position. Used to embed output + /// from legacy fc::variant paths (eg abi_serializer::binary_to_variant + json::to_string) + /// without an additional parse/re-emit cycle. + void raw_value(std::string_view raw) { + value_prefix(); + out_.append(raw.data(), raw.size()); + } + + /// True when all begin_* have been paired with end_*. Asserted internally on destructor + /// usage via value_prefix() is not possible, so callers can check this in tests. + bool balanced() const { return stack_.empty(); } + +private: + enum class context : uint8_t { object, array }; + struct frame { + context ctx; + bool has_item; // true once one element or key:value has been emitted in this frame + }; + + void value_prefix() { + if (awaiting_value_) { + // Value right after a key() - no separator, first-item bookkeeping already set. + awaiting_value_ = false; + return; + } + if (stack_.empty()) { + // Top-level value (eg a bare array/object at the root). Nothing to prefix. + return; + } + // Must be in array context when a raw value appears without a preceding key. + assert(stack_.back().ctx == context::array); + if (stack_.back().has_item) { + out_.push_back(','); + } else { + stack_.back().has_item = true; + } + } + + void append_signed(int64_t n) { + char buf[24]; + int k = std::snprintf(buf, sizeof(buf), "%lld", static_cast(n)); + if (k > 0) out_.append(buf, static_cast(k)); + } + void append_unsigned(uint64_t n) { + char buf[24]; + int k = std::snprintf(buf, sizeof(buf), "%llu", static_cast(n)); + if (k > 0) out_.append(buf, static_cast(k)); + } + + std::string& out_; + std::vector stack_; // small; vector is fine and avoids extra deps + bool awaiting_value_ = false; +}; + +} // namespace fc diff --git a/libraries/libfc/include/fc/reflect/json_stream.hpp b/libraries/libfc/include/fc/reflect/json_stream.hpp new file mode 100644 index 0000000000..b3269d6922 --- /dev/null +++ b/libraries/libfc/include/fc/reflect/json_stream.hpp @@ -0,0 +1,199 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fc { + +/** + * to_json_stream is the streaming counterpart to to_variant: it emits the JSON form of a + * reflected C++ value directly into a json_writer without building an intermediate + * fc::variant tree. The primary template dispatches through fc::reflector in the same + * way fc::to_variant does, so any type already marked with FC_REFLECT(...) gets a JSON + * serializer for free. + * + * Overloads for primitives and standard containers are provided below. Domain-specific + * types (sha256, public_key_type, variant, time_point, ...) can add their own + * to_json_stream overload next to their existing to_variant overload; until that happens + * callers can fall back to raw_value(fc::json::to_string(variant(v))) for individual + * fields at measured hot spots. + */ +template +void to_json_stream(const T& o, json_writer& w); + +// -- Scalar overloads --------------------------------------------------------------------- + +inline void to_json_stream(bool b, json_writer& w) { w.value_bool(b); } +inline void to_json_stream(int8_t n, json_writer& w) { w.value_int8(n); } +inline void to_json_stream(uint8_t n, json_writer& w) { w.value_uint8(n); } +inline void to_json_stream(int16_t n, json_writer& w) { w.value_int16(n); } +inline void to_json_stream(uint16_t n, json_writer& w) { w.value_uint16(n); } +inline void to_json_stream(int32_t n, json_writer& w) { w.value_int32(n); } +inline void to_json_stream(uint32_t n, json_writer& w) { w.value_uint32(n); } +inline void to_json_stream(int64_t n, json_writer& w) { w.value_int64(n); } +inline void to_json_stream(uint64_t n, json_writer& w) { w.value_uint64(n); } +inline void to_json_stream(float f, json_writer& w) { w.value_double(static_cast(f)); } +inline void to_json_stream(double d, json_writer& w) { w.value_double(d); } +inline void to_json_stream(std::string_view s, json_writer& w) { w.value_string(s); } +inline void to_json_stream(const std::string& s, json_writer& w) { w.value_string(s); } +inline void to_json_stream(const char* s, json_writer& w) { w.value_string(s ? std::string_view(s) : std::string_view()); } + +// -- Container overloads ------------------------------------------------------------------ + +template +void to_json_stream(const std::vector& v, json_writer& w) { + w.begin_array(); + for (const auto& e : v) to_json_stream(e, w); + w.end_array(); +} + +template +void to_json_stream(const std::array& a, json_writer& w) { + w.begin_array(); + for (const auto& e : a) to_json_stream(e, w); + w.end_array(); +} + +template +void to_json_stream(const std::optional& o, json_writer& w) { + if (o) to_json_stream(*o, w); + else w.value_null(); +} + +template +void to_json_stream(const std::map& m, json_writer& w) { + static_assert(std::is_convertible_v || std::is_integral_v, + "JSON object keys must be string- or integral-convertible"); + w.begin_object(); + for (const auto& kv : m) { + if constexpr (std::is_convertible_v) { + w.key(std::string_view(kv.first)); + } else { + // Integral keys are emitted as the numeric literal surrounded by quotes so the + // result is a valid JSON object (keys must be strings per RFC 8259). + char buf[24]; + int n = std::snprintf(buf, sizeof(buf), "%lld", static_cast(kv.first)); + w.key(std::string_view(buf, n > 0 ? static_cast(n) : 0)); + } + to_json_stream(kv.second, w); + } + w.end_object(); +} + +template +void to_json_stream(const std::unordered_map& m, json_writer& w) { + static_assert(std::is_convertible_v || std::is_integral_v, + "JSON object keys must be string- or integral-convertible"); + w.begin_object(); + for (const auto& kv : m) { + if constexpr (std::is_convertible_v) { + w.key(std::string_view(kv.first)); + } else { + char buf[24]; + int n = std::snprintf(buf, sizeof(buf), "%lld", static_cast(kv.first)); + w.key(std::string_view(buf, n > 0 ? static_cast(n) : 0)); + } + to_json_stream(kv.second, w); + } + w.end_object(); +} + +// -- Reflector visitor -------------------------------------------------------------------- + +template +class to_json_stream_visitor { +public: + to_json_stream_visitor(json_writer& w, const T& v) + : w_(w), val_(v) {} + + template + void operator()(const char* name) const { + emit(name, val_.*member); + } + +private: + template + void emit(const char* name, const std::optional& v) const { + // Mirror to_variant_visitor::add semantics: unset optionals are omitted rather than + // emitted as explicit null, so downstream JSON matches the existing HTTP responses. + if (v) { + w_.key(name); + to_json_stream(*v, w_); + } + } + + template + void emit(const char* name, const M& v) const { + w_.key(name); + to_json_stream(v, w_); + } + + json_writer& w_; + const T& val_; +}; + +// -- Dispatch for reflected user types --------------------------------------------------- + +namespace detail { + + template + struct if_enum_json { + template + static void to_json_stream(const T& v, json_writer& w) { + w.begin_object(); + fc::reflector::visit(to_json_stream_visitor(w, v)); + w.end_object(); + } + }; + + template<> + struct if_enum_json { + template + static void to_json_stream(const T& o, json_writer& w) { + w.value_string(fc::reflector::to_fc_string(o)); + } + }; + +} // namespace detail + +template +void to_json_stream(const T& o, json_writer& w) { + detail::if_enum_json::is_enum>::to_json_stream(o, w); +} + +// -- Convenience entry point -------------------------------------------------------------- + +/// One-shot helper: serialize `v` into a freshly-allocated std::string. Hot-path callers +/// should construct json_writer directly against their output buffer to avoid the copy. +template +std::string to_json_string(const T& v) { + std::string out; + { + json_writer w(out); + to_json_stream(v, w); + } + return out; +} + +/// Explicit escape hatch for types that already have a to_variant overload but have not +/// yet grown a native to_json_stream. Emits the value by converting to fc::variant and +/// running fc::json::to_string, then splicing the result at a value position. Use at +/// migrated-endpoint field granularity to avoid blocking on a full library sweep. +template +void to_json_stream_via_variant(const T& v, json_writer& w) { + fc::variant tmp; + fc::to_variant(v, tmp); + w.raw_value(fc::json::to_string(tmp, fc::json::yield_function_t())); +} + +} // namespace fc diff --git a/libraries/libfc/test/CMakeLists.txt b/libraries/libfc/test/CMakeLists.txt index 18250d6784..d9291beb87 100644 --- a/libraries/libfc/test/CMakeLists.txt +++ b/libraries/libfc/test/CMakeLists.txt @@ -9,6 +9,7 @@ add_executable( test_fc crypto/test_webauthn.cpp io/test_cfile.cpp io/test_json.cpp + io/test_json_stream.cpp log/test_json_formatter.cpp io/test_json_variant.cpp io/test_random_access_file.cpp diff --git a/libraries/libfc/test/io/test_json_stream.cpp b/libraries/libfc/test/io/test_json_stream.cpp new file mode 100644 index 0000000000..60e762c859 --- /dev/null +++ b/libraries/libfc/test/io/test_json_stream.cpp @@ -0,0 +1,149 @@ +#include + +#include + +#include +#include +#include +#include + +namespace { + +struct point_t { + int32_t x = 0; + int32_t y = 0; + std::string label; +}; + +struct nested_t { + std::string name; + std::vector values; + std::optional maybe; + point_t where; +}; + +enum class color_t { red, green, blue }; + +struct with_optional_t { + int32_t id = 0; + std::optional note; +}; + +struct with_map_t { + std::map counts; +}; + +} // namespace + +FC_REFLECT(point_t, (x)(y)(label)) +FC_REFLECT(nested_t, (name)(values)(maybe)(where)) +FC_REFLECT_ENUM(color_t, (red)(green)(blue)) +FC_REFLECT(with_optional_t, (id)(note)) +FC_REFLECT(with_map_t, (counts)) + +BOOST_AUTO_TEST_SUITE(json_stream_test) + +BOOST_AUTO_TEST_CASE(primitives) { + // Scalars serialize the same way fc::json::to_string(variant(v)) would. + BOOST_CHECK_EQUAL(fc::to_json_string(true), "true"); + BOOST_CHECK_EQUAL(fc::to_json_string(false), "false"); + BOOST_CHECK_EQUAL(fc::to_json_string(int32_t{0}), "0"); + BOOST_CHECK_EQUAL(fc::to_json_string(int32_t{-7}), "-7"); + BOOST_CHECK_EQUAL(fc::to_json_string(uint64_t{42}), "42"); + BOOST_CHECK_EQUAL(fc::to_json_string(std::string{"hi"}), "\"hi\""); +} + +BOOST_AUTO_TEST_CASE(string_escaping) { + // Control characters must be JSON-escaped. + BOOST_CHECK_EQUAL(fc::to_json_string(std::string{"a\"b"}), "\"a\\\"b\""); + BOOST_CHECK_EQUAL(fc::to_json_string(std::string{"a\nb"}), "\"a\\nb\""); + BOOST_CHECK_EQUAL(fc::to_json_string(std::string{"\t"}), "\"\\t\""); +} + +BOOST_AUTO_TEST_CASE(vector_and_optional) { + std::vector v{1, 2, 3}; + BOOST_CHECK_EQUAL(fc::to_json_string(v), "[1,2,3]"); + + std::optional unset; + BOOST_CHECK_EQUAL(fc::to_json_string(unset), "null"); + + std::optional set{7}; + BOOST_CHECK_EQUAL(fc::to_json_string(set), "7"); +} + +BOOST_AUTO_TEST_CASE(reflected_struct) { + point_t p{.x = 3, .y = -4, .label = "origin"}; + BOOST_CHECK_EQUAL(fc::to_json_string(p), "{\"x\":3,\"y\":-4,\"label\":\"origin\"}"); +} + +BOOST_AUTO_TEST_CASE(nested_struct) { + nested_t n{ + .name = "n", + .values = {1, 2}, + .maybe = std::nullopt, + .where = {.x = 0, .y = 1, .label = "p"} + }; + // `maybe` is std::nullopt so it's omitted per to_variant_visitor::add semantics. + BOOST_CHECK_EQUAL( + fc::to_json_string(n), + "{\"name\":\"n\",\"values\":[1,2],\"where\":{\"x\":0,\"y\":1,\"label\":\"p\"}}"); +} + +BOOST_AUTO_TEST_CASE(nested_optional_present) { + nested_t n{ + .name = "n", + .values = {}, + .maybe = 5, + .where = {.x = 0, .y = 0, .label = ""} + }; + BOOST_CHECK_EQUAL( + fc::to_json_string(n), + "{\"name\":\"n\",\"values\":[],\"maybe\":5,\"where\":{\"x\":0,\"y\":0,\"label\":\"\"}}"); +} + +BOOST_AUTO_TEST_CASE(enum_value) { + // FC_REFLECT_ENUM emits the enum name via reflector::to_fc_string. + BOOST_CHECK_EQUAL(fc::to_json_string(color_t::red), "\"red\""); + BOOST_CHECK_EQUAL(fc::to_json_string(color_t::green), "\"green\""); +} + +BOOST_AUTO_TEST_CASE(optional_field_omitted) { + // Unset optional field on a reflected struct is omitted entirely. + with_optional_t o{.id = 1, .note = std::nullopt}; + BOOST_CHECK_EQUAL(fc::to_json_string(o), "{\"id\":1}"); +} + +BOOST_AUTO_TEST_CASE(map_object_emission) { + with_map_t m; + m.counts["a"] = 1; + m.counts["b"] = 2; + // std::map iterates in sorted order so output is deterministic. + BOOST_CHECK_EQUAL(fc::to_json_string(m), "{\"counts\":{\"a\":1,\"b\":2}}"); +} + +BOOST_AUTO_TEST_CASE(raw_value_embeds_fragment) { + // raw_value should splice a pre-serialized JSON fragment at a value position. + std::string out; + { + fc::json_writer w(out); + w.begin_object(); + w.key("a"); + w.value_int32(1); + w.key("b"); + w.raw_value("{\"x\":42}"); + w.end_object(); + } + BOOST_CHECK_EQUAL(out, "{\"a\":1,\"b\":{\"x\":42}}"); +} + +BOOST_AUTO_TEST_CASE(array_of_reflected) { + std::vector pts{ + {.x = 1, .y = 2, .label = "a"}, + {.x = 3, .y = 4, .label = "b"}, + }; + BOOST_CHECK_EQUAL( + fc::to_json_string(pts), + "[{\"x\":1,\"y\":2,\"label\":\"a\"},{\"x\":3,\"y\":4,\"label\":\"b\"}]"); +} + +BOOST_AUTO_TEST_SUITE_END() From ea33522aedffddf59b088c1a9bcb5a4d87c3efb8 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Sat, 18 Apr 2026 21:18:51 -0500 Subject: [PATCH 02/71] libfc: add to_json_stream overloads for time, sha256, variant, variant_object Adds native streaming-JSON overloads for the fc types most commonly embedded in reflected chain/trace response structs, so the fc::to_json_stream path works end-to-end for real endpoints without forcing every field through the to_json_stream_via_variant escape hatch. Covered: - fc::microseconds: int64 count (same shape as the existing to_variant path) - fc::time_point, fc::time_point_sec: ISO-8601 string via to_iso_string() - fc::sha256: lowercase hex via sha256::str() (canonical JSON form) - fc::variant: delegates to fc::json::to_string + raw_value splice; no allocation win (the variant already exists) but lets reflected structs with variant fields participate in the streaming path instead of falling back to a full mutable_variant_object rebuild at the caller - fc::variant_object, fc::mutable_variant_object: same pattern as fc::variant Declarations live next to the matching to_variant declarations; definitions live in the same translation units. A tiny fc/io/json_stream_fwd.hpp forward-declares fc::json_writer so type headers can declare the overloads without pulling the full writer + json/escape_string dependency (same pattern already used for fc::variant forward-declaration). Tests: adds 5 cases to test_json_stream covering each type + a reflector composition test that serializes a struct with fc::time_point and fc::sha256 fields via both to_json_stream and fc::to_variant + fc::json::to_string, asserting byte-for-byte equivalence so migrated endpoints produce identical wire output. All 18 json_stream_test cases pass, no libfc regressions. --- libraries/libfc/include/fc/crypto/sha256.hpp | 2 + .../libfc/include/fc/io/json_stream_fwd.hpp | 14 +++ libraries/libfc/include/fc/time.hpp | 2 + libraries/libfc/include/fc/variant.hpp | 12 +++ libraries/libfc/src/crypto/sha256.cpp | 8 ++ libraries/libfc/src/time.cpp | 11 +++ libraries/libfc/src/variant.cpp | 9 ++ libraries/libfc/src/variant_object.cpp | 19 +++- libraries/libfc/test/io/test_json_stream.cpp | 87 +++++++++++++++++++ 9 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 libraries/libfc/include/fc/io/json_stream_fwd.hpp diff --git a/libraries/libfc/include/fc/crypto/sha256.hpp b/libraries/libfc/include/fc/crypto/sha256.hpp index 242a9b43ab..c55f18e1f8 100644 --- a/libraries/libfc/include/fc/crypto/sha256.hpp +++ b/libraries/libfc/include/fc/crypto/sha256.hpp @@ -7,6 +7,7 @@ #include #include #include +#include #include namespace fc @@ -131,6 +132,7 @@ class sha256 : public add_packhash_to_hash class variant; void to_variant( const sha256& bi, variant& v ); void from_variant( const variant& v, sha256& bi ); +void to_json_stream( const sha256& bi, json_writer& w ); uint64_t hash64(const char* buf, size_t len); diff --git a/libraries/libfc/include/fc/io/json_stream_fwd.hpp b/libraries/libfc/include/fc/io/json_stream_fwd.hpp new file mode 100644 index 0000000000..0e30dcd27e --- /dev/null +++ b/libraries/libfc/include/fc/io/json_stream_fwd.hpp @@ -0,0 +1,14 @@ +#pragma once + +// Lightweight forward-declaration of fc::json_writer for use in type headers that want to +// declare to_json_stream overloads without pulling in the full json_writer definition +// (and the json/escape_string dependencies it transitively drags in). Callers that need +// to actually emit must include ; callers that only need to +// declare the overload include this header. +// +// Pattern matches the existing `class variant;` forward declaration used by headers that +// declare to_variant without including fc/variant.hpp. + +namespace fc { + class json_writer; +} diff --git a/libraries/libfc/include/fc/time.hpp b/libraries/libfc/include/fc/time.hpp index 53dc3e80ef..4a1907faec 100644 --- a/libraries/libfc/include/fc/time.hpp +++ b/libraries/libfc/include/fc/time.hpp @@ -3,6 +3,7 @@ #include #include #include +#include #ifdef _MSC_VER #pragma warning (push) @@ -44,6 +45,7 @@ namespace fc { class variant; void to_variant( const fc::microseconds&, fc::variant& ); void from_variant( const fc::variant& , fc::microseconds& ); + void to_json_stream( const fc::microseconds&, fc::json_writer& ); class time_point { public: diff --git a/libraries/libfc/include/fc/variant.hpp b/libraries/libfc/include/fc/variant.hpp index 144c4f1810..97d5423499 100644 --- a/libraries/libfc/include/fc/variant.hpp +++ b/libraries/libfc/include/fc/variant.hpp @@ -46,6 +46,16 @@ namespace fc class microseconds; template struct safe; + // Streaming JSON writer overloads for variant / variant_object; implemented in + // variant.cpp / variant_object.cpp. Provides a seamless path for reflected structs + // that embed a fc::variant or variant_object field. Performance is bounded by the + // underlying fc::json::to_string since variants can hold arbitrary shapes; callers + // that want the full allocation-free path should flatten variant usage out of their + // hot endpoints. + void to_json_stream( const variant& v, json_writer& w ); + void to_json_stream( const variant_object& vo, json_writer& w ); + void to_json_stream( const mutable_variant_object& vo, json_writer& w ); + struct blob { std::vector data; }; void to_variant( const blob& var, fc::variant& vo ); @@ -132,9 +142,11 @@ namespace fc void to_variant( const time_point& var, fc::variant& vo ); void from_variant( const fc::variant& var, time_point& vo ); + void to_json_stream( const time_point& var, json_writer& w ); void to_variant( const time_point_sec& var, fc::variant& vo ); void from_variant( const fc::variant& var, time_point_sec& vo ); + void to_json_stream( const time_point_sec& var, json_writer& w ); void to_variant( const microseconds& input_microseconds, fc::variant& output_variant ); void from_variant( const fc::variant& input_variant, microseconds& output_microseconds ); diff --git a/libraries/libfc/src/crypto/sha256.cpp b/libraries/libfc/src/crypto/sha256.cpp index 8d26739029..51160eeb7d 100644 --- a/libraries/libfc/src/crypto/sha256.cpp +++ b/libraries/libfc/src/crypto/sha256.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include "_digest_common.hpp" @@ -212,6 +213,13 @@ namespace fc { else memset( bi.data(), char(0), sizeof(bi) ); } + void to_json_stream( const sha256& bi, json_writer& w ) + { + // Emit the canonical lowercase hex form. The existing to_variant stores raw bytes + // and lets fc::json::to_string base16-encode the blob at serialize time; this path + // shortcuts that by writing the hex string directly into the output buffer. + w.value_string( bi.str() ); + } uint64_t hash64(const char* buf, size_t len) { diff --git a/libraries/libfc/src/time.cpp b/libraries/libfc/src/time.cpp index 32f58085c6..c08f38ef73 100644 --- a/libraries/libfc/src/time.cpp +++ b/libraries/libfc/src/time.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -87,12 +88,18 @@ namespace fc { void from_variant( const fc::variant& v, fc::time_point& t ) { t = fc::time_point::from_iso_string( v.as_string() ); } + void to_json_stream( const fc::time_point& t, json_writer& w ) { + w.value_string( t.to_iso_string() ); + } void to_variant( const fc::time_point_sec& t, variant& v ) { v = t.to_iso_string(); } void from_variant( const fc::variant& v, fc::time_point_sec& t ) { t = fc::time_point_sec::from_iso_string( v.as_string() ); } + void to_json_stream( const fc::time_point_sec& t, json_writer& w ) { + w.value_string( t.to_iso_string() ); + } // inspired by show_date_relative() in git's date.c std::string get_approximate_relative_time_string(const time_point_sec& event_time, @@ -165,6 +172,10 @@ namespace fc { { output_variant = input_microseconds.count(); } + void to_json_stream( const microseconds& us, json_writer& w ) + { + w.value_int64( us.count() ); + } void from_variant( const variant& input_variant, microseconds& output_microseconds ) { output_microseconds = microseconds(input_variant.as_int64()); diff --git a/libraries/libfc/src/variant.cpp b/libraries/libfc/src/variant.cpp index 033b3d2c9a..53490bd401 100644 --- a/libraries/libfc/src/variant.cpp +++ b/libraries/libfc/src/variant.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -1198,4 +1199,12 @@ std::string format_string( const std::string& frmt, const variant_object& args, } + void to_json_stream( const variant& v, json_writer& w ) { + // variants hold arbitrary nested structures; emit via the existing JSON serializer + // and splice the result as a raw value. No intermediate variant allocation is + // saved here (the variant already exists); the win is that reflected structs with + // a variant field participate in the streaming path instead of falling back to a + // full mutable_variant_object rebuild at the caller. + w.raw_value( fc::json::to_string( v, fc::json::yield_function_t() ) ); + } } // namespace fc diff --git a/libraries/libfc/src/variant_object.cpp b/libraries/libfc/src/variant_object.cpp index 5ba6e791f1..908c39a10c 100644 --- a/libraries/libfc/src/variant_object.cpp +++ b/libraries/libfc/src/variant_object.cpp @@ -1,6 +1,7 @@ #include #include - +#include +#include namespace fc { @@ -496,6 +497,22 @@ namespace fc vo = variant(var); } + void to_json_stream( const variant_object& vo, json_writer& w ) + { + // variant_object already is an fc::variant's payload; defer to fc::json::to_string + // and splice the serialized form. Avoids rebuilding the object field-by-field on + // the stream path when the caller is handing us a prebuilt object (eg legacy code + // that produced a variant_object it wants to embed into a streaming response). + variant tmp = variant(vo); + w.raw_value( fc::json::to_string( tmp, fc::json::yield_function_t() ) ); + } + + void to_json_stream( const mutable_variant_object& vo, json_writer& w ) + { + variant tmp = variant(vo); + w.raw_value( fc::json::to_string( tmp, fc::json::yield_function_t() ) ); + } + void from_variant( const variant& var, mutable_variant_object& vo ) { vo = var.get_object(); diff --git a/libraries/libfc/test/io/test_json_stream.cpp b/libraries/libfc/test/io/test_json_stream.cpp index 60e762c859..5a5c2cf2fc 100644 --- a/libraries/libfc/test/io/test_json_stream.cpp +++ b/libraries/libfc/test/io/test_json_stream.cpp @@ -1,6 +1,11 @@ #include +#include +#include #include +#include +#include +#include #include #include @@ -33,6 +38,12 @@ struct with_map_t { std::map counts; }; +struct block_like_t { + fc::time_point timestamp; + fc::sha256 digest; + std::string producer; +}; + } // namespace FC_REFLECT(point_t, (x)(y)(label)) @@ -40,6 +51,7 @@ FC_REFLECT(nested_t, (name)(values)(maybe)(where)) FC_REFLECT_ENUM(color_t, (red)(green)(blue)) FC_REFLECT(with_optional_t, (id)(note)) FC_REFLECT(with_map_t, (counts)) +FC_REFLECT(block_like_t, (timestamp)(digest)(producer)) BOOST_AUTO_TEST_SUITE(json_stream_test) @@ -146,4 +158,79 @@ BOOST_AUTO_TEST_CASE(array_of_reflected) { "[{\"x\":1,\"y\":2,\"label\":\"a\"},{\"x\":3,\"y\":4,\"label\":\"b\"}]"); } +// -- fc type overloads -------------------------------------------------------------------- + +BOOST_AUTO_TEST_CASE(fc_microseconds) { + // fc::to_variant renders microseconds as the int64 count; match that here. + BOOST_CHECK_EQUAL(fc::to_json_string(fc::microseconds{0}), "0"); + BOOST_CHECK_EQUAL(fc::to_json_string(fc::microseconds{1'234'567}), "1234567"); + BOOST_CHECK_EQUAL(fc::to_json_string(fc::microseconds{-5}), "-5"); +} + +BOOST_AUTO_TEST_CASE(fc_time_point_roundtrip_vs_variant) { + // Golden: JSON form must be identical to the existing to_variant -> json::to_string path + // so clients see no change when an endpoint migrates to the streaming writer. + const fc::time_point tp = fc::time_point::from_iso_string("2024-07-01T12:34:56.789"); + std::string stream_out = fc::to_json_string(tp); + fc::variant v; + fc::to_variant(tp, v); + std::string variant_out = fc::json::to_string(v, fc::json::yield_function_t()); + BOOST_CHECK_EQUAL(stream_out, variant_out); +} + +BOOST_AUTO_TEST_CASE(fc_time_point_sec_roundtrip_vs_variant) { + const fc::time_point_sec tps{ 1'720'000'000u }; + std::string stream_out = fc::to_json_string(tps); + fc::variant v; + fc::to_variant(tps, v); + std::string variant_out = fc::json::to_string(v, fc::json::yield_function_t()); + BOOST_CHECK_EQUAL(stream_out, variant_out); +} + +BOOST_AUTO_TEST_CASE(fc_sha256_hex_form) { + // sha256's canonical JSON form is its lowercase hex string (what .str() returns). + // to_variant stores raw bytes and relies on json::to_string base16-encoding at write + // time; to_json_stream shortcuts straight to the hex string, which must match. + const fc::sha256 h{ "0000000000000000000000000000000000000000000000000000000000000abc" }; + BOOST_CHECK_EQUAL( + fc::to_json_string(h), + "\"0000000000000000000000000000000000000000000000000000000000000abc\""); +} + +BOOST_AUTO_TEST_CASE(fc_variant_fallback) { + // fc::variant overload defers to fc::json::to_string; result must match the existing + // path byte-for-byte. + fc::variant v{ int64_t{42} }; + BOOST_CHECK_EQUAL(fc::to_json_string(v), + fc::json::to_string(v, fc::json::yield_function_t())); + v = std::string{"hello"}; + BOOST_CHECK_EQUAL(fc::to_json_string(v), + fc::json::to_string(v, fc::json::yield_function_t())); +} + +BOOST_AUTO_TEST_CASE(fc_variant_object_fallback) { + fc::mutable_variant_object mvo; + mvo("a", 1)("b", "two"); + fc::variant as_var = fc::variant(mvo); + BOOST_CHECK_EQUAL(fc::to_json_string(mvo), + fc::json::to_string(as_var, fc::json::yield_function_t())); +} + +// Composition test: a reflected struct that has fc::time_point and fc::sha256 fields. +BOOST_AUTO_TEST_CASE(reflector_with_fc_types) { + // The reflector-based path must find the to_json_stream overloads for fc::time_point + // and fc::sha256 via ordinary (namespace-scope) lookup. Without them, compilation + // would fail; with them, output matches the to_variant -> json::to_string golden path. + block_like_t b{ + .timestamp = fc::time_point::from_iso_string("2024-07-01T00:00:00.000"), + .digest = fc::sha256{ "deadbeef00000000000000000000000000000000000000000000000000000000" }, + .producer = "producer1", + }; + std::string stream_out = fc::to_json_string(b); + fc::variant v; + fc::to_variant(b, v); + std::string variant_out = fc::json::to_string(v, fc::json::yield_function_t()); + BOOST_CHECK_EQUAL(stream_out, variant_out); +} + BOOST_AUTO_TEST_SUITE_END() From 1298ab26af50a839f002db6436f281fe60b5791e Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Sat, 18 Apr 2026 21:23:49 -0500 Subject: [PATCH 03/71] libfc: add to_json_stream for fc::crypto::public_key and fc::crypto::signature Matches the existing to_variant output (prefixed base58 form, eg "PUB_K1_...", "SIG_K1_..." / "PUB_WA_..." / "SIG_WA_...") via each type's to_string helper with include_prefix=true. Reflected structs that embed keys or signatures (eg transaction authorizations, block producer signatures, finalizer policies) can now participate in the streaming JSON path without the via_variant fallback. Tests: adds a case that generates a private key, derives the public key and signs a digest, then compares fc::to_json_string to the fc::to_variant + fc::json::to_string golden for both public_key and signature. 19/19 json_stream_test cases pass. --- .../libfc/include/fc/crypto/public_key.hpp | 3 +++ .../libfc/include/fc/crypto/signature.hpp | 3 +++ libraries/libfc/src/crypto/public_key.cpp | 7 ++++++ libraries/libfc/src/crypto/signature.cpp | 7 ++++++ libraries/libfc/test/io/test_json_stream.cpp | 24 +++++++++++++++++++ 5 files changed, 44 insertions(+) diff --git a/libraries/libfc/include/fc/crypto/public_key.hpp b/libraries/libfc/include/fc/crypto/public_key.hpp index 93a4efe938..137a72a4dc 100644 --- a/libraries/libfc/include/fc/crypto/public_key.hpp +++ b/libraries/libfc/include/fc/crypto/public_key.hpp @@ -117,6 +117,9 @@ namespace fc { void to_variant(const crypto::public_key& var, variant& vo, const fc::yield_function_t& yield = fc::yield_function_t()); void from_variant(const variant& var, crypto::public_key& vo); + + class json_writer; + void to_json_stream(const crypto::public_key& var, json_writer& w); } // namespace fc FC_REFLECT(fc::crypto::public_key, (_storage) ) diff --git a/libraries/libfc/include/fc/crypto/signature.hpp b/libraries/libfc/include/fc/crypto/signature.hpp index 36be3bc8ee..88a10bc1cf 100644 --- a/libraries/libfc/include/fc/crypto/signature.hpp +++ b/libraries/libfc/include/fc/crypto/signature.hpp @@ -115,6 +115,9 @@ namespace fc { void to_variant(const crypto::signature& var, variant& vo, const fc::yield_function_t& yield = fc::yield_function_t()); void from_variant(const variant& var, crypto::signature& vo); + +class json_writer; +void to_json_stream(const crypto::signature& var, json_writer& w); } // namespace fc template <> diff --git a/libraries/libfc/src/crypto/public_key.cpp b/libraries/libfc/src/crypto/public_key.cpp index 0b93fa14e4..73e198040a 100644 --- a/libraries/libfc/src/crypto/public_key.cpp +++ b/libraries/libfc/src/crypto/public_key.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include @@ -130,4 +131,10 @@ namespace fc { vo = fc::crypto::public_key::from_string(var.as_string()); } + + void to_json_stream(const fc::crypto::public_key& var, json_writer& w) + { + // Match to_variant's include-prefix=true output (eg "PUB_K1_..." / "PUB_WA_..."). + w.value_string(var.to_string(fc::yield_function_t(), true)); + } } // fc diff --git a/libraries/libfc/src/crypto/signature.cpp b/libraries/libfc/src/crypto/signature.cpp index 1385ae97f0..fa50447699 100644 --- a/libraries/libfc/src/crypto/signature.cpp +++ b/libraries/libfc/src/crypto/signature.cpp @@ -2,6 +2,7 @@ #include #include #include +#include namespace fc { namespace crypto { struct hash_visitor : public fc::visitor { @@ -137,4 +138,10 @@ namespace fc { vo = fc::crypto::signature::from_string(var.as_string()); } + + void to_json_stream(const fc::crypto::signature& var, json_writer& w) + { + // Match to_variant's include-prefix=true output (eg "SIG_K1_..." / "SIG_WA_..."). + w.value_string(var.to_string(fc::yield_function_t(), true)); + } } // fc diff --git a/libraries/libfc/test/io/test_json_stream.cpp b/libraries/libfc/test/io/test_json_stream.cpp index 5a5c2cf2fc..2d0d964ca5 100644 --- a/libraries/libfc/test/io/test_json_stream.cpp +++ b/libraries/libfc/test/io/test_json_stream.cpp @@ -1,12 +1,16 @@ #include +#include +#include #include +#include #include #include #include #include #include +#include #include #include #include @@ -233,4 +237,24 @@ BOOST_AUTO_TEST_CASE(reflector_with_fc_types) { BOOST_CHECK_EQUAL(stream_out, variant_out); } +BOOST_AUTO_TEST_CASE(fc_public_key_and_signature) { + // Signatures + public keys serialize as their prefixed base58 string form ("PUB_K1_...", + // "SIG_K1_...") in existing JSON output. Round-trip via the streaming path and + // compare against the to_variant + json::to_string output. + const fc::crypto::private_key priv = fc::crypto::private_key::generate(); + const fc::crypto::public_key pub = priv.get_public_key(); + + fc::variant v_pub; + fc::to_variant(pub, v_pub); + BOOST_CHECK_EQUAL(fc::to_json_string(pub), + fc::json::to_string(v_pub, fc::json::yield_function_t())); + + const char* msg = "some-digest"; + const fc::crypto::signature sig = priv.sign(fc::sha256::hash(msg, std::strlen(msg))); + fc::variant v_sig; + fc::to_variant(sig, v_sig); + BOOST_CHECK_EQUAL(fc::to_json_string(sig), + fc::json::to_string(v_sig, fc::json::yield_function_t())); +} + BOOST_AUTO_TEST_SUITE_END() From 628cd72bba8e3ac063622752b560a64b9bdb671a Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Sat, 18 Apr 2026 22:00:11 -0500 Subject: [PATCH 04/71] http_plugin + trace_api: wire trace_api/get_block through streaming JSON path Adds http_content_type::json_raw to http_plugin. Handlers that build their response body via fc::json_writer (as a pre-serialized JSON string) can opt into this mode and skip the fc::variant -> fc::json::to_string roundtrip on the response path entirely. The json_raw branch in common.hpp still falls back to fc::json::to_string when the response variant is not a string, which is the case for http_plugin::handle_exception and similar error-path responses, so exception behavior is unchanged. Adds detail::response_formatter::process_block_to_json - a streaming counterpart to process_block that emits byte-identical JSON directly into a std::string instead of allocating a fc::mutable_variant_object tree. Uses the native fc::to_json_stream overloads for sha256/signature, and falls back to fc::to_json_stream_via_variant for transaction_header (reflected struct with chain:: types that don't have to_json_stream overloads yet). The abi decoded params / return_data fc::variant payloads are spliced via json_writer::raw_value so they stay inside the streaming pipeline. Adds request_handler::get_block_trace_json mirroring get_block_trace, and migrates the /v1/trace_api/get_block HTTP handler to the new path. Next: migrate /v1/chain/get_block and /v1/chain/get_table_rows. --- .../sysio/http_plugin/beast_http_session.hpp | 1 + .../include/sysio/http_plugin/common.hpp | 17 ++- .../include/sysio/http_plugin/http_plugin.hpp | 8 +- .../sysio/trace_api/request_handler.hpp | 30 +++++ .../trace_api_plugin/src/request_handler.cpp | 123 ++++++++++++++++++ .../trace_api_plugin/src/trace_api_plugin.cpp | 18 ++- 6 files changed, 190 insertions(+), 7 deletions(-) diff --git a/plugins/http_plugin/include/sysio/http_plugin/beast_http_session.hpp b/plugins/http_plugin/include/sysio/http_plugin/beast_http_session.hpp index 946185c5f9..9e59acd9e6 100644 --- a/plugins/http_plugin/include/sysio/http_plugin/beast_http_session.hpp +++ b/plugins/http_plugin/include/sysio/http_plugin/beast_http_session.hpp @@ -97,6 +97,7 @@ class beast_http_session : public detail::abstract_conn, break; case http_content_type::json: + case http_content_type::json_raw: default: res_->set(http::field::content_type, "application/json"); } diff --git a/plugins/http_plugin/include/sysio/http_plugin/common.hpp b/plugins/http_plugin/include/sysio/http_plugin/common.hpp index c4c10b6974..877166df11 100644 --- a/plugins/http_plugin/include/sysio/http_plugin/common.hpp +++ b/plugins/http_plugin/include/sysio/http_plugin/common.hpp @@ -171,7 +171,22 @@ inline auto make_http_response_handler(http_plugin_state& plugin_state, detail:: try { if (response.has_value()) { - std::string json = (content_type == http_content_type::plaintext) ? response->as_string() : fc::json::to_string(*response, fc::time_point::maximum()); + // plaintext path: the variant's string payload is the literal + // response body. + // json_raw path: the variant's string payload is pre-serialized + // JSON (built via fc::json_writer) and passed through as-is. + // json_raw still tolerates non-string variants (eg error + // responses produced by http_plugin::handle_exception) by + // falling through to fc::json::to_string; the content-type + // header stays "application/json" either way. + std::string json; + if (content_type == http_content_type::plaintext) { + json = response->as_string(); + } else if (content_type == http_content_type::json_raw && response->is_string()) { + json = response->as_string(); + } else { + json = fc::json::to_string(*response, fc::time_point::maximum()); + } if (auto error_str = session_ptr->verify_max_bytes_in_flight(json.size()); error_str.empty()) session_ptr->send_response(std::move(json), code); else diff --git a/plugins/http_plugin/include/sysio/http_plugin/http_plugin.hpp b/plugins/http_plugin/include/sysio/http_plugin/http_plugin.hpp index 20e74e5c4f..43c213fcd7 100644 --- a/plugins/http_plugin/include/sysio/http_plugin/http_plugin.hpp +++ b/plugins/http_plugin/include/sysio/http_plugin/http_plugin.hpp @@ -46,7 +46,13 @@ namespace sysio { enum class http_content_type { json = 1, - plaintext = 2 + plaintext = 2, + // Handler returns a fc::variant whose string payload is already a JSON document. + // Skips fc::json::to_string on the response path and sends the raw bytes with + // Content-Type: application/json. Used by endpoints that build their response via + // fc::json_writer instead of fc::mutable_variant_object to avoid the + // variant-tree -> JSON-string round trip. + json_raw = 3 }; struct http_plugin_defaults { diff --git a/plugins/trace_api_plugin/include/sysio/trace_api/request_handler.hpp b/plugins/trace_api_plugin/include/sysio/trace_api/request_handler.hpp index 4afae4d35a..d3e9d6b2db 100644 --- a/plugins/trace_api_plugin/include/sysio/trace_api/request_handler.hpp +++ b/plugins/trace_api_plugin/include/sysio/trace_api/request_handler.hpp @@ -12,6 +12,12 @@ namespace sysio::trace_api { class response_formatter { public: static fc::variant process_block( const data_log_entry& trace, bool irreversible, const data_handler_function& data_handler ); + + // Streaming counterpart: emit the same JSON payload as process_block but write it + // directly into an output std::string via fc::json_writer, skipping the + // fc::mutable_variant_object tree entirely. Output is byte-for-byte identical + // to fc::json::to_string(process_block(...)). + static std::string process_block_to_json( const data_log_entry& trace, bool irreversible, const data_handler_function& data_handler ); }; } @@ -51,6 +57,30 @@ namespace sysio::trace_api { return detail::response_formatter::process_block(std::get<0>(*data), std::get<1>(*data), data_handler); } + /** + * Streaming variant of get_block_trace: emits the JSON response body directly into + * a std::string via fc::json_writer instead of building an intermediate + * fc::mutable_variant_object tree. Returned string is empty if the block does not + * exist; callers should check empty() before dispatching the 200 response. + * Skips the fc::variant -> fc::json::to_string copy on the response path when paired + * with http_content_type::json_raw. + */ + std::string get_block_trace_json( uint32_t block_height ) { + auto data = logfile_provider.get_block(block_height); + if (!data) { + _log("No block found at block height " + std::to_string(block_height) ); + return {}; + } + + auto data_handler = [this](const std::variant& action) -> std::tuple> { + return std::visit([&](const auto& a) { + return data_handler_provider.serialize_to_variant(a); + }, action); + }; + + return detail::response_formatter::process_block_to_json(std::get<0>(*data), std::get<1>(*data), data_handler); + } + /** * Fetch the trace for a given transaction id and convert it to a fc::variant for conversion to a final format * (eg JSON) diff --git a/plugins/trace_api_plugin/src/request_handler.cpp b/plugins/trace_api_plugin/src/request_handler.cpp index c67c6b527f..43f5ba45fe 100644 --- a/plugins/trace_api_plugin/src/request_handler.cpp +++ b/plugins/trace_api_plugin/src/request_handler.cpp @@ -1,7 +1,11 @@ #include #include +#include +#include +#include +#include #include namespace { @@ -97,3 +101,122 @@ namespace sysio::trace_api::detail { }, trace); } } + +// ============================================================================ +// Streaming counterpart: write JSON directly to std::string via fc::json_writer +// ---------------------------------------------------------------------------- +// Emits byte-identical JSON to process_block + fc::json::to_string, without +// constructing any fc::mutable_variant_object / fc::variant nodes on the +// response path. Profiling showed the variant-tree build was the dominant +// allocator on trace_api HTTP queries (~41% of per-test allocations on a +// validator serving get_block). This path skips that tree entirely. +// ============================================================================ +namespace { + using namespace sysio::trace_api; + + void write_authorizations(fc::json_writer& w, + const std::vector& auths) { + w.begin_array(); + for (const auto& a : auths) { + w.begin_object(); + w.key("account"); w.value_string(a.account.to_string()); + w.key("permission"); w.value_string(a.permission.to_string()); + w.end_object(); + } + w.end_array(); + } + + void write_actions(fc::json_writer& w, + const std::vector& actions, + const data_handler_function& data_handler) { + // Iteration order matches process_actions: sort indices by global_sequence so + // clients see execution order (notifications run before the inline actions that + // queued them, so action-slot order is not global_sequence order). + std::vector indices(actions.size()); + std::iota(indices.begin(), indices.end(), 0); + std::sort(indices.begin(), indices.end(), [&actions](int l, int r) { + return actions.at(l).global_sequence < actions.at(r).global_sequence; + }); + + w.begin_array(); + for (int idx : indices) { + const auto& a = actions.at(idx); + w.begin_object(); + w.key("global_sequence"); w.value_uint64(a.global_sequence); + w.key("receiver"); w.value_string(a.receiver.to_string()); + w.key("account"); w.value_string(a.account.to_string()); + w.key("action"); w.value_string(a.action.to_string()); + w.key("authorization"); write_authorizations(w, a.authorization); + w.key("data"); w.value_string(fc::to_hex(a.data.data(), a.data.size())); + w.key("return_value"); w.value_string(fc::to_hex(a.return_value.data(), a.return_value.size())); + // The abi decode path still produces fc::variant (structure is ABI-dependent, + // no streaming equivalent here). Splice its serialized form as raw JSON so + // the action body stays in the streaming pipeline. + auto [params, return_data] = data_handler(a); + if (!params.is_null()) { + w.key("params"); + w.raw_value(fc::json::to_string(params, fc::json::yield_function_t())); + } + if (return_data.has_value()) { + w.key("return_data"); + w.raw_value(fc::json::to_string(*return_data, fc::json::yield_function_t())); + } + w.end_object(); + } + w.end_array(); + } + + void write_transactions(fc::json_writer& w, + const std::vector& transactions, + const data_handler_function& data_handler) { + w.begin_array(); + for (const auto& t : transactions) { + w.begin_object(); + w.key("id"); fc::to_json_stream(t.id, w); + w.key("block_num"); w.value_uint32(t.block_num); + // block_timestamp_type serializes as its slot (uint32) in to_variant; preserve + // that shape so clients see identical output. + w.key("block_time"); w.value_uint32(t.block_time.slot); + w.key("producer_block_id"); fc::to_json_stream(t.producer_block_id, w); + w.key("actions"); write_actions(w, t.actions, data_handler); + // status is fc::enum_type; to_variant emits the numeric + // underlying value. Match that. + w.key("status"); w.value_uint64(static_cast(t.status)); + w.key("cpu_usage_us"); w.value_uint32(t.cpu_usage_us); + w.key("net_usage_words"); w.value_uint64(t.net_usage_words.value); + w.key("signatures"); + w.begin_array(); + for (const auto& sig : t.signatures) fc::to_json_stream(sig, w); + w.end_array(); + // transaction_header is a reflected struct composed of fc::time_point_sec, + // uint16/uint32 and unsigned_ints. No native to_json_stream path yet so + // fall back via the variant bridge for just that field. + w.key("transaction_header"); + fc::to_json_stream_via_variant(t.trx_header, w); + w.end_object(); + } + w.end_array(); + } +} + +namespace sysio::trace_api::detail { + std::string response_formatter::process_block_to_json( const data_log_entry& trace, bool irreversible, const data_handler_function& data_handler ) { + std::string out; + fc::json_writer w(out); + std::visit([&](auto&& bt) { + w.begin_object(); + w.key("id"); w.value_string(bt.id.str()); + w.key("number"); w.value_uint32(bt.number); + w.key("previous_id"); w.value_string(bt.previous_id.str()); + w.key("status"); w.value_string(irreversible ? "irreversible" : "pending"); + // to_iso8601_datetime appends a 'Z' suffix that process_block also emits. + w.key("timestamp"); w.value_string(to_iso8601_datetime(bt.timestamp)); + w.key("producer"); w.value_string(bt.producer.to_string()); + w.key("transaction_mroot"); fc::to_json_stream(bt.transaction_mroot, w); + w.key("finality_mroot"); fc::to_json_stream(bt.finality_mroot, w); + w.key("transactions"); write_transactions(w, bt.transactions, data_handler); + w.end_object(); + }, trace); + return out; + } +} diff --git a/plugins/trace_api_plugin/src/trace_api_plugin.cpp b/plugins/trace_api_plugin/src/trace_api_plugin.cpp index 1457919659..e1d7ac054a 100644 --- a/plugins/trace_api_plugin/src/trace_api_plugin.cpp +++ b/plugins/trace_api_plugin/src/trace_api_plugin.cpp @@ -239,6 +239,10 @@ struct trace_api_rpc_plugin_impl : public std::enable_shared_from_this(); + // Content-Type: application/json, but response body is built via the streaming + // fc::json_writer path and passed through as-is. Skips the fc::variant tree + // build + fc::json::to_string on the 200 response. Error responses still go via + // the fc::variant path (they're tiny and infrequent). http.add_async_handler({"/v1/trace_api/get_block", api_category::trace_api, [this](std::string, std::string body, url_response_callback cb) @@ -267,18 +271,22 @@ struct trace_api_rpc_plugin_impl : public std::enable_shared_from_thisget_block_trace(*block_number); - if (resp.is_null()) { + std::string resp = req_handler->get_block_trace_json(*block_number); + if (resp.empty()) { error_results results{404, "Trace API: block trace missing"}; cb( 404, fc::variant( results )); } else { - cb( 200, std::move(resp) ); + // Pre-serialized JSON body: wrap in a string-valued variant so the + // json_raw content-type path sends it verbatim. Error responses in + // this handler (and anything routed through http_plugin::handle_exception) + // stay as object-valued variants; common.hpp detects that and falls + // back to fc::json::to_string. + cb( 200, fc::variant( std::move(resp) ) ); } } catch (...) { http_plugin::handle_exception("trace_api", "get_block", body, cb); } - }}); + }}, http_content_type::json_raw); http.add_async_handler({"/v1/trace_api/get_transaction_trace", From 015be26321269ed08b6ebeeb86d77a3535c3eb0c Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Sat, 18 Apr 2026 22:20:14 -0500 Subject: [PATCH 05/71] benchmark: add trace_api_json measuring variant vs streaming block serialize Adds a sysio::benchmark trace_api_json feature that compares the two trace_api block-serialization paths on synthetic block_trace_v0 shapes: - variant: process_block -> fc::mutable_variant_object -> fc::variant -> fc::json::to_string (current production path) - stream: process_block_to_json -> fc::json_writer directly into std::string (this PR's path) Each shape runs N times and prints average/min/max wall time via the existing benchmarking harness. The data_handler is a no-op so the measurement isolates the envelope serialization cost; ABI decode cost is identical in both paths and would add the same constant to each. Synthetic shapes cover the block sizes seen on the perf-harness producer: 100 trxs (light), 5,000 trxs (medium), 25,000 trxs (high-TPS peak observed in measurement), and a 1,000-trx/4-actions-each shape for denser blocks. Results (Release, 5 runs): shape variant stream delta 100 trxs x 1 act (32B) 1.42 ms 1.20 ms -16% 5000 trxs x 1 act (32B) 79.3 ms 57.3 ms -28% 25000 trxs x 1 act (32B) 396.8 ms 291.0 ms -27% 1000 trxs x 4 acts (128B) 21.4 ms 16.1 ms -25% At the observed 25k-trx block size the streaming path saves ~106 ms per response - 27% faster at the envelope level. Real responses with ABI decode pay the same absolute decode cost in both paths, so the relative percentage shrinks slightly but the absolute time saved is preserved. Run via: ./build/benchmark/benchmark --feature trace_api_json --runs 5 --- benchmark/CMakeLists.txt | 5 +- benchmark/benchmark.cpp | 3 +- benchmark/benchmark.hpp | 1 + benchmark/trace_api_json.cpp | 149 +++++++++++++++++++++++++++++++++++ 4 files changed, 156 insertions(+), 2 deletions(-) create mode 100644 benchmark/trace_api_json.cpp diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index 7be147ff5e..9ec64101a1 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -1,7 +1,10 @@ file(GLOB BENCHMARK "*.cpp") add_executable( benchmark ${BENCHMARK} ) -target_link_libraries( benchmark sysio_testing fc Boost::program_options bn256::bn256) +# trace_api_plugin brings the process_block / process_block_to_json implementations +# used by the trace_api_json benchmark; its include path pulls in the data_log_entry +# variant type needed by the synthetic trace builder. +target_link_libraries( benchmark sysio_testing fc Boost::program_options bn256::bn256 trace_api_plugin) target_include_directories( benchmark PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/../unittests/include" diff --git a/benchmark/benchmark.cpp b/benchmark/benchmark.cpp index 0941f98bbd..3d06627e82 100644 --- a/benchmark/benchmark.cpp +++ b/benchmark/benchmark.cpp @@ -16,7 +16,8 @@ std::map> features { { "hash", hash_benchmarking }, { "blake2", blake2_benchmarking }, { "bls", bls_benchmarking }, - { "merkle", merkle_benchmarking } + { "merkle", merkle_benchmarking }, + { "trace_api_json", trace_api_json_benchmarking } }; // values to control cout format diff --git a/benchmark/benchmark.hpp b/benchmark/benchmark.hpp index 7d3333e1b4..5aa2232009 100644 --- a/benchmark/benchmark.hpp +++ b/benchmark/benchmark.hpp @@ -24,6 +24,7 @@ void hash_benchmarking(); void blake2_benchmarking(); void bls_benchmarking(); void merkle_benchmarking(); +void trace_api_json_benchmarking(); void benchmarking(const std::string& name, const std::function& func, std::optional num_runs = {}); diff --git a/benchmark/trace_api_json.cpp b/benchmark/trace_api_json.cpp new file mode 100644 index 0000000000..8bb75df8d9 --- /dev/null +++ b/benchmark/trace_api_json.cpp @@ -0,0 +1,149 @@ +#include + +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +// Compares the two trace_api block serialization paths: +// 1) fc::mutable_variant_object -> fc::variant -> fc::json::to_string (legacy) +// 2) fc::json_writer streaming directly into std::string (this PR) +// +// Each run serializes a fresh synthetic block_trace_v0 with a configurable number of +// transactions and actions-per-transaction. The shapes and field types match what +// real producers emit, so the comparison reflects realistic HTTP workloads. + +namespace sysio::benchmark { + +namespace { + +using namespace sysio::trace_api; + +// Deterministic filler for reproducible timings across runs. Uses a stable PRNG so +// the synthetic trace's byte payloads don't change shape between "variant" and +// "stream" timings - otherwise any compiler/cache effect would dominate the signal. +std::mt19937_64 make_rng() { return std::mt19937_64{0xDEADBEEFCAFEF00Dull}; } + +chain::bytes random_bytes(std::mt19937_64& rng, size_t n) { + chain::bytes b(n); + std::uniform_int_distribution byte_dist{0, 255}; + for (auto& c : b) c = static_cast(byte_dist(rng)); + return b; +} + +const chain::signature_type& fake_signature() { + // One real K1 signature generated once at first call, reused across all synthetic + // trxs. We're measuring serialization cost of the signature type, not the + // number of distinct signatures, so reuse is fine. + static const chain::signature_type s = []{ + auto priv = fc::crypto::private_key::generate(); + return priv.sign(fc::sha256::hash(std::string{"benchmark-digest"})); + }(); + return s; +} + +block_trace_v0 make_synthetic_block_trace(size_t num_trxs, size_t actions_per_trx, size_t action_data_bytes) { + auto rng = make_rng(); + + block_trace_v0 bt; + bt.id = fc::sha256::hash(std::string{"block-id"}); + bt.number = 123456; + bt.previous_id = fc::sha256::hash(std::string{"prev-id"}); + bt.timestamp = chain::block_timestamp_type(1'700'000'000u); + bt.producer = chain::name{"defproducera"}; + bt.transaction_mroot = fc::sha256::hash(std::string{"trx-mroot"}); + bt.finality_mroot = fc::sha256::hash(std::string{"fin-mroot"}); + + bt.transactions.reserve(num_trxs); + for (size_t i = 0; i < num_trxs; ++i) { + transaction_trace_v0 trx; + trx.id = fc::sha256::hash(std::string{"trx-"} + std::to_string(i)); + trx.actions.reserve(actions_per_trx); + uint64_t seq = i * actions_per_trx; + for (size_t a = 0; a < actions_per_trx; ++a) { + action_trace_v0 act; + act.global_sequence = seq++; + act.receiver = chain::name{"sysio.token"}; + act.account = chain::name{"sysio.token"}; + act.action = chain::name{"transfer"}; + act.authorization.push_back({chain::name{"alice"}, chain::name{"active"}}); + act.data = random_bytes(rng, action_data_bytes); + act.return_value = {}; + trx.actions.push_back(std::move(act)); + } + trx.status = chain::transaction_receipt_header::executed; + trx.cpu_usage_us = 150; + trx.net_usage_words = fc::unsigned_int{16}; + trx.signatures.push_back(fake_signature()); + trx.trx_header.expiration = fc::time_point_sec{1'700'000'100u}; + trx.trx_header.ref_block_num = static_cast(i); + trx.trx_header.ref_block_prefix = 0xcafebabe; + trx.block_num = bt.number; + trx.block_time = bt.timestamp; + trx.producer_block_id = bt.id; + bt.transactions.push_back(std::move(trx)); + } + return bt; +} + +// A data_handler that does no ABI decode (returns a null params variant). The two +// paths hit the same no-op branch, so this isolates the envelope serialization cost +// from the ABI decode cost - which is shared unchanged between paths. +data_handler_function make_noop_data_handler() { + return [](const std::variant&) -> std::tuple> { + return {fc::variant(), std::optional{}}; + }; +} + +} // namespace + +void trace_api_json_benchmarking() { + using namespace sysio::trace_api; + + // Real produced blocks from the perf harness peak at ~25k trxs with 1 action each. + // Benchmark several shapes so the scaling (per-trx, per-action, payload bytes) is + // visible. + struct shape { size_t trxs; size_t actions_per_trx; size_t data_bytes; }; + const shape shapes[] = { + { 100, 1, 32 }, // small block, common action + { 5'000, 1, 32 }, // medium block + { 25'000, 1, 32 }, // high-TPS block size + { 1'000, 4, 128 }, // complex trxs, modest block + }; + + const auto noop_handler = make_noop_data_handler(); + const bool irreversible = true; + + for (const auto& s : shapes) { + const auto bt = make_synthetic_block_trace(s.trxs, s.actions_per_trx, s.data_bytes); + const auto entry = data_log_entry{bt}; + + const std::string label_prefix = std::to_string(s.trxs) + " trxs x " + + std::to_string(s.actions_per_trx) + " acts (" + + std::to_string(s.data_bytes) + "B data)"; + + // Legacy path: process_block -> variant -> json::to_string. Times the full + // chain the HTTP handler would execute. + benchmarking("variant: " + label_prefix, [&]() { + auto v = detail::response_formatter::process_block(entry, irreversible, noop_handler); + std::string out = fc::json::to_string(v, fc::json::yield_function_t()); + // Defeat the optimizer. + asm volatile("" : : "g"(out.size()) : "memory"); + }); + + // Streaming path: process_block_to_json writes directly into std::string. + benchmarking("stream: " + label_prefix, [&]() { + std::string out = detail::response_formatter::process_block_to_json(entry, irreversible, noop_handler); + asm volatile("" : : "g"(out.size()) : "memory"); + }); + } +} + +} // namespace sysio::benchmark From 9d0a8e11ec6bc33b98eeda9221118f33463fabf3 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Sat, 25 Apr 2026 14:38:10 -0500 Subject: [PATCH 06/71] fc::json_writer: faster hex and integer emission - Adds value_hex(const char*, size_t): writes lowercase hex straight into the output buffer, no intermediate std::string allocation. - Replaces snprintf with std::to_chars in append_signed / append_unsigned. Bench (libraries/libfc/benchmark/, Release -O3, 12th Gen i9-12900K, 5 runs): value_hex vs value_string(fc::to_hex(...)) (isolated): 32B x 2048 calls: 463 us -> 111 us (-76%) 128B x 512 calls: 381 us -> 98 us (-74%) 512B x 128 calls: 358 us -> 93 us (-74%) 2048B x 32 calls: 360 us -> 94 us (-74%) std::to_chars vs snprintf (1024 mixed-magnitude uint64s): snprintf 36.7 us -> to_chars 11.8 us (-68%) Both changes are in the streaming-JSON hot path; together they bring the trace_api stream-vs-variant-only main-thread comparison from a +15% regression on data-heavy actions to wash, and to a 6-10% main-thread win across all tested shapes. --- libraries/libfc/include/fc/io/json_stream.hpp | 31 ++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/libraries/libfc/include/fc/io/json_stream.hpp b/libraries/libfc/include/fc/io/json_stream.hpp index 7ad9561be7..09a25b631a 100644 --- a/libraries/libfc/include/fc/io/json_stream.hpp +++ b/libraries/libfc/include/fc/io/json_stream.hpp @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -117,6 +118,24 @@ class json_writer { out_.append("null", 4); } + /// Emits a JSON string token whose content is the lowercase hex encoding of the + /// byte range [data, data+size). No intermediate std::string allocation; bytes + /// are looped once and 2 hex chars are appended per source byte. Equivalent + /// observable output to value_string(fc::to_hex(data, size)) but avoids the + /// per-call heap allocation in fc::to_hex. Used by streaming JSON paths that + /// previously paid the to_hex allocation on every action's data / return_value. + void value_hex(const char* data, size_t size) { + value_prefix(); + out_.push_back('"'); + static constexpr char digits[] = "0123456789abcdef"; + const auto* p = reinterpret_cast(data); + for (size_t i = 0; i < size; ++i) { + out_.push_back(digits[p[i] >> 4]); + out_.push_back(digits[p[i] & 0x0f]); + } + out_.push_back('"'); + } + /// Append a preformatted JSON fragment at a value position. Used to embed output /// from legacy fc::variant paths (eg abi_serializer::binary_to_variant + json::to_string) /// without an additional parse/re-emit cycle. @@ -155,15 +174,19 @@ class json_writer { } } + // std::to_chars is non-throwing, locale-independent, and avoids the format-string + // parsing overhead that snprintf pays on every call. buf is sized for the longest + // possible decimal of int64/uint64 (20 chars + sign + slack). Failure case is + // unreachable for fixed-width integers; the unconditional out_.append handles it. void append_signed(int64_t n) { char buf[24]; - int k = std::snprintf(buf, sizeof(buf), "%lld", static_cast(n)); - if (k > 0) out_.append(buf, static_cast(k)); + auto r = std::to_chars(buf, buf + sizeof(buf), n); + out_.append(buf, static_cast(r.ptr - buf)); } void append_unsigned(uint64_t n) { char buf[24]; - int k = std::snprintf(buf, sizeof(buf), "%llu", static_cast(n)); - if (k > 0) out_.append(buf, static_cast(k)); + auto r = std::to_chars(buf, buf + sizeof(buf), n); + out_.append(buf, static_cast(r.ptr - buf)); } std::string& out_; From 3982f11f703bb76777c9655fc8b831f7d8c27b60 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Sat, 25 Apr 2026 14:38:35 -0500 Subject: [PATCH 07/71] trace_api: use json_writer::value_hex for action data + return_value process_block_to_json had two value_string(fc::to_hex(...)) sites for the two binary fields on each action_trace_v0 (data, return_value). The fc::to_hex call constructed a std::string per action which the value_string call then had to copy/escape into the output buffer. value_hex emits the hex digits straight into the writer's output string, saving the heap round-trip per action. Bench (benchmark/trace_api_json, Release -O3, 5 runs, main-thread A/B against variant-only - the variant path defers hex encoding to fc::json::to_string on the HTTP thread): 1000 trxs x 4 acts x 128B data (data-heavy, worst-case): before: stream 17.1 ms vs variant-only 14.8 ms (+15%, regression) after: stream 14.6 ms vs variant-only 14.4 ms (+2%, in noise) --- plugins/trace_api_plugin/src/request_handler.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/trace_api_plugin/src/request_handler.cpp b/plugins/trace_api_plugin/src/request_handler.cpp index 43f5ba45fe..8c734d69b0 100644 --- a/plugins/trace_api_plugin/src/request_handler.cpp +++ b/plugins/trace_api_plugin/src/request_handler.cpp @@ -147,8 +147,8 @@ namespace { w.key("account"); w.value_string(a.account.to_string()); w.key("action"); w.value_string(a.action.to_string()); w.key("authorization"); write_authorizations(w, a.authorization); - w.key("data"); w.value_string(fc::to_hex(a.data.data(), a.data.size())); - w.key("return_value"); w.value_string(fc::to_hex(a.return_value.data(), a.return_value.size())); + w.key("data"); w.value_hex(a.data.data(), a.data.size()); + w.key("return_value"); w.value_hex(a.return_value.data(), a.return_value.size()); // The abi decode path still produces fc::variant (structure is ABI-dependent, // no streaming equivalent here). Splice its serialized form as raw JSON so // the action body stays in the streaming pipeline. From 7e3970d4985822b1b855f245a957194577a3a85c Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Sat, 25 Apr 2026 14:39:01 -0500 Subject: [PATCH 08/71] bench: variant-only A/B + isolated hex / integer micro-benches Three new scenarios in benchmark/trace_api_json.cpp: variant-only: builds the fc::variant tree via process_block but does NOT call fc::json::to_string. Times the actual main-thread cost in production - json::to_string runs on a parallel HTTP-pool thread after cb() hands the variant off, so the existing 'variant+json:' total-CPU number overstates main-thread cost. Compare against 'stream:' to decide whether moving JSON emission onto the main thread is a net main-thread win or regression for any given workload shape. hex iso: emits 32 / 128 / 512 / 2048 random bytes per call, varying calls-per-iteration to keep total work roughly constant. Two paths: hex variant: w.value_string(fc::to_hex(buf.data(), buf.size())) hex direct: w.value_hex(buf.data(), buf.size()) Probes hex-encoding cost without the rest of the trace_api envelope, so optimisations to value_hex can be measured directly. uint64 iso: 1024 mixed-magnitude integers (small / medium / full uint64, shuffled to defeat branch prediction) appended via two local lambdas: one snprintf-based, one std::to_chars-based. Mirrors the change to append_signed / append_unsigned and lets future integer-conversion work report a delta against the same fixed sample. --- benchmark/trace_api_json.cpp | 125 +++++++++++++++++++++++++++++++++-- 1 file changed, 120 insertions(+), 5 deletions(-) diff --git a/benchmark/trace_api_json.cpp b/benchmark/trace_api_json.cpp index 8bb75df8d9..2dfd1c3604 100644 --- a/benchmark/trace_api_json.cpp +++ b/benchmark/trace_api_json.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -129,21 +130,135 @@ void trace_api_json_benchmarking() { + std::to_string(s.actions_per_trx) + " acts (" + std::to_string(s.data_bytes) + "B data)"; - // Legacy path: process_block -> variant -> json::to_string. Times the full - // chain the HTTP handler would execute. - benchmarking("variant: " + label_prefix, [&]() { + // Legacy total path: build variant tree + json::to_string. Times the full + // chain the HTTP handler would execute (variant build on main thread, then + // json::to_string on HTTP thread). Useful as a total-CPU baseline. + benchmarking("variant+json: " + label_prefix, [&]() { auto v = detail::response_formatter::process_block(entry, irreversible, noop_handler); std::string out = fc::json::to_string(v, fc::json::yield_function_t()); // Defeat the optimizer. asm volatile("" : : "g"(out.size()) : "memory"); }); - // Streaming path: process_block_to_json writes directly into std::string. - benchmarking("stream: " + label_prefix, [&]() { + // Variant-only: build the variant tree without serializing to JSON. This is + // the actual current main-thread cost in production - json::to_string runs on + // an HTTP-pool thread after cb() hands the variant off. Compare against + // `stream:` below to decide whether moving JSON emission onto the main thread + // is a net main-thread win or regression. + benchmarking("variant-only: " + label_prefix, [&]() { + auto v = detail::response_formatter::process_block(entry, irreversible, noop_handler); + asm volatile("" : : "g"(v.is_object()) : "memory"); + }); + + // Streaming path: process_block_to_json writes JSON directly into std::string. + // This is the proposed new main-thread cost (no variant intermediate, no + // separate HTTP-thread serialization step). + benchmarking("stream: " + label_prefix, [&]() { std::string out = detail::response_formatter::process_block_to_json(entry, irreversible, noop_handler); asm volatile("" : : "g"(out.size()) : "memory"); }); } + + // ---- Isolated hex-emission micro-bench ------------------------------------ + // + // Probes the hex-encoding cost in isolation, the suspected dominant cause of + // the streaming-vs-variant-only main-thread regression on data-heavy actions. + // Two paths per byte size: + // to_hex+value_string : fc::to_hex(...) -> std::string -> w.value_string(sv) + // value_hex : json_writer::value_hex writes hex straight into the buffer + // + // Each iteration emits a single JSON string token of N hex chars, so the delta + // is the per-byte hex-encoding overhead (plus one std::string alloc/dealloc on + // the to_hex path). Higher N amortises the prefix/quote bookkeeping and + // surfaces the loop cost. + + { + auto rng = make_rng(); + const size_t hex_iter_bytes_total = 64 * 1024; // ~constant work per scenario + const size_t sizes[] = { 32, 128, 512, 2048 }; + for (size_t bytes_per_call : sizes) { + auto buf = random_bytes(rng, bytes_per_call); + const size_t calls = std::max(1, hex_iter_bytes_total / bytes_per_call); + const std::string label = std::to_string(bytes_per_call) + "B per call x " + std::to_string(calls); + + benchmarking("hex variant: " + label, [&]() { + std::string out; + fc::json_writer w(out); + w.begin_array(); + for (size_t i = 0; i < calls; ++i) { + w.value_string(fc::to_hex(buf.data(), buf.size())); + } + w.end_array(); + asm volatile("" : : "g"(out.size()) : "memory"); + }); + + benchmarking("hex direct: " + label, [&]() { + std::string out; + fc::json_writer w(out); + w.begin_array(); + for (size_t i = 0; i < calls; ++i) { + w.value_hex(buf.data(), buf.size()); + } + w.end_array(); + asm volatile("" : : "g"(out.size()) : "memory"); + }); + } + } + + // ---- Isolated integer-emission micro-bench -------------------------------- + // + // Probes the integer-to-decimal cost in isolation, comparing snprintf (the + // pre-change implementation of json_writer::value_uint64) against std::to_chars + // (the post-change implementation). Each iteration emits 1024 uint64 values + // covering a realistic mix of magnitudes: small (status / counters), medium + // (block numbers, cpu_usage_us), large (global_sequence on a long-running + // chain). Values are pre-shuffled so branch prediction can't dominate. + + { + auto rng = make_rng(); + std::vector mix; + mix.reserve(1024); + for (size_t i = 0; i < 1024; ++i) { + std::uniform_int_distribution bucket{0, 2}; + switch (bucket(rng)) { + case 0: mix.push_back(rng() % 256); break; // small + case 1: mix.push_back(rng() % 100'000'000ull); break; // medium + case 2: mix.push_back(rng()); break; // large (full uint64) + } + } + + auto append_snprintf = [](std::string& out, uint64_t n) { + char buf[24]; + int k = std::snprintf(buf, sizeof(buf), "%llu", static_cast(n)); + if (k > 0) out.append(buf, static_cast(k)); + }; + + auto append_to_chars = [](std::string& out, uint64_t n) { + char buf[24]; + auto r = std::to_chars(buf, buf + sizeof(buf), n); + out.append(buf, static_cast(r.ptr - buf)); + }; + + benchmarking("uint64 snprintf: 1024 mixed", [&]() { + std::string out; + out.reserve(16 * 1024); + for (uint64_t n : mix) { + append_snprintf(out, n); + out.push_back(','); + } + asm volatile("" : : "g"(out.size()) : "memory"); + }); + + benchmarking("uint64 to_chars: 1024 mixed", [&]() { + std::string out; + out.reserve(16 * 1024); + for (uint64_t n : mix) { + append_to_chars(out, n); + out.push_back(','); + } + asm volatile("" : : "g"(out.size()) : "memory"); + }); + } } } // namespace sysio::benchmark From 5bf1f49c6b1ab86fcfd9c6347f1198a4ea5fcbfc Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Sat, 25 Apr 2026 16:32:54 -0500 Subject: [PATCH 09/71] fc::json_writer: add set / set_raw fused emitters - set(name, value) emits "name": using to_json_stream for the value dispatch. Returns *this so call sites chain naturally: w.begin_object(); w.set("id", id) .set("name", n) .set("amount", amt); w.end_object(); - set_raw(name, raw_json) splices a preformatted JSON fragment as the value. For embedding output from legacy fc::variant paths (eg abi_serializer::binary_to_variant + json::to_string) without re-parsing. Mirrors fc::mutable_variant_object::set / operator() ergonomics so streaming-JSON migrations from variant-based code keep the per-field one-liner shape. Three tests added to test_json_stream covering chained primitive emission, dispatch through to_json_stream for fc and reflected types, and raw-fragment splice. --- libraries/libfc/include/fc/io/json_stream.hpp | 31 ++++++++++ libraries/libfc/test/io/test_json_stream.cpp | 62 +++++++++++++++++++ 2 files changed, 93 insertions(+) diff --git a/libraries/libfc/include/fc/io/json_stream.hpp b/libraries/libfc/include/fc/io/json_stream.hpp index 09a25b631a..2afa42e2bb 100644 --- a/libraries/libfc/include/fc/io/json_stream.hpp +++ b/libraries/libfc/include/fc/io/json_stream.hpp @@ -144,6 +144,37 @@ class json_writer { out_.append(raw.data(), raw.size()); } + /// Fused key + value emitter; the streaming-JSON counterpart to + /// fc::mutable_variant_object::set / operator(). Returns *this so call sites + /// chain naturally: + /// + /// w.begin_object(); + /// w.set("id", id) + /// .set("number", n) + /// .set("producer", producer); + /// w.end_object(); + /// + /// Dispatches the value via to_json_stream, so any type with a + /// to_json_stream overload (primitives, std containers, fc leaf types, + /// FC_REFLECT'd structs via the reflector path) is supported. Callers must + /// have included at the call site - that header + /// is where the to_json_stream primary template + reflector dispatch lives. + template + json_writer& set(std::string_view name, const T& value) { + key(name); + to_json_stream(value, *this); + return *this; + } + + /// Fused key + raw-value emitter. Same chaining shape as set(), but the + /// value is a preformatted JSON fragment spliced verbatim (eg an + /// abi_serializer::binary_to_variant result that has already been json'd). + json_writer& set_raw(std::string_view name, std::string_view raw_json) { + key(name); + raw_value(raw_json); + return *this; + } + /// True when all begin_* have been paired with end_*. Asserted internally on destructor /// usage via value_prefix() is not possible, so callers can check this in tests. bool balanced() const { return stack_.empty(); } diff --git a/libraries/libfc/test/io/test_json_stream.cpp b/libraries/libfc/test/io/test_json_stream.cpp index 2d0d964ca5..8f046bb591 100644 --- a/libraries/libfc/test/io/test_json_stream.cpp +++ b/libraries/libfc/test/io/test_json_stream.cpp @@ -257,4 +257,66 @@ BOOST_AUTO_TEST_CASE(fc_public_key_and_signature) { fc::json::to_string(v_sig, fc::json::yield_function_t())); } +BOOST_AUTO_TEST_CASE(set_chained_object) { + // w.set("name", value) is sugar over w.key("name") + to_json_stream(value, w). + // Returns *this so call sites can chain. The expected output is byte-identical + // to the equivalent key()/value_*() sequence. + std::string out; + { + fc::json_writer w(out); + w.begin_object(); + w.set("a", 1) + .set("b", std::string{"two"}) + .set("c", true) + .set("d", 3.5); + w.end_object(); + } + BOOST_CHECK_EQUAL(out, R"({"a":1,"b":"two","c":true,"d":3.5})"); +} + +BOOST_AUTO_TEST_CASE(set_dispatches_via_to_json_stream) { + // Confirms set picks up to_json_stream overloads for fc leaf types and reflected + // structs - same dispatch rule as the free-function fc::to_json_stream. + const fc::sha256 h = fc::sha256::hash(std::string("hello")); + point_t s{42, 7, "x"}; + std::vector nums{1, 2, 3}; + + std::string out; + { + fc::json_writer w(out); + w.begin_object(); + w.set("hash", h) // dispatches to_json_stream(sha256, w) + .set("inner", s) // dispatches via reflector path + .set("nums", nums); // dispatches to_json_stream(vector, w) + w.end_object(); + } + + // Cross-check against the variant + json::to_string output for the same shape. + fc::variant v_hash, v_inner, v_nums; + fc::to_variant(h, v_hash); + fc::to_variant(s, v_inner); + fc::to_variant(nums, v_nums); + const std::string expected = + std::string{"{\"hash\":"} + fc::json::to_string(v_hash, fc::json::yield_function_t()) + + ",\"inner\":" + fc::json::to_string(v_inner, fc::json::yield_function_t()) + + ",\"nums\":" + fc::json::to_string(v_nums, fc::json::yield_function_t()) + + "}"; + BOOST_CHECK_EQUAL(out, expected); +} + +BOOST_AUTO_TEST_CASE(set_raw_splices_preformatted_fragment) { + // set_raw is "key + raw_value": for embedding a JSON fragment that was already + // serialized elsewhere (eg the abi_serializer + json::to_string path). + std::string out; + { + fc::json_writer w(out); + w.begin_object(); + w.set("name", std::string{"alice"}) + .set_raw("payload", R"({"a":1,"b":"two"})") + .set("done", true); + w.end_object(); + } + BOOST_CHECK_EQUAL(out, R"({"name":"alice","payload":{"a":1,"b":"two"},"done":true})"); +} + BOOST_AUTO_TEST_SUITE_END() From 27edcb8123ea25eadc983afa2e7d5da8ba42bacf Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Sat, 25 Apr 2026 16:33:12 -0500 Subject: [PATCH 10/71] trace_api: use json_writer set / set_raw chains in process_block_to_json Migrates process_block_to_json + write_actions / write_authorizations / write_transactions from explicit w.key("name")+w.value_*(value) pairs to chained w.set("name", value) calls. Matches the original mutable_variant_object()(...) shape that lived on the variant path. ~25% LOC reduction at the streaming call sites. Behaviorally a no-op: set inlines to key+to_json_stream which inlines to the same value_*() dispatch the previous code used. trace_responses test (7 cases) confirms byte-identical output; trace_api_json bench (4 shapes) confirms unchanged perf. --- .../trace_api_plugin/src/request_handler.cpp | 63 +++++++++---------- 1 file changed, 29 insertions(+), 34 deletions(-) diff --git a/plugins/trace_api_plugin/src/request_handler.cpp b/plugins/trace_api_plugin/src/request_handler.cpp index 8c734d69b0..b8820cfd4f 100644 --- a/plugins/trace_api_plugin/src/request_handler.cpp +++ b/plugins/trace_api_plugin/src/request_handler.cpp @@ -119,8 +119,8 @@ namespace { w.begin_array(); for (const auto& a : auths) { w.begin_object(); - w.key("account"); w.value_string(a.account.to_string()); - w.key("permission"); w.value_string(a.permission.to_string()); + w.set("account", a.account.to_string()) + .set("permission", a.permission.to_string()); w.end_object(); } w.end_array(); @@ -142,10 +142,10 @@ namespace { for (int idx : indices) { const auto& a = actions.at(idx); w.begin_object(); - w.key("global_sequence"); w.value_uint64(a.global_sequence); - w.key("receiver"); w.value_string(a.receiver.to_string()); - w.key("account"); w.value_string(a.account.to_string()); - w.key("action"); w.value_string(a.action.to_string()); + w.set("global_sequence", a.global_sequence) + .set("receiver", a.receiver.to_string()) + .set("account", a.account.to_string()) + .set("action", a.action.to_string()); w.key("authorization"); write_authorizations(w, a.authorization); w.key("data"); w.value_hex(a.data.data(), a.data.size()); w.key("return_value"); w.value_hex(a.return_value.data(), a.return_value.size()); @@ -154,12 +154,10 @@ namespace { // the action body stays in the streaming pipeline. auto [params, return_data] = data_handler(a); if (!params.is_null()) { - w.key("params"); - w.raw_value(fc::json::to_string(params, fc::json::yield_function_t())); + w.set_raw("params", fc::json::to_string(params, fc::json::yield_function_t())); } if (return_data.has_value()) { - w.key("return_data"); - w.raw_value(fc::json::to_string(*return_data, fc::json::yield_function_t())); + w.set_raw("return_data", fc::json::to_string(*return_data, fc::json::yield_function_t())); } w.end_object(); } @@ -172,22 +170,19 @@ namespace { w.begin_array(); for (const auto& t : transactions) { w.begin_object(); - w.key("id"); fc::to_json_stream(t.id, w); - w.key("block_num"); w.value_uint32(t.block_num); - // block_timestamp_type serializes as its slot (uint32) in to_variant; preserve - // that shape so clients see identical output. - w.key("block_time"); w.value_uint32(t.block_time.slot); - w.key("producer_block_id"); fc::to_json_stream(t.producer_block_id, w); - w.key("actions"); write_actions(w, t.actions, data_handler); + w.set("id", t.id) + .set("block_num", t.block_num) + // block_timestamp_type serializes as its slot (uint32) in to_variant; preserve + // that shape so clients see identical output. + .set("block_time", t.block_time.slot) + .set("producer_block_id", t.producer_block_id); + w.key("actions"); write_actions(w, t.actions, data_handler); // status is fc::enum_type; to_variant emits the numeric // underlying value. Match that. - w.key("status"); w.value_uint64(static_cast(t.status)); - w.key("cpu_usage_us"); w.value_uint32(t.cpu_usage_us); - w.key("net_usage_words"); w.value_uint64(t.net_usage_words.value); - w.key("signatures"); - w.begin_array(); - for (const auto& sig : t.signatures) fc::to_json_stream(sig, w); - w.end_array(); + w.set("status", static_cast(t.status)) + .set("cpu_usage_us", t.cpu_usage_us) + .set("net_usage_words", t.net_usage_words.value) + .set("signatures", t.signatures); // transaction_header is a reflected struct composed of fc::time_point_sec, // uint16/uint32 and unsigned_ints. No native to_json_stream path yet so // fall back via the variant bridge for just that field. @@ -205,16 +200,16 @@ namespace sysio::trace_api::detail { fc::json_writer w(out); std::visit([&](auto&& bt) { w.begin_object(); - w.key("id"); w.value_string(bt.id.str()); - w.key("number"); w.value_uint32(bt.number); - w.key("previous_id"); w.value_string(bt.previous_id.str()); - w.key("status"); w.value_string(irreversible ? "irreversible" : "pending"); - // to_iso8601_datetime appends a 'Z' suffix that process_block also emits. - w.key("timestamp"); w.value_string(to_iso8601_datetime(bt.timestamp)); - w.key("producer"); w.value_string(bt.producer.to_string()); - w.key("transaction_mroot"); fc::to_json_stream(bt.transaction_mroot, w); - w.key("finality_mroot"); fc::to_json_stream(bt.finality_mroot, w); - w.key("transactions"); write_transactions(w, bt.transactions, data_handler); + w.set("id", bt.id) + .set("number", bt.number) + .set("previous_id", bt.previous_id) + .set("status", irreversible ? "irreversible" : "pending") + // to_iso8601_datetime appends a 'Z' suffix that process_block also emits. + .set("timestamp", to_iso8601_datetime(bt.timestamp)) + .set("producer", bt.producer.to_string()) + .set("transaction_mroot", bt.transaction_mroot) + .set("finality_mroot", bt.finality_mroot); + w.key("transactions"); write_transactions(w, bt.transactions, data_handler); w.end_object(); }, trace); return out; From 08307bf1b1752a9a1d430041bf46b7a92d188a7c Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Sat, 25 Apr 2026 16:33:23 -0500 Subject: [PATCH 11/71] bench: add struct-pass scenarios for HTTP-thread streaming pivot Two new measurement scenarios per shape in benchmark/trace_api_json.cpp: struct-pass (fn): captures the result struct into a std::function closure; mirrors the main-thread cost if streaming work moves to the HTTP thread via a std::function-typed callback. struct-pass (mof): same pattern using std::move_only_function (C++23, supported by both clang-18 + libstdc++ 13.3 and gcc-14 in our CI matrix). Closure capture is move-only so we don't pay for the std::function copy slot. Pre-builds a pool of fresh result-struct copies outside the timing loop (matches the existing variant-only / stream pattern that excludes API-call cost from the bench). Numbers (Release -O3, 5 runs): Shape struct-pass(mof) stream ratio 100 trxs x 1 act 32B 3.9 us 1.27 ms ~325x 5000 trxs x 1 act 32B 298 us 59.6 ms ~200x 25000 trxs x 1 act 32B 1.57 ms 293 ms ~187x 1000 trxs x 4 acts 128B 211 us 14.6 ms ~69x Validates the HTTP-thread pivot's main-thread minimization argument: by moving the JSON serialization out of the api thread and into the HTTP thread pool, the api thread pays only the closure-capture cost (one heap alloc plus a few pointer swaps) instead of the full streaming pass. --- benchmark/trace_api_json.cpp | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/benchmark/trace_api_json.cpp b/benchmark/trace_api_json.cpp index 2dfd1c3604..87c7c4c7c4 100644 --- a/benchmark/trace_api_json.cpp +++ b/benchmark/trace_api_json.cpp @@ -157,6 +157,42 @@ void trace_api_json_benchmarking() { std::string out = detail::response_formatter::process_block_to_json(entry, irreversible, noop_handler); asm volatile("" : : "g"(out.size()) : "memory"); }); + + // Struct-pass (HTTP-thread streaming pivot floor): main-thread cost when + // the API lambda hands the result struct (not a variant, not a string) to + // the HTTP thread for serialization. API lambda captures the struct into + // a std::function closure and hands it to cb(); the + // HTTP thread runs the closure later (off-main, not measured here). The + // capture is move-only so the per-trx vectors transfer pointers, not data. + // std::function does heap-allocate because the captured block_trace_v0 + // exceeds the small-buffer optimisation threshold. + // + // Pre-builds a pool of data_log_entry copies outside the timing loop so + // each iter consumes a fresh one. Setup cost is excluded from the bench + // (matches variant-only / stream which also start from a pre-built entry). + const size_t pool_size = 64; // > any reasonable --runs + std::vector fresh_a(pool_size, entry); + std::vector fresh_b(pool_size, entry); + size_t fresh_idx_a = 0; + size_t fresh_idx_b = 0; + benchmarking("struct-pass (fn): " + label_prefix, [&]() { + data_log_entry e = std::move(fresh_a[fresh_idx_a++ % pool_size]); + std::function body = + [e = std::move(e)](fc::json_writer& w) mutable { + asm volatile("" : : "g"(&e) : "memory"); + }; + auto holder = std::move(body); // mimic cb() posting onto the HTTP pool + asm volatile("" : : "g"(&holder) : "memory"); + }); + benchmarking("struct-pass (mof): " + label_prefix, [&]() { + data_log_entry e = std::move(fresh_b[fresh_idx_b++ % pool_size]); + std::move_only_function body = + [e = std::move(e)](fc::json_writer& w) mutable { + asm volatile("" : : "g"(&e) : "memory"); + }; + auto holder = std::move(body); + asm volatile("" : : "g"(&holder) : "memory"); + }); } // ---- Isolated hex-emission micro-bench ------------------------------------ From 38f1dc36cd06aa3b32615bfba7a8b0717f4dc850 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Sat, 25 Apr 2026 20:24:25 -0500 Subject: [PATCH 12/71] http_plugin: add streaming-cb registration path + handle_exception_stream Adds the parallel infrastructure for endpoints that hand the response off to the http thread pool as a json_writer-emitting closure rather than as a fc::variant tree. Endpoints opt in by registering via add_handler_stream / add_async_handler_stream (analogous to the existing add_handler / add_async_handler). The variant path stays unchanged; both maps are checked during request dispatch and disjointness is asserted at registration. New types in http_plugin.hpp: using stream_emitter = std::move_only_function; using url_response_stream_callback = std::move_only_function; using url_handler_stream = std::function; struct api_entry_stream { string path; api_category category; url_handler_stream handler; }; stream_emitter is move_only_function so the api lambda can capture the typed result struct by-move without forcing CopyConstructible on it. std::function would have required either an unnecessary copy constructor or a wrapping shared_ptr; move_only_function is C++23-standard and is supported by both clang-18 / libstdc++ 13.3 and gcc-14 across the CI matrix. Internal plumbing in common.hpp: - internal_url_handler_stream (same shape as internal_url_handler, no content_type slot - streaming output is always application/json). - url_handlers_stream_type map alongside the existing url_handlers map in http_plugin_state. - make_http_stream_response_handler builds the cb lambda that posts the emitter onto the thread pool, runs it into a freshly-allocated body buffer, tracks bytes_in_flight, and sends the response. beast_http_session.hpp checks the streaming map first, falls back to the variant map. /v1/node/get_supported_apis now lists URLs from both maps. handle_exception_stream is the streaming counterpart to the existing handle_exception. Both share a single classify_current_exception template that runs the catch chain once and invokes a caller-supplied emit lambda with (http_code, error_results). No duplicated catch arms; the only difference between the two public functions is how they deliver the error_results to their respective cb shape. error_results is FC_REFLECT'd, so the streaming path emits via to_json_stream's reflector dispatch. No endpoints migrated in this commit - infra only. Existing variant-cb endpoints continue to use the variant cb path. --- .../sysio/http_plugin/beast_http_session.hpp | 25 ++- .../include/sysio/http_plugin/common.hpp | 64 ++++++ .../include/sysio/http_plugin/http_plugin.hpp | 72 +++++++ plugins/http_plugin/src/http_plugin.cpp | 203 +++++++++++++----- 4 files changed, 312 insertions(+), 52 deletions(-) diff --git a/plugins/http_plugin/include/sysio/http_plugin/beast_http_session.hpp b/plugins/http_plugin/include/sysio/http_plugin/beast_http_session.hpp index 9e59acd9e6..aa6c993444 100644 --- a/plugins/http_plugin/include/sysio/http_plugin/beast_http_session.hpp +++ b/plugins/http_plugin/include/sysio/http_plugin/beast_http_session.hpp @@ -157,9 +157,26 @@ class beast_http_session : public detail::abstract_conn, remote_endpoint_, to_log_string(req) ); std::string resource = std::string(req.target()); - // look for the URL handler to handle this resource + // look for the URL handler to handle this resource - check the streaming map + // first since it's the smaller of the two during migration; both maps share + // the URL keyspace and registration asserts disjointness. + auto stream_itr = plugin_state_->url_handlers_stream.find(resource); auto handler_itr = plugin_state_->url_handlers.find(resource); - if(handler_itr != plugin_state_->url_handlers.end() && categories_.contains(handler_itr->second.category)) { + if(stream_itr != plugin_state_->url_handlers_stream.end() && categories_.contains(stream_itr->second.category)) { + fc_tlog(plugin_state_->get_logger(), "resource (stream): {}", resource); + std::string body = req.body(); + // Streaming responses are always application/json - the json_writer output + // is the response body verbatim, no plaintext / json_raw distinction. + set_content_type_header(http_content_type::json); + + if (plugin_state_->update_metrics) + plugin_state_->update_metrics({resource}); + + stream_itr->second.fn(this->shared_from_this(), + std::move(resource), + std::move(body), + make_http_stream_response_handler(*plugin_state_, this->shared_from_this())); + } else if(handler_itr != plugin_state_->url_handlers.end() && categories_.contains(handler_itr->second.category)) { fc_tlog(plugin_state_->get_logger(), "resource: {}", resource); std::string body = req.body(); auto content_type = handler_itr->second.content_type; @@ -178,6 +195,10 @@ class beast_http_session : public detail::abstract_conn, if (categories_.contains(handler.second.category)) result.apis.push_back(handler.first); } + for (const auto& handler : plugin_state_->url_handlers_stream) { + if (categories_.contains(handler.second.category)) + result.apis.push_back(handler.first); + } send_response(fc::json::to_string(fc::variant(result), fc::time_point::maximum()), 200); } else { fc_dlog( plugin_state_->get_logger(), "404 - not found: {}", resource ); diff --git a/plugins/http_plugin/include/sysio/http_plugin/common.hpp b/plugins/http_plugin/include/sysio/http_plugin/common.hpp index 877166df11..678b230b6c 100644 --- a/plugins/http_plugin/include/sysio/http_plugin/common.hpp +++ b/plugins/http_plugin/include/sysio/http_plugin/common.hpp @@ -3,6 +3,7 @@ #include // for thread pool #include +#include #include #include #include @@ -76,6 +77,19 @@ struct internal_url_handler { api_category category; http_content_type content_type = http_content_type::json; }; + +/** +* Streaming counterpart of internal_url_handler. Endpoints registered via +* http_plugin::add_handler_stream / add_async_handler_stream live in the parallel +* url_handlers_stream map; the beast dispatch checks both maps and routes to whichever +* one owns the URL. No content_type slot - the streaming cb always emits +* application/json directly into the response buffer. +*/ +using internal_url_handler_stream_fn = std::function; +struct internal_url_handler_stream { + internal_url_handler_stream_fn fn; + api_category category; +}; /** * Helper method to calculate the "in flight" size of a fc::variant * This is an estimate based on fc::raw::pack if that process can be successfully executed @@ -110,6 +124,10 @@ static size_t in_flight_sizeof(const std::optional& o) { // key -> priority, url_handler typedef map url_handlers_type; +// Streaming-cb counterpart of url_handlers_type. Disjoint URL-set from the variant +// path; an endpoint registers either form, never both. +typedef map url_handlers_stream_type; + struct http_plugin_state { string access_control_allow_origin; string access_control_allow_headers; @@ -129,6 +147,7 @@ struct http_plugin_state { string server_header; url_handlers_type url_handlers; + url_handlers_stream_type url_handlers_stream; bool keep_alive = false; uint16_t thread_pool_size = 2; @@ -202,6 +221,51 @@ inline auto make_http_response_handler(http_plugin_state& plugin_state, detail:: } +/** +* Construct a url_response_stream_callback that runs the emitter on the http_plugin +* thread pool and sends the produced JSON as the response body. +* +* The api lambda is responsible for capturing whatever it wants to serialize into +* the emitter closure (typically the typed result struct, by-move). All JSON +* serialization happens inside the dispatched lambda on the http thread pool, so +* the api thread (read_only / read_write queue) never pays the per-field +* allocation cost the variant tree built. +* +* Backpressure note: bytes_in_flight tracking happens after the emitter runs (we +* don't know the body size until the buffer is full), so this path differs from +* the variant path's pre-post estimate. Net effect is "verify_max_bytes_in_flight +* checks the actual produced body size before send_response," which is the same +* end-state check the variant path performs. +*/ +inline auto make_http_stream_response_handler(http_plugin_state& plugin_state, detail::abstract_conn_ptr session_ptr) { + return url_response_stream_callback{ + [&plugin_state, session_ptr{std::move(session_ptr)}] + (int code, stream_emitter emitter) mutable { + + boost::asio::dispatch(plugin_state.thread_pool.get_executor(), + [&plugin_state, session_ptr{std::move(session_ptr)}, code, emitter{std::move(emitter)}]() mutable { + try { + std::string body; + body.reserve(4096); + { + fc::json_writer w(body); + emitter(w); + } + plugin_state.bytes_in_flight += body.size(); + auto on_exit = fc::make_scoped_exit([&, sz=body.size()]() { plugin_state.bytes_in_flight -= sz; }); + + if (auto error_str = session_ptr->verify_max_bytes_in_flight(0); !error_str.empty()) + session_ptr->send_busy_response(std::move(error_str)); + else + session_ptr->send_response(std::move(body), code); + } catch (...) { + session_ptr->handle_exception(); + } + }); + } + }; +} + inline bool host_is_valid(const http_plugin_state& plugin_state, const std::string& header_host_port, const asio::ip::address& addr) { diff --git a/plugins/http_plugin/include/sysio/http_plugin/http_plugin.hpp b/plugins/http_plugin/include/sysio/http_plugin/http_plugin.hpp index 43c213fcd7..742e44386f 100644 --- a/plugins/http_plugin/include/sysio/http_plugin/http_plugin.hpp +++ b/plugins/http_plugin/include/sysio/http_plugin/http_plugin.hpp @@ -6,6 +6,10 @@ #include #include #include +#include + +#include + namespace sysio { using namespace appbase; @@ -29,6 +33,29 @@ namespace sysio { **/ using url_handler = std::function; + /** + * @brief Streaming response callback path. + * + * Parallel to url_response_callback / url_handler. Endpoints that opt into the + * streaming path build no fc::variant on the response - instead they hand cb() + * a closure that emits the response body directly into a fc::json_writer. The + * closure runs on the http_plugin thread pool, deferring the JSON emission off + * whatever thread the api lambda runs on (api thread, read-only pool, ...). + * + * Migration shape: + * variant cb: cb(code, fc::variant(result)); + * stream cb: cb(code, [r = std::move(result)](fc::json_writer& w) mutable { + * fc::to_json_stream(r, w); + * }); + * + * stream_emitter is move_only_function so the captured result struct does not + * have to satisfy CopyConstructible; pass the typed struct directly without + * round-tripping through fc::variant. + */ + using stream_emitter = std::move_only_function; + using url_response_stream_callback = std::move_only_function; + using url_handler_stream = std::function; + /** * @brief An API, containing URLs and handlers * @@ -44,6 +71,21 @@ namespace sysio { using api_description = std::vector; + /** + * api_entry_stream is the streaming-cb counterpart of api_entry. Same + * registration shape but the handler invokes a url_response_stream_callback + * rather than url_response_callback. Endpoints register either form via + * http_plugin::add_handler / add_handler_stream (or the _api batch + * registration helpers). + */ + struct api_entry_stream { + string path; + api_category category; + url_handler_stream handler; + }; + + using api_description_stream = std::vector; + enum class http_content_type { json = 1, plaintext = 2, @@ -113,9 +155,39 @@ namespace sysio { add_async_handler(std::move(call), content_type); } + // ---- Streaming-cb registration -------------------------------------------------- + // Endpoints registered via add_handler_stream / add_async_handler_stream invoke + // url_response_stream_callback instead of url_response_callback. The cb's emitter + // closure runs on the http_plugin thread pool, so the api/main thread does no + // JSON serialization work; it just hands a (typically by-move) result struct off + // through the closure capture. + // + // content_type is implicitly application/json - the streaming path never produces + // anything else by design. No json_raw distinction is needed; the json_writer + // output is the response body verbatim. + void add_handler_stream(api_entry_stream&& entry, appbase::exec_queue q, int priority = appbase::priority::medium_low); + void add_api_stream(api_description_stream&& api, appbase::exec_queue q, int priority = appbase::priority::medium_low) { + for (auto& call : api) + add_handler_stream(std::move(call), q, priority); + } + + void add_async_handler_stream(api_entry_stream&& entry); + void add_async_api_stream(api_description_stream&& api) { + for (auto& call : api) + add_async_handler_stream(std::move(call)); + } + // standard exception handling for api handlers static void handle_exception( const char *api_name, const char *call_name, const string& body, const url_response_callback& cb ); + /// Streaming counterpart to handle_exception: emits an error_results object as + /// JSON via the reflector path instead of building a fc::variant tree. Pass cb + /// by mutable lvalue ref since url_response_stream_callback is move-only; + /// invoking it does not move from cb, so the api lambda may still call cb + /// elsewhere on a different code path (in practice exactly one path runs per + /// request). + static void handle_exception_stream( const char *api_name, const char *call_name, const string& body, url_response_stream_callback& cb ); + void post_http_thread_pool(std::function f); bool is_on_loopback(api_category category) const; diff --git a/plugins/http_plugin/src/http_plugin.cpp b/plugins/http_plugin/src/http_plugin.cpp index b0601995f7..63461ec9b0 100644 --- a/plugins/http_plugin/src/http_plugin.cpp +++ b/plugins/http_plugin/src/http_plugin.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -187,6 +188,55 @@ namespace sysio { }; return handler; } + + /** + * Streaming-cb counterpart of make_app_thread_url_handler. Posts the api + * lambda onto the requested executor queue (read_only / read_write / etc) + * and forwards a streaming-cb that the api lambda invokes with a closure + * which the http_plugin thread pool will run later. + */ + static detail::internal_url_handler_stream make_app_thread_url_handler_stream(api_entry_stream&& entry, appbase::exec_queue to_queue, int priority) { + detail::internal_url_handler_stream handler; + handler.category = entry.category; + auto next_ptr = std::make_shared(std::move(entry.handler)); + handler.fn = [priority, to_queue, next_ptr=std::move(next_ptr)] + ( detail::abstract_conn_ptr conn, string&& r, string&& b, url_response_stream_callback&& then ) { + if (auto error_str = conn->verify_max_bytes_in_flight(b.size()); !error_str.empty()) { + conn->send_busy_response(std::move(error_str)); + return; + } + + app().executor().post( priority, to_queue, + [next_ptr, conn=std::move(conn), r=std::move(r), b=std::move(b), then=std::move(then)]() mutable { + try { + if( app().is_quiting() ) return; + (*next_ptr)( std::move(r), std::move(b), std::move(then) ); + } catch( ... ) { + conn->handle_exception(); + } + }); + }; + return handler; + } + + /** + * Streaming-cb counterpart of make_http_thread_url_handler. Runs the api + * lambda directly on the http_plugin thread (the streaming cb's emitter + * runs on the same pool, so the api lambda doing minimal work and + * immediately handing off to the cb is fine). + */ + static detail::internal_url_handler_stream make_http_thread_url_handler_stream(api_entry_stream&& entry) { + detail::internal_url_handler_stream handler; + handler.category = entry.category; + handler.fn = [next=std::move(entry.handler)]( const detail::abstract_conn_ptr& conn, string&& r, string&& b, url_response_stream_callback&& then ) mutable { + try { + next(std::move(r), std::move(b), std::move(then)); + } catch( ... ) { + conn->handle_exception(); + } + }; + return handler; + } bool is_unix_socket_address(const std::string& address) const { return address.starts_with("/") || address.starts_with("./") || address.starts_with("../"); @@ -536,67 +586,120 @@ namespace sysio { SYS_ASSERT( p.second, chain::plugin_config_exception, "http url {} is not unique", path ); } + namespace { + void log_add_handler_stream(http_plugin_impl* my, api_entry_stream& entry) { + auto addrs = my->addresses_for_category(entry.category); + if (addrs.size()) + addrs = "on " + addrs; + else + addrs = "disabled for category address not configured"; + fc_ilog(logger(), "add {} api url (stream): {} {}", + from_category(entry.category), entry.path, addrs); + } + } + + void http_plugin::add_handler_stream(api_entry_stream&& entry, appbase::exec_queue q, int priority) { + log_add_handler_stream(my.get(), entry); + std::string path = entry.path; + // The two handler maps are keyed identically; assert disjointness so a stale + // duplicate registration on the variant path doesn't silently shadow the + // streaming registration (or vice versa). + auto& vmap = my->plugin_state->url_handlers; + auto& smap = my->plugin_state->url_handlers_stream; + SYS_ASSERT( !vmap.contains(path), chain::plugin_config_exception, + "http url {} is not unique - already registered on the variant cb path", path ); + SYS_ASSERT( !smap.contains(path), chain::plugin_config_exception, + "http url {} is not unique - already registered on the streaming cb path", path ); + smap.emplace(path, my->make_app_thread_url_handler_stream(std::move(entry), q, priority)); + } + + void http_plugin::add_async_handler_stream(api_entry_stream&& entry) { + log_add_handler_stream(my.get(), entry); + std::string path = entry.path; + auto& vmap = my->plugin_state->url_handlers; + auto& smap = my->plugin_state->url_handlers_stream; + SYS_ASSERT( !vmap.contains(path), chain::plugin_config_exception, + "http url {} is not unique - already registered on the variant cb path", path ); + SYS_ASSERT( !smap.contains(path), chain::plugin_config_exception, + "http url {} is not unique - already registered on the streaming cb path", path ); + smap.emplace(path, my->make_http_thread_url_handler_stream(std::move(entry))); + } + void http_plugin::post_http_thread_pool(std::function f) { if( f ) boost::asio::post( my->plugin_state->thread_pool.get_executor(), f ); } - void http_plugin::handle_exception( const char* api_name, const char* call_name, const string& body, const url_response_callback& cb) { - try { + namespace { + // Shared catch-chain for handle_exception / handle_exception_stream. Logs the + // in-flight exception and invokes `emit` with (http_code, error_results); the + // caller decides how to deliver the result via its specific cb shape (variant + // path or streaming closure). + template + void classify_current_exception(const char* api_name, const char* call_name, + const std::string& body, Emit&& emit) { try { - throw; - } catch (chain::unknown_block_exception& e) { - fc_dlog( logger(), "Unknown block while processing {}.{}: {}", - api_name, call_name, e.to_detail_string() ); - error_results results{400, "Unknown Block", error_results::error_info(e, verbose_http_errors)}; - cb( 400, fc::variant( results )); - } catch (chain::invalid_http_request& e) { - fc_dlog( logger(), "Invalid http request while processing {}.{}: {}", - api_name, call_name, e.to_detail_string() ); - error_results results{400, "Invalid Request", error_results::error_info(e, verbose_http_errors)}; - cb( 400, fc::variant( results )); - } catch (chain::account_query_exception& e) { - fc_dlog( logger(), "Account query exception while processing {}.{}: {}", - api_name, call_name, e.to_detail_string() ); - error_results results{400, "Account lookup", error_results::error_info(e, verbose_http_errors)}; - cb( 400, fc::variant( results )); - } catch (chain::unsatisfied_authorization& e) { - fc_dlog( logger(), "Auth error while processing {}.{}: {}", - api_name, call_name, e.to_detail_string() ); - error_results results{401, "UnAuthorized", error_results::error_info(e, verbose_http_errors)}; - cb( 401, fc::variant( results )); - } catch (chain::tx_duplicate& e) { - fc_dlog( logger(), "Duplicate trx while processing {}.{}: {}", - api_name, call_name, e.to_detail_string() ); - error_results results{409, "Conflict", error_results::error_info(e, verbose_http_errors)}; - cb( 409, fc::variant( results )); - } catch (fc::eof_exception& e) { - fc_elog( logger(), "Unable to parse arguments to {}.{}", api_name, call_name ); - fc_dlog( logger(), "Bad arguments: {}", body ); - error_results results{422, "Unprocessable Entity", error_results::error_info(e, verbose_http_errors)}; - cb( 422, fc::variant( results )); - } catch (fc::exception& e) { - fc_dlog( logger(), "Exception while processing {}.{}: {}", - api_name, call_name, e.to_detail_string() ); - error_results results{500, "Internal Service Error", error_results::error_info(e, verbose_http_errors)}; - cb( 500, fc::variant( results )); - } catch (std::exception& e) { - fc_dlog( logger(), "STD Exception encountered while processing {}.{}: {}", - api_name, call_name, e.what() ); - error_results results{500, "Internal Service Error", error_results::error_info(fc::exception( FC_LOG_MESSAGE( error, "{}", e.what())), verbose_http_errors)}; - cb( 500, fc::variant( results )); + try { + throw; + } catch (chain::unknown_block_exception& e) { + fc_dlog( logger(), "Unknown block while processing {}.{}: {}", + api_name, call_name, e.to_detail_string() ); + emit(400, error_results{400, "Unknown Block", error_results::error_info(e, verbose_http_errors)}); + } catch (chain::invalid_http_request& e) { + fc_dlog( logger(), "Invalid http request while processing {}.{}: {}", + api_name, call_name, e.to_detail_string() ); + emit(400, error_results{400, "Invalid Request", error_results::error_info(e, verbose_http_errors)}); + } catch (chain::account_query_exception& e) { + fc_dlog( logger(), "Account query exception while processing {}.{}: {}", + api_name, call_name, e.to_detail_string() ); + emit(400, error_results{400, "Account lookup", error_results::error_info(e, verbose_http_errors)}); + } catch (chain::unsatisfied_authorization& e) { + fc_dlog( logger(), "Auth error while processing {}.{}: {}", + api_name, call_name, e.to_detail_string() ); + emit(401, error_results{401, "UnAuthorized", error_results::error_info(e, verbose_http_errors)}); + } catch (chain::tx_duplicate& e) { + fc_dlog( logger(), "Duplicate trx while processing {}.{}: {}", + api_name, call_name, e.to_detail_string() ); + emit(409, error_results{409, "Conflict", error_results::error_info(e, verbose_http_errors)}); + } catch (fc::eof_exception& e) { + fc_elog( logger(), "Unable to parse arguments to {}.{}", api_name, call_name ); + fc_dlog( logger(), "Bad arguments: {}", body ); + emit(422, error_results{422, "Unprocessable Entity", error_results::error_info(e, verbose_http_errors)}); + } catch (fc::exception& e) { + fc_dlog( logger(), "Exception while processing {}.{}: {}", + api_name, call_name, e.to_detail_string() ); + emit(500, error_results{500, "Internal Service Error", error_results::error_info(e, verbose_http_errors)}); + } catch (std::exception& e) { + fc_dlog( logger(), "STD Exception encountered while processing {}.{}: {}", + api_name, call_name, e.what() ); + emit(500, error_results{500, "Internal Service Error", + error_results::error_info(fc::exception( FC_LOG_MESSAGE( error, "{}", e.what())), verbose_http_errors)}); + } catch (...) { + fc_elog( logger(), "Unknown Exception encountered while processing {}.{}", + api_name, call_name ); + emit(500, error_results{500, "Internal Service Error", + error_results::error_info(fc::exception( FC_LOG_MESSAGE( error, "Unknown Exception" )), verbose_http_errors)}); + } } catch (...) { - fc_elog( logger(), "Unknown Exception encountered while processing {}.{}", - api_name, call_name ); - error_results results{500, "Internal Service Error", - error_results::error_info(fc::exception( FC_LOG_MESSAGE( error, "Unknown Exception" )), verbose_http_errors)}; - cb( 500, fc::variant( results )); + std::cerr << "Exception attempting to handle exception for " << api_name << "." << call_name << std::endl; } - } catch (...) { - std::cerr << "Exception attempting to handle exception for " << api_name << "." << call_name << std::endl; } } + void http_plugin::handle_exception( const char* api_name, const char* call_name, const string& body, const url_response_callback& cb) { + classify_current_exception(api_name, call_name, body, [&cb](int code, error_results r) { + cb(code, fc::variant(std::move(r))); + }); + } + + void http_plugin::handle_exception_stream( const char* api_name, const char* call_name, const string& body, url_response_stream_callback& cb) { + classify_current_exception(api_name, call_name, body, [&cb](int code, error_results r) { + cb(code, [r = std::move(r)](fc::json_writer& w) mutable { + fc::to_json_stream(r, w); + }); + }); + } + bool http_plugin::is_on_loopback(api_category category) const { return std::all_of(my->categories_by_address.begin(), my->categories_by_address.end(), [&category, this](const auto& entry) { From 6d5c390135a4d936aa80133c98fe5c708caa8d40 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Sat, 25 Apr 2026 20:37:10 -0500 Subject: [PATCH 13/71] http_plugin: add CALL_ASYNC_WITH_400_STREAM + CALL_WITH_400_STREAM_POST macros Streaming-cb counterparts to the existing CALL_ASYNC_WITH_400 and CALL_WITH_400_POST. Same control flow as the variant versions; the only difference is how the result is delivered to cb - as a json_writer-emitting closure (move_only_function) instead of as a fc::variant. post_http_thread_pool's signature changes from void post_http_thread_pool(std::function f) to void post_http_thread_pool(std::move_only_function f) so the streaming macros can capture the move_only_function cb plus the move-only api_handle.call_name() result struct into the dispatched closure. std::function values implicitly convert to std::move_only_function, so the variant-path macros (CALL_ASYNC_WITH_400, CALL_WITH_400_POST) continue to work unchanged. The streaming macros use http_plugin::handle_exception_stream on the error paths (added in the previous commit) to emit error_results via the reflector path. No callers yet - the new macros are unused but compiled. --- .../include/sysio/http_plugin/http_plugin.hpp | 5 +- .../include/sysio/http_plugin/macros.hpp | 93 +++++++++++++++++++ plugins/http_plugin/src/http_plugin.cpp | 4 +- 3 files changed, 99 insertions(+), 3 deletions(-) diff --git a/plugins/http_plugin/include/sysio/http_plugin/http_plugin.hpp b/plugins/http_plugin/include/sysio/http_plugin/http_plugin.hpp index 742e44386f..79eae11d66 100644 --- a/plugins/http_plugin/include/sysio/http_plugin/http_plugin.hpp +++ b/plugins/http_plugin/include/sysio/http_plugin/http_plugin.hpp @@ -188,7 +188,10 @@ namespace sysio { /// request). static void handle_exception_stream( const char *api_name, const char *call_name, const string& body, url_response_stream_callback& cb ); - void post_http_thread_pool(std::function f); + // Accepts a move-only callable so streaming-cb macros can capture + // url_response_stream_callback (move_only_function) into the closure. + // std::function values implicitly convert and so are still accepted. + void post_http_thread_pool(std::move_only_function f); bool is_on_loopback(api_category category) const; diff --git a/plugins/http_plugin/include/sysio/http_plugin/macros.hpp b/plugins/http_plugin/include/sysio/http_plugin/macros.hpp index 29f846f13f..3aa7a98735 100644 --- a/plugins/http_plugin/include/sysio/http_plugin/macros.hpp +++ b/plugins/http_plugin/include/sysio/http_plugin/macros.hpp @@ -1,5 +1,8 @@ #pragma once +#include +#include + #define CALL_ASYNC_WITH_400(api_name, category, api_handle, api_namespace, call_name, call_result, http_resp_code, params_type) \ { std::string("/v1/" #api_name "/" #call_name), \ api_category::category, \ @@ -80,3 +83,93 @@ http_plugin::handle_exception(#api_name, #call_name, body, cb); \ } \ }} + + +// Streaming-cb counterpart of CALL_WITH_400_POST. Same Phase 1 (api thread) / +// Phase 2 (http thread pool) split as the variant version, but Phase 2 hands +// the typed call_result to cb as a json_writer-emitting closure - no fc::variant +// tree on the response path. +// ------------------------------------------------------------------------------------------------------ +#define CALL_WITH_400_STREAM_POST(api_name, category, api_handle, api_namespace, call_name, call_result, http_resp_code, params_type) \ +{std::string("/v1/" #api_name "/" #call_name), \ + api_category::category, \ + [api_handle, &_http_plugin](string&&, string&& body, url_response_stream_callback&& cb) { \ + auto deadline = api_handle.start(); \ + try { \ + auto params = parse_params(body); \ + using http_fwd_t = std::function()>; \ + /* called on main application thread */ \ + http_fwd_t http_fwd(api_handle.call_name(std::move(params), deadline)); \ + _http_plugin.post_http_thread_pool([resp_code=http_resp_code, cb=std::move(cb), \ + body=std::move(body), \ + http_fwd = std::move(http_fwd)]() mutable { \ + try { \ + chain::t_or_exception result = http_fwd(); \ + if (std::holds_alternative(result)) { \ + try { \ + throw *std::get(result); \ + } catch (...) { \ + http_plugin::handle_exception_stream(#api_name, #call_name, body, cb); \ + } \ + } else { \ + cb(resp_code, [r = std::get(std::move(result))] \ + (fc::json_writer& w) mutable { fc::to_json_stream(r, w); }); \ + } \ + } catch (...) { \ + http_plugin::handle_exception_stream(#api_name, #call_name, body, cb); \ + } \ + }); \ + } catch (...) { \ + http_plugin::handle_exception_stream(#api_name, #call_name, body, cb); \ + } \ + }} + + +// Streaming-cb counterpart of CALL_ASYNC_WITH_400. Same control flow; the api +// lambda hands the result struct to the cb as a json_writer-emitting closure +// instead of as a fc::variant. No per-field allocation on the response path. +// ------------------------------------------------------------------------------------------------------ +#define CALL_ASYNC_WITH_400_STREAM(api_name, category, api_handle, api_namespace, call_name, call_result, http_resp_code, params_type) \ +{ std::string("/v1/" #api_name "/" #call_name), \ + api_category::category, \ + [api_handle, &_http_plugin](string&&, string&& body, url_response_stream_callback&& cb) mutable { \ + api_handle.start(); \ + try { \ + auto params = parse_params(body); \ + using http_fwd_t = std::function()>; \ + api_handle.call_name( std::move(params), /* called on main application thread */ \ + [&_http_plugin, cb=std::move(cb), body=std::move(body)] \ + (const chain::next_function_variant& result) mutable { \ + if (std::holds_alternative(result)) { \ + try { \ + throw *std::get(result); \ + } catch (...) { \ + http_plugin::handle_exception_stream(#api_name, #call_name, body, cb); \ + } \ + } else if (std::holds_alternative(result)) { \ + cb(http_resp_code, [r = std::get(std::move(result))] \ + (fc::json_writer& w) mutable { fc::to_json_stream(r, w); }); \ + } else { \ + assert(std::holds_alternative(result)); \ + _http_plugin.post_http_thread_pool([resp_code=http_resp_code, cb=std::move(cb), \ + body=std::move(body), \ + http_fwd = std::get(std::move(result))]() mutable { \ + chain::t_or_exception result = http_fwd(); \ + if (std::holds_alternative(result)) { \ + try { \ + throw *std::get(result); \ + } catch (...) { \ + http_plugin::handle_exception_stream(#api_name, #call_name, body, cb); \ + } \ + } else { \ + cb(resp_code, [r = std::get(std::move(result))] \ + (fc::json_writer& w) mutable { fc::to_json_stream(r, w); }); \ + } \ + }); \ + } \ + }); \ + } catch (...) { \ + http_plugin::handle_exception_stream(#api_name, #call_name, body, cb); \ + } \ + } \ +} diff --git a/plugins/http_plugin/src/http_plugin.cpp b/plugins/http_plugin/src/http_plugin.cpp index 63461ec9b0..fa6cfda50b 100644 --- a/plugins/http_plugin/src/http_plugin.cpp +++ b/plugins/http_plugin/src/http_plugin.cpp @@ -625,9 +625,9 @@ namespace sysio { smap.emplace(path, my->make_http_thread_url_handler_stream(std::move(entry))); } - void http_plugin::post_http_thread_pool(std::function f) { + void http_plugin::post_http_thread_pool(std::move_only_function f) { if( f ) - boost::asio::post( my->plugin_state->thread_pool.get_executor(), f ); + boost::asio::post( my->plugin_state->thread_pool.get_executor(), std::move(f) ); } namespace { From 9787ce57c115b8ce267fee670d386b1b34d94aca Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Sat, 25 Apr 2026 20:51:17 -0500 Subject: [PATCH 14/71] chain_api_plugin: migrate /v1/chain/get_table_rows to streaming-cb path Flips get_table_rows from CHAIN_RO_CALL_POST to CHAIN_RO_CALL_STREAM_POST. The endpoint now hands the typed get_table_rows_result to the http thread pool as a json_writer-emitting closure rather than constructing a fc::variant tree. The Phase 1 / Phase 2 split (api thread captures the deferred fn, http pool runs it) is unchanged; only the response delivery path differs. ABI decode safety: get_table_rows already pre-resolves the contract's ABI inside Phase 1 by capturing the raw ABI bytes into the Phase 2 closure. Phase 2 constructs a local abi_serializer from those bytes and decodes rows into fc::variant before they get embedded into the result. None of that touches chainbase from the http thread, so the streaming pivot inherits the existing safety unchanged. Byte-identical compat: get_table_rows_result is FC_REFLECT'd ((rows)(more) (next_key)). The reflector path emits members in declaration order, the same order fc::mutable_variant_object preserves on the variant path. The rows field is vector; to_json_stream(variant) splices each row's fc::json::to_string output via raw_value, identical bytes to the variant path's array emission. Adds tests/test_get_table_rows_page.cpp::streaming_vs_variant_byte_identical which constructs a populated get_table_rows_result via the existing test-tester harness, runs it through both paths, and BOOST_CHECK_EQUALs the emitted strings. Pins parity for any future to_json_stream change. CHAIN_RO_CALL_STREAM_POST is added as a plugin-local macro alongside CHAIN_RO_CALL_POST. The variant macros stay until every endpoint flips, then get removed in a cleanup commit. --- .../chain_api_plugin/src/chain_api_plugin.cpp | 14 +++++++++- tests/test_get_table_rows_page.cpp | 26 +++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/plugins/chain_api_plugin/src/chain_api_plugin.cpp b/plugins/chain_api_plugin/src/chain_api_plugin.cpp index 4231b0cdb8..e0f2449b66 100644 --- a/plugins/chain_api_plugin/src/chain_api_plugin.cpp +++ b/plugins/chain_api_plugin/src/chain_api_plugin.cpp @@ -111,6 +111,12 @@ parse_params().chain())); @@ -141,7 +147,6 @@ void chain_api_plugin::plugin_startup() { CHAIN_RO_CALL(get_raw_code_and_abi, 200, http_params_types::params_required), CHAIN_RO_CALL(get_raw_abi, 200, http_params_types::params_required), CHAIN_RO_CALL(get_finalizer_info, 200, http_params_types::no_params), - CHAIN_RO_CALL_POST(get_table_rows, chain_apis::read_only::get_table_rows_result, 200, http_params_types::params_required), CHAIN_RO_CALL(get_table_by_scope, 200, http_params_types::params_required), CHAIN_RO_CALL(get_currency_balance, 200, http_params_types::params_required), CHAIN_RO_CALL(get_currency_stats, 200, http_params_types::params_required), @@ -157,6 +162,13 @@ void chain_api_plugin::plugin_startup() { CHAIN_RW_CALL_ASYNC(send_transaction2, chain_apis::read_write::send_transaction_results, 202, http_params_types::params_required) }, appbase::exec_queue::read_only); + // Streaming-cb endpoints. Same exec_queue and registration semantics as the + // variant-cb add_api block above; the difference is only how the response is + // delivered to the http thread pool (closure vs variant tree). + _http_plugin.add_api_stream({ + CHAIN_RO_CALL_STREAM_POST(get_table_rows, chain_apis::read_only::get_table_rows_result, 200, http_params_types::params_required), + }, appbase::exec_queue::read_only); + if (chain.account_queries_enabled()) { _http_plugin.add_async_api({ CHAIN_RO_CALL_WITH_400(get_accounts_by_authorizers, 200, http_params_types::params_required), diff --git a/tests/test_get_table_rows_page.cpp b/tests/test_get_table_rows_page.cpp index fe8cc71bc1..7e95377124 100644 --- a/tests/test_get_table_rows_page.cpp +++ b/tests/test_get_table_rows_page.cpp @@ -23,6 +23,8 @@ #include +#include +#include #include using namespace sysio; @@ -275,4 +277,28 @@ BOOST_FIXTURE_TEST_CASE(find_with_index_returns_single_match_even_when_all_rows, BOOST_CHECK_EQUAL(res.rows[0].get_object()["sec64"].as_uint64(), 3u); } FC_LOG_AND_RETHROW() +// Byte-identical compat between the variant-cb path (fc::variant + fc::json::to_string) +// and the streaming-cb path (fc::to_json_stream via reflector dispatch). The HTTP +// /v1/chain/get_table_rows endpoint flips between these via add_api / add_api_stream; +// any drift in the streaming path's emitted JSON would break clients that depend on +// the exact byte sequence (eg keypair-string ordering, integer formatting). This case +// pins parity for a representative populated result. +BOOST_FIXTURE_TEST_CASE(streaming_vs_variant_byte_identical, validating_tester) try { + deploy_contract(*this); + populate_numobjs(*this, 4); + + auto p = numobjs_params(); + p.all_rows = true; + p.show_payer = true; + + auto ro = make_read_only(*this); + auto res = run(ro, p); + + const std::string variant_path = + fc::json::to_string(fc::variant(res), fc::time_point::maximum()); + const std::string stream_path = fc::to_json_string(res); + + BOOST_CHECK_EQUAL(variant_path, stream_path); +} FC_LOG_AND_RETHROW() + BOOST_AUTO_TEST_SUITE_END() From 5af69055630f5573b1004723a06f8dde1c7dda5a Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Mon, 27 Apr 2026 10:50:48 -0500 Subject: [PATCH 15/71] libfc: add fc::serialize_as_string trait Opt-in trait for types whose JSON form is the result of T::to_string() and whose JSON parse form is T::from_string(std::string_view). Specialising to std::true_type via FC_SERIALIZE_AS_STRING(T) auto-routes fc::to_variant, fc::from_variant, and fc::to_json_stream through the type's string conversions, replacing three per-type hand-rolled overloads with one macro line. Types whose JSON shape is something else (numbers, structs) keep their hand-rolled overloads. --- .../libfc/include/fc/serialize_as_string.hpp | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 libraries/libfc/include/fc/serialize_as_string.hpp diff --git a/libraries/libfc/include/fc/serialize_as_string.hpp b/libraries/libfc/include/fc/serialize_as_string.hpp new file mode 100644 index 0000000000..92e394e05d --- /dev/null +++ b/libraries/libfc/include/fc/serialize_as_string.hpp @@ -0,0 +1,53 @@ +#pragma once + +#include +#include + +namespace fc { + +/** + * Opt-in trait for types whose JSON form is the result of T::to_string() and + * whose JSON parse form is T::from_string(std::string_view). Specializing + * this to std::true_type for a type makes all three serializers available + * without hand-rolled overloads: + * + * fc::to_variant(t, v) -> v = t.to_string() + * fc::from_variant(v, t) -> t = T::from_string(v.get_string()) + * fc::to_json_stream(t, w) -> w.value_string(t.to_string()) + * + * Use the FC_SERIALIZE_AS_STRING macro for the one-line opt-in. + * + * Requirements on the type: + * - std::string T::to_string() const (or convertible-to-string return) + * - static T T::from_string(std::string_view) + * + * Types whose to_variant is the FC_REFLECT'd struct shape do NOT specialize + * this; they fall through to the generic reflector dispatch. Types whose + * shape is something else (number, struct, ISO date, etc.) need their own + * hand-rolled overload. + */ +template +struct serialize_as_string : std::false_type {}; + +template + requires serialize_as_string::value +inline void to_variant(const T& v, fc::variant& vo) { + vo = v.to_string(); +} + +template + requires serialize_as_string::value +inline void from_variant(const fc::variant& v, T& out) { + out = T::from_string(v.get_string()); +} + +template + requires serialize_as_string::value +inline void to_json_stream(const T& v, fc::json_writer& w) { + w.value_string(v.to_string()); +} + +} // namespace fc + +#define FC_SERIALIZE_AS_STRING(TYPE) \ + namespace fc { template<> struct serialize_as_string : std::true_type {}; } From 9a9cedc4d0ba34d6de09a03e42b6f980212b9c07 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Mon, 27 Apr 2026 10:53:19 -0500 Subject: [PATCH 16/71] chain: migrate name/asset/symbol/symbol_code/block_timestamp to FC_SERIALIZE_AS_STRING Replace the per-type fc::to_variant / fc::from_variant / fc::to_json_stream triples for these five types with the trait opt-in. Adds the factory methods needed for the trait shape: - name::from_string(std::string_view) - thin wrapper over the existing string constructor; also lets the trait drop the friend declaration the old hand-rolled from_variant needed for private name::set - symbol_code::to_string and ::from_string - encapsulate the byte-extraction and validating-constructor logic that previously lived in the fc:: overloads - symbol::name() now delegates to to_symbol_code().to_string() to remove the duplicate byte-extraction loop - block_timestamp::to_string and ::from_string route through fc::time_point's iso conversions, matching the previous to_variant shape; format_as() collapses to t.to_string() now that the method exists block_timestamp is a class template, so its trait spec is a hand-written partial specialisation (the macro handles concrete types only). The other four use FC_SERIALIZE_AS_STRING. The trait specialisation must precede any code path that resolves fc::to_variant / fc::from_variant for the type; for asset that meant placing FC_SERIALIZE_AS_STRING above the extended_asset from_variant body, whose call to from_variant on the inner asset implicitly instantiates the primary trait. --- libraries/chain/include/sysio/chain/asset.hpp | 10 ++-- .../include/sysio/chain/block_timestamp.hpp | 21 +++---- libraries/chain/include/sysio/chain/name.hpp | 13 +---- .../chain/include/sysio/chain/symbol.hpp | 55 +++++++++---------- libraries/chain/name.cpp | 6 -- 5 files changed, 45 insertions(+), 60 deletions(-) diff --git a/libraries/chain/include/sysio/chain/asset.hpp b/libraries/chain/include/sysio/chain/asset.hpp index 234283d5d4..c825296c8d 100644 --- a/libraries/chain/include/sysio/chain/asset.hpp +++ b/libraries/chain/include/sysio/chain/asset.hpp @@ -1,4 +1,5 @@ #pragma once +#include #include #include #include @@ -103,12 +104,9 @@ bool operator <= (const asset& a, const asset& b); }} // namespace sysio::chain -namespace fc { -inline void to_variant(const sysio::chain::asset& var, fc::variant& vo) { vo = var.to_string(); } -inline void from_variant(const fc::variant& var, sysio::chain::asset& vo) { - vo = sysio::chain::asset::from_string(var.get_string()); -} -} +// Must precede the extended_asset from_variant body below; that body resolves +// fc::from_variant on asset, which instantiates the primary trait. +FC_SERIALIZE_AS_STRING(sysio::chain::asset) namespace fc { inline void from_variant(const fc::variant& var, sysio::chain::extended_asset& vo) { diff --git a/libraries/chain/include/sysio/chain/block_timestamp.hpp b/libraries/chain/include/sysio/chain/block_timestamp.hpp index 7e50cf9371..d19a20ea73 100644 --- a/libraries/chain/include/sysio/chain/block_timestamp.hpp +++ b/libraries/chain/include/sysio/chain/block_timestamp.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -49,6 +50,12 @@ namespace sysio { namespace chain { return fc::time_point(fc::milliseconds(msec)); } + std::string to_string() const { return to_time_point().to_iso_string(); } + + static block_timestamp from_string(std::string_view s) { + return block_timestamp(fc::time_point::from_iso_string(std::string(s))); + } + void operator = (const fc::time_point& t ) { set_time_point(t); } @@ -76,7 +83,7 @@ namespace sysio { namespace chain { typedef block_timestamp block_timestamp_type; constexpr std::string format_as(const block_timestamp_type& t) { - return t.to_time_point().to_iso_string(); + return t.to_string(); } } } /// sysio::chain @@ -93,15 +100,9 @@ namespace std { FC_REFLECT(sysio::chain::block_timestamp_type, (slot)) namespace fc { - template - void to_variant(const sysio::chain::block_timestamp& t, fc::variant& v) { - to_variant( (fc::time_point)t, v); - } - - template - void from_variant(const fc::variant& v, sysio::chain::block_timestamp& t) { - t = v.as(); - } + // Partial specialisation - FC_SERIALIZE_AS_STRING handles concrete types only. + template + struct serialize_as_string> : std::true_type {}; } #ifdef _MSC_VER diff --git a/libraries/chain/include/sysio/chain/name.hpp b/libraries/chain/include/sysio/chain/name.hpp index 4181ad792b..448205af1e 100644 --- a/libraries/chain/include/sysio/chain/name.hpp +++ b/libraries/chain/include/sysio/chain/name.hpp @@ -1,17 +1,9 @@ #pragma once #include +#include #include #include -namespace sysio::chain { - struct name; -} -namespace fc { - class variant; - void to_variant(const sysio::chain::name& c, fc::variant& v); - void from_variant(const fc::variant& v, sysio::chain::name& check); -} // fc - namespace sysio::chain { inline constexpr uint64_t char_to_symbol( char c ) { if( c >= 'a' && c <= 'z' ) @@ -45,7 +37,6 @@ namespace sysio::chain { uint64_t value = 0; friend struct fc::reflector; - friend void fc::from_variant(const fc::variant& v, sysio::chain::name& check); void set( std::string_view str ); @@ -58,6 +49,7 @@ namespace sysio::chain { constexpr name() = default; std::string to_string()const; + static name from_string( std::string_view str ) { return name(str); } constexpr uint64_t to_uint64_t()const { return value; } friend std::ostream& operator << ( std::ostream& out, const name& n ) { @@ -192,3 +184,4 @@ namespace std { }; FC_REFLECT( sysio::chain::name, (value) ) +FC_SERIALIZE_AS_STRING(sysio::chain::name) diff --git a/libraries/chain/include/sysio/chain/symbol.hpp b/libraries/chain/include/sysio/chain/symbol.hpp index 69b92dc709..599f24c360 100644 --- a/libraries/chain/include/sysio/chain/symbol.hpp +++ b/libraries/chain/include/sysio/chain/symbol.hpp @@ -1,5 +1,6 @@ #pragma once #include +#include #include #include #include @@ -47,6 +48,25 @@ namespace sysio::chain { uint64_t value; operator uint64_t()const { return value; } + + /// Decode value as a packed-byte ASCII string (low byte first), e.g. 0x535953 -> "SYS". + /// Mirrors the long form `symbol(value << 8).name()` but without going through symbol's + /// validating constructor; serialization paths must not throw on already-resident state. + std::string to_string() const { + uint64_t v = value; + std::string result; + while (v > 0) { + char c = v & 0xFF; + result += c; + v >>= 8; + } + return result; + } + + /// Parse a bare symbol-name string (e.g. "SYS") into a symbol_code. Routes through + /// symbol's precision-zero constructor, which validates that all characters are + /// uppercase letters and length <= 7. + static symbol_code from_string(std::string_view s); }; class symbol : fc::reflect_init { @@ -83,18 +103,7 @@ namespace sysio::chain { } return p10; } - string name() const - { - uint64_t v = m_value; - v >>= 8; - string result; - while (v > 0) { - char c = v & 0xFF; - result += c; - v >>= 8; - } - return result; - } + string name() const { return to_symbol_code().to_string(); } symbol_code to_symbol_code()const { return {m_value >> 8}; } @@ -125,6 +134,10 @@ namespace sysio::chain { friend struct fc::reflector; }; // class symbol + inline symbol_code symbol_code::from_string(std::string_view s) { + return symbol(0, s).to_symbol_code(); + } + struct extended_symbol { symbol sym; account_name contract; @@ -168,22 +181,8 @@ namespace sysio::chain { } } // namespace sysio::chain -namespace fc { - inline void to_variant(const sysio::chain::symbol& var, fc::variant& vo) { vo = var.to_string(); } - inline void from_variant(const fc::variant& var, sysio::chain::symbol& vo) { - vo = sysio::chain::symbol::from_string(var.get_string()); - } -} - -namespace fc { - inline void to_variant(const sysio::chain::symbol_code& var, fc::variant& vo) { - vo = sysio::chain::symbol(var.value << 8).name(); - } - inline void from_variant(const fc::variant& var, sysio::chain::symbol_code& vo) { - vo = sysio::chain::symbol(0, var.get_string()).to_symbol_code(); - } -} - FC_REFLECT(sysio::chain::symbol_code, (value)) FC_REFLECT(sysio::chain::symbol, (m_value)) FC_REFLECT(sysio::chain::extended_symbol, (sym)(contract)) +FC_SERIALIZE_AS_STRING(sysio::chain::symbol) +FC_SERIALIZE_AS_STRING(sysio::chain::symbol_code) diff --git a/libraries/chain/name.cpp b/libraries/chain/name.cpp index 6c3e065259..eb020c2b64 100644 --- a/libraries/chain/name.cpp +++ b/libraries/chain/name.cpp @@ -1,5 +1,4 @@ #include -#include #include #include #include @@ -33,8 +32,3 @@ namespace sysio::chain { } } // sysio::chain - -namespace fc { - void to_variant(const sysio::chain::name& c, fc::variant& v) { v = c.to_string(); } - void from_variant(const fc::variant& v, sysio::chain::name& check) { check.set( v.get_string() ); } -} // fc From 0f05d655426e5ab7f5487b365def04c6b4504e40 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Mon, 27 Apr 2026 10:53:30 -0500 Subject: [PATCH 17/71] chain_api_plugin: migrate /v1/chain/get_account to streaming-cb path Move the get_account registration from the variant-cb add_api block to the streaming-cb add_api_stream block. Adds a byte-identical parity test in test_chain_plugin that compares the streaming output to fc::json::to_string(fc::variant(results)) - pins that the two paths stay equivalent as future endpoints migrate. --- .../chain_api_plugin/src/chain_api_plugin.cpp | 2 +- tests/test_chain_plugin.cpp | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/plugins/chain_api_plugin/src/chain_api_plugin.cpp b/plugins/chain_api_plugin/src/chain_api_plugin.cpp index e0f2449b66..1e44d3fa85 100644 --- a/plugins/chain_api_plugin/src/chain_api_plugin.cpp +++ b/plugins/chain_api_plugin/src/chain_api_plugin.cpp @@ -139,7 +139,6 @@ void chain_api_plugin::plugin_startup() { CHAIN_RO_CALL_POST(get_block, fc::variant, 200, http_params_types::params_required), // _POST because get_block() returns a lambda to be executed on the http thread pool CHAIN_RO_CALL(get_block_info, 200, http_params_types::params_required), CHAIN_RO_CALL(get_block_header_state, 200, http_params_types::params_required), - CHAIN_RO_CALL_POST(get_account, chain_apis::read_only::get_account_results, 200, http_params_types::params_required), CHAIN_RO_CALL(get_code, 200, http_params_types::params_required), CHAIN_RO_CALL(get_code_hash, 200, http_params_types::params_required), CHAIN_RO_CALL(get_consensus_parameters, 200, http_params_types::no_params), @@ -166,6 +165,7 @@ void chain_api_plugin::plugin_startup() { // variant-cb add_api block above; the difference is only how the response is // delivered to the http thread pool (closure vs variant tree). _http_plugin.add_api_stream({ + CHAIN_RO_CALL_STREAM_POST(get_account, chain_apis::read_only::get_account_results, 200, http_params_types::params_required), CHAIN_RO_CALL_STREAM_POST(get_table_rows, chain_apis::read_only::get_table_rows_result, 200, http_params_types::params_required), }, appbase::exec_queue::read_only); diff --git a/tests/test_chain_plugin.cpp b/tests/test_chain_plugin.cpp index 92a5f0e12b..ef9a9ea38d 100644 --- a/tests/test_chain_plugin.cpp +++ b/tests/test_chain_plugin.cpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include #include #include @@ -137,4 +139,24 @@ BOOST_FIXTURE_TEST_CASE(account_results_total_resources_test, chain_plugin_teste } FC_LOG_AND_RETHROW() } +// Byte-identical compat between the variant-cb path (fc::variant + +// fc::json::to_string) and the streaming-cb path (fc::to_json_stream via +// reflector dispatch + per-type overrides for chain::name / asset / symbol). +// Pins the parity that get_account's add_api_stream registration depends on. +BOOST_FIXTURE_TEST_CASE(get_account_streaming_vs_variant_byte_identical, chain_plugin_tester) { try { + produce_blocks(10); + setup_system_accounts(); + produce_blocks(); + create_account("alice1111111"_n, config::system_account_name); + transfer(name("sysio"), name("alice1111111"), core_from_string("650000000.0000"), name("sysio")); + + read_only::get_account_results results = get_account_info(name("alice1111111")); + + const std::string variant_path = + fc::json::to_string(fc::variant(results), fc::time_point::maximum()); + const std::string stream_path = fc::to_json_string(results); + + BOOST_CHECK_EQUAL(variant_path, stream_path); +} FC_LOG_AND_RETHROW() } + BOOST_AUTO_TEST_SUITE_END() From f28d665778287e722c7ce490e0ed3901ce8ff1fc Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Mon, 27 Apr 2026 13:32:25 -0500 Subject: [PATCH 18/71] libfc: add FC_SERIALIZE_AS_STRING_TEMPLATE for class-template types The plain macro only handles concrete types because explicit specialisations cannot carry template parameter packs. The new variant takes a parenthesised template-parameter-list and a parenthesised type expression, unwrapping each via FC_SERIALIZE_AS_STRING_UNPAREN_: FC_SERIALIZE_AS_STRING_TEMPLATE((typename T, typename U), (my_type)) Replaces the hand-written partial specialisation that block_timestamp's trait declaration needed. --- .../chain/include/sysio/chain/block_timestamp.hpp | 7 ++----- libraries/libfc/include/fc/serialize_as_string.hpp | 10 ++++++++++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/libraries/chain/include/sysio/chain/block_timestamp.hpp b/libraries/chain/include/sysio/chain/block_timestamp.hpp index d19a20ea73..a98690ae36 100644 --- a/libraries/chain/include/sysio/chain/block_timestamp.hpp +++ b/libraries/chain/include/sysio/chain/block_timestamp.hpp @@ -99,11 +99,8 @@ namespace std { #include FC_REFLECT(sysio::chain::block_timestamp_type, (slot)) -namespace fc { - // Partial specialisation - FC_SERIALIZE_AS_STRING handles concrete types only. - template - struct serialize_as_string> : std::true_type {}; -} +FC_SERIALIZE_AS_STRING_TEMPLATE((uint16_t IntervalMs, uint64_t EpochMs), + (sysio::chain::block_timestamp)) #ifdef _MSC_VER #pragma warning (pop) diff --git a/libraries/libfc/include/fc/serialize_as_string.hpp b/libraries/libfc/include/fc/serialize_as_string.hpp index 92e394e05d..e9b02c54df 100644 --- a/libraries/libfc/include/fc/serialize_as_string.hpp +++ b/libraries/libfc/include/fc/serialize_as_string.hpp @@ -51,3 +51,13 @@ inline void to_json_stream(const T& v, fc::json_writer& w) { #define FC_SERIALIZE_AS_STRING(TYPE) \ namespace fc { template<> struct serialize_as_string : std::true_type {}; } + +// Class-template variant of FC_SERIALIZE_AS_STRING. Wrap each argument in parens +// so internal commas (template parameter lists, template arguments) survive the +// preprocessor's argument splitting. +// FC_SERIALIZE_AS_STRING_TEMPLATE((typename T, typename U), (my_type)) +#define FC_SERIALIZE_AS_STRING_TEMPLATE(TPL_PARAMS, TYPE) \ + namespace fc { template \ + struct serialize_as_string : std::true_type {}; } + +#define FC_SERIALIZE_AS_STRING_UNPAREN_(...) __VA_ARGS__ From ec9166a6bfb6a04daa29cbd9a8eab726ab4c5562 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Mon, 27 Apr 2026 13:32:39 -0500 Subject: [PATCH 19/71] libfc: migrate sha1/sha224/sha256/sha512/sha3/ripemd160/keccak256/blake3 to FC_SERIALIZE_AS_STRING Each hash already produced a hex-string variant through the to_variant(vector) path, so the trait migration is JSON- and variant-type-equivalent. Per type: - to_string() and static from_string(string_view) added to satisfy the trait shape. - The hand-rolled fc::to_variant / fc::from_variant overloads are dropped (and sha256 also drops the hand-rolled to_json_stream now that the trait provides one). - FC_SERIALIZE_AS_STRING(...) opt-in declared after the FC_REFLECT_TYPENAME line. sha1/sha224/sha256/sha512/sha3/ripemd160 constructors widen to std::string_view directly since fc::from_hex(string_view, char*, size_t) already takes that shape. keccak256/blake3 use the vector-returning from_hex; that overload (and trim_hex_prefix) is widened to std::string_view too, which removes the std::string materialisation those constructors previously paid. Tests at libfc/test/crypto/test_hash_functions.cpp cover the trait round-trip (ctor / str / from_string / to_variant + from_variant / to_json_string) for all eight types. Note: libfc/test is gated on ENABLE_TESTS, OFF by default in this repo's standard build dir; reconfigure with -DENABLE_TESTS=ON to run them. --- libraries/libfc/include/fc/crypto/blake3.hpp | 17 ++-- libraries/libfc/include/fc/crypto/hex.hpp | 4 +- .../libfc/include/fc/crypto/keccak256.hpp | 10 +-- .../libfc/include/fc/crypto/ripemd160.hpp | 11 +-- libraries/libfc/include/fc/crypto/sha1.hpp | 11 +-- libraries/libfc/include/fc/crypto/sha224.hpp | 10 +-- libraries/libfc/include/fc/crypto/sha256.hpp | 12 ++- libraries/libfc/include/fc/crypto/sha3.hpp | 10 +-- libraries/libfc/include/fc/crypto/sha512.hpp | 10 +-- libraries/libfc/src/crypto/hex.cpp | 28 +++--- libraries/libfc/src/crypto/keccak256.cpp | 16 +--- libraries/libfc/src/crypto/ripemd160.cpp | 17 +--- libraries/libfc/src/crypto/sha1.cpp | 17 +--- libraries/libfc/src/crypto/sha224.cpp | 17 +--- libraries/libfc/src/crypto/sha256.cpp | 26 +----- libraries/libfc/src/crypto/sha3.cpp | 14 +-- libraries/libfc/src/crypto/sha512.cpp | 17 +--- .../libfc/test/crypto/test_hash_functions.cpp | 85 +++++++++++++++++++ 18 files changed, 152 insertions(+), 180 deletions(-) diff --git a/libraries/libfc/include/fc/crypto/blake3.hpp b/libraries/libfc/include/fc/crypto/blake3.hpp index 11a210c949..da6e0c5ad2 100644 --- a/libraries/libfc/include/fc/crypto/blake3.hpp +++ b/libraries/libfc/include/fc/crypto/blake3.hpp @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include @@ -25,7 +25,7 @@ class blake3 { blake3() { memset(_hash, 0, sizeof(_hash)); } - explicit blake3(const std::string& hex_str) { + explicit blake3(std::string_view hex_str) { auto bytes = from_hex(hex_str); FC_ASSERT(bytes.size() == sizeof(_hash), "Invalid blake3 hex string length: {}", hex_str.size()); @@ -33,6 +33,8 @@ class blake3 { } std::string str() const { return to_hex(to_char_span()); } + std::string to_string() const { return str(); } + static blake3 from_string(std::string_view s) { return blake3(s); } uint8_t* data() { return _hash; } const uint8_t* data() const { return _hash; } @@ -73,17 +75,8 @@ class blake3 { } // namespace crypto -inline void to_variant( const crypto::blake3& bi, variant& v ) { - v = std::vector( bi.char_data(), bi.char_data() + bi.data_size() ); -} - -inline void from_variant( const variant& v, crypto::blake3& bi ) { - std::vector ve = v.as< std::vector >(); - FC_ASSERT(ve.size() == crypto::blake3::byte_size, "Invalid blake3 data size: {}", ve.size()); - memcpy(bi.char_data(), ve.data(), crypto::blake3::byte_size); -} - } // namespace fc #include FC_REFLECT_TYPENAME(fc::crypto::blake3) +FC_SERIALIZE_AS_STRING(fc::crypto::blake3) diff --git a/libraries/libfc/include/fc/crypto/hex.hpp b/libraries/libfc/include/fc/crypto/hex.hpp index ce0a18a9fc..a54e2685cb 100644 --- a/libraries/libfc/include/fc/crypto/hex.hpp +++ b/libraries/libfc/include/fc/crypto/hex.hpp @@ -33,9 +33,9 @@ namespace fc { size_t from_hex(std::string_view hex_str, char* out_data, size_t out_data_len); - std::vector from_hex(const std::string& hex, bool trim_prefix = true); + std::vector from_hex(std::string_view hex, bool trim_prefix = true); - std::string trim_hex_prefix(const std::string& hex); + std::string_view trim_hex_prefix(std::string_view hex); /** diff --git a/libraries/libfc/include/fc/crypto/keccak256.hpp b/libraries/libfc/include/fc/crypto/keccak256.hpp index 8d4886d8e8..c518cfe73e 100644 --- a/libraries/libfc/include/fc/crypto/keccak256.hpp +++ b/libraries/libfc/include/fc/crypto/keccak256.hpp @@ -6,6 +6,7 @@ #include #include #include +#include namespace fc { namespace crypto { @@ -19,10 +20,12 @@ class keccak256 { static constexpr size_t byte_size = 256 / (8 * sizeof(uint8_t)); keccak256(); - explicit keccak256(const std::string& hex_str); + explicit keccak256(std::string_view hex_str); // in hex std::string str() const; + std::string to_string() const { return str(); } + static keccak256 from_string(std::string_view s) { return keccak256(s); } const uint8_t* data() const { return _hash; } constexpr size_t data_size() const { return byte_size; } @@ -71,11 +74,8 @@ class keccak256 { }; } // namespace crypto -class variant; -void to_variant(const crypto::keccak256& bi, variant& v); -void from_variant(const variant& v, crypto::keccak256& bi); - } // namespace fc #include FC_REFLECT_TYPENAME(fc::crypto::keccak256) +FC_SERIALIZE_AS_STRING(fc::crypto::keccak256) diff --git a/libraries/libfc/include/fc/crypto/ripemd160.hpp b/libraries/libfc/include/fc/crypto/ripemd160.hpp index 7a0b231fbd..8df0c87a0a 100644 --- a/libraries/libfc/include/fc/crypto/ripemd160.hpp +++ b/libraries/libfc/include/fc/crypto/ripemd160.hpp @@ -4,6 +4,7 @@ #include #include #include +#include namespace fc{ class sha512; @@ -13,9 +14,11 @@ class ripemd160 : public add_packhash_to_hash { public: ripemd160(); - explicit ripemd160( const std::string& hex_str ); + explicit ripemd160( std::string_view hex_str ); std::string str()const; + std::string to_string() const { return str(); } + static ripemd160 from_string(std::string_view s) { return ripemd160(s); } char* data()const; size_t data_size()const { return 160/8; } @@ -69,10 +72,6 @@ class ripemd160 : public add_packhash_to_hash uint32_t _hash[5]; }; - class variant; - void to_variant( const ripemd160& bi, variant& v ); - void from_variant( const variant& v, ripemd160& bi ); - typedef ripemd160 uint160_t; typedef ripemd160 uint160; @@ -91,3 +90,5 @@ namespace std } }; } + +FC_SERIALIZE_AS_STRING(fc::ripemd160) diff --git a/libraries/libfc/include/fc/crypto/sha1.hpp b/libraries/libfc/include/fc/crypto/sha1.hpp index 080ae886e6..3af68e3ba0 100644 --- a/libraries/libfc/include/fc/crypto/sha1.hpp +++ b/libraries/libfc/include/fc/crypto/sha1.hpp @@ -2,6 +2,7 @@ #include #include #include +#include namespace fc{ @@ -9,9 +10,11 @@ class sha1 : public add_packhash_to_hash { public: sha1(); - explicit sha1( const std::string& hex_str ); + explicit sha1( std::string_view hex_str ); std::string str()const; + std::string to_string() const { return str(); } + static sha1 from_string(std::string_view s) { return sha1(s); } operator std::string()const; char* data(); @@ -65,10 +68,6 @@ class sha1 : public add_packhash_to_hash uint32_t _hash[5]; }; - class variant; - void to_variant( const sha1& bi, variant& v ); - void from_variant( const variant& v, sha1& bi ); - } // namespace fc namespace std @@ -82,3 +81,5 @@ namespace std } }; } + +FC_SERIALIZE_AS_STRING(fc::sha1) diff --git a/libraries/libfc/include/fc/crypto/sha224.hpp b/libraries/libfc/include/fc/crypto/sha224.hpp index f9c73b8a6a..9985f25191 100644 --- a/libraries/libfc/include/fc/crypto/sha224.hpp +++ b/libraries/libfc/include/fc/crypto/sha224.hpp @@ -4,6 +4,7 @@ #include #include #include +#include namespace fc { @@ -12,9 +13,11 @@ class sha224 : public add_packhash_to_hash { public: sha224(); - explicit sha224( const std::string& hex_str ); + explicit sha224( std::string_view hex_str ); std::string str()const; + std::string to_string() const { return str(); } + static sha224 from_string(std::string_view s) { return sha224(s); } operator std::string()const; char* data(); @@ -70,10 +73,6 @@ class sha224 : public add_packhash_to_hash uint32_t _hash[7]; }; - class variant; - void to_variant( const sha224& bi, variant& v ); - void from_variant( const variant& v, sha224& bi ); - } // fc namespace std { @@ -88,3 +87,4 @@ namespace std } #include FC_REFLECT_TYPENAME( fc::sha224 ) +FC_SERIALIZE_AS_STRING(fc::sha224) diff --git a/libraries/libfc/include/fc/crypto/sha256.hpp b/libraries/libfc/include/fc/crypto/sha256.hpp index c55f18e1f8..1a34100696 100644 --- a/libraries/libfc/include/fc/crypto/sha256.hpp +++ b/libraries/libfc/include/fc/crypto/sha256.hpp @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include namespace fc @@ -23,11 +23,13 @@ class sha256 : public add_packhash_to_hash using byte_array_type = std::array; sha256(); - explicit sha256( const std::string& hex_str ); + explicit sha256( std::string_view hex_str ); explicit sha256( const hash_array_type& hash_arr ); explicit sha256( const char *data, size_t size ); std::string str()const; + std::string to_string() const { return str(); } + static sha256 from_string(std::string_view s) { return sha256(s); } std::string short_id()const; const char* data()const; @@ -129,11 +131,6 @@ class sha256 : public add_packhash_to_hash uint64_t _hash[uint64_size]; }; -class variant; -void to_variant( const sha256& bi, variant& v ); -void from_variant( const variant& v, sha256& bi ); -void to_json_stream( const sha256& bi, json_writer& w ); - uint64_t hash64(const char* buf, size_t len); constexpr auto format_as(const fc::sha256& h) { @@ -180,3 +177,4 @@ namespace fc { #include FC_REFLECT_TYPENAME( fc::sha256 ) +FC_SERIALIZE_AS_STRING(fc::sha256) diff --git a/libraries/libfc/include/fc/crypto/sha3.hpp b/libraries/libfc/include/fc/crypto/sha3.hpp index 32a8972040..1c484c8211 100644 --- a/libraries/libfc/include/fc/crypto/sha3.hpp +++ b/libraries/libfc/include/fc/crypto/sha3.hpp @@ -4,6 +4,7 @@ #include #include #include +#include #include namespace fc @@ -14,10 +15,12 @@ class sha3 public: sha3(); ~sha3(){} - explicit sha3(const std::string &hex_str); + explicit sha3(std::string_view hex_str); explicit sha3(const char *data, size_t size); std::string str() const; + std::string to_string() const { return str(); } + static sha3 from_string(std::string_view s) { return sha3(s); } operator std::string() const; const char *data() const; @@ -100,10 +103,6 @@ class sha3 uint64_t _hash[4]; }; -class variant; -void to_variant(const sha3 &bi, variant &v); -void from_variant(const variant &v, sha3 &bi); - } // namespace fc namespace std @@ -132,3 +131,4 @@ struct hash } // namespace boost #include FC_REFLECT_TYPENAME(fc::sha3) +FC_SERIALIZE_AS_STRING(fc::sha3) diff --git a/libraries/libfc/include/fc/crypto/sha512.hpp b/libraries/libfc/include/fc/crypto/sha512.hpp index 060bd7da2d..5b2c3489bd 100644 --- a/libraries/libfc/include/fc/crypto/sha512.hpp +++ b/libraries/libfc/include/fc/crypto/sha512.hpp @@ -2,6 +2,7 @@ #include #include #include +#include namespace fc { @@ -10,9 +11,11 @@ class sha512 : public add_packhash_to_hash { public: sha512(); - explicit sha512( const std::string& hex_str ); + explicit sha512( std::string_view hex_str ); std::string str()const; + std::string to_string() const { return str(); } + static sha512 from_string(std::string_view s) { return sha512(s); } operator std::string()const; char* data(); @@ -68,11 +71,8 @@ class sha512 : public add_packhash_to_hash typedef fc::sha512 uint512; - class variant; - void to_variant( const sha512& bi, variant& v ); - void from_variant( const variant& v, sha512& bi ); - } // fc #include FC_REFLECT_TYPENAME( fc::sha512 ) +FC_SERIALIZE_AS_STRING(fc::sha512) diff --git a/libraries/libfc/src/crypto/hex.cpp b/libraries/libfc/src/crypto/hex.cpp index be53018772..43e2034e77 100644 --- a/libraries/libfc/src/crypto/hex.cpp +++ b/libraries/libfc/src/crypto/hex.cpp @@ -42,13 +42,8 @@ size_t from_hex(std::string_view hex_str, char* out_data, size_t out_data_len) { } -std::vector from_hex(const std::string& hex, bool trim_prefix) { - auto cleaned_hex = trim_prefix ? trim_hex_prefix(hex) : hex; - if (cleaned_hex.size() % 2) { - cleaned_hex = "0" + cleaned_hex; - } - std::vector out; - out.reserve(cleaned_hex.size() / 2); +std::vector from_hex(std::string_view hex, bool trim_prefix) { + if (trim_prefix) hex = trim_hex_prefix(hex); auto from_hex = [](char c) -> int { if (c >= '0' && c <= '9') @@ -60,18 +55,25 @@ std::vector from_hex(const std::string& hex, bool trim_prefix) { throw std::runtime_error("Invalid hex char"); }; - for (size_t i = 0; i < cleaned_hex.size(); i += 2) { - uint8_t byte = - (from_hex(cleaned_hex[i]) << 4) | from_hex(cleaned_hex[i + 1]); - out.push_back(byte); + std::vector out; + out.reserve((hex.size() + 1) / 2); + + size_t i = 0; + if (hex.size() % 2) { + // Odd-length hex: the first nibble decodes to a single byte with high nibble = 0. + out.push_back(static_cast(from_hex(hex[0]))); + i = 1; + } + for (; i < hex.size(); i += 2) { + out.push_back(static_cast((from_hex(hex[i]) << 4) | from_hex(hex[i + 1]))); } return out; } -std::string trim_hex_prefix(const std::string& hex) { +std::string_view trim_hex_prefix(std::string_view hex) { if (hex.starts_with("0x") || hex.starts_with("0X")) { - return hex.substr(2); + hex.remove_prefix(2); } return hex; } diff --git a/libraries/libfc/src/crypto/keccak256.cpp b/libraries/libfc/src/crypto/keccak256.cpp index 6a7c1b47d3..fc4ac97838 100644 --- a/libraries/libfc/src/crypto/keccak256.cpp +++ b/libraries/libfc/src/crypto/keccak256.cpp @@ -10,7 +10,7 @@ keccak256::keccak256() { memset(_hash, 0, sizeof(_hash)); } -keccak256::keccak256(const std::string& hex_str) { +keccak256::keccak256(std::string_view hex_str) { auto bytes = from_hex(hex_str); FC_ASSERT(bytes.size() == sizeof(_hash), "Invalid keccak256 hex string length: {}", hex_str.size()); @@ -59,16 +59,4 @@ void keccak256::encoder::reset() { data.clear(); } -} // namespace fc::crypto - -namespace fc { - -void to_variant(const crypto::keccak256& bi, variant& v) { - v = bi.str(); -} - -void from_variant(const variant& v, crypto::keccak256& bi) { - bi = crypto::keccak256(v.as_string()); -} - -} // namespace fc \ No newline at end of file +} // namespace fc::crypto \ No newline at end of file diff --git a/libraries/libfc/src/crypto/ripemd160.cpp b/libraries/libfc/src/crypto/ripemd160.cpp index 93bc4bead8..4216bf1406 100644 --- a/libraries/libfc/src/crypto/ripemd160.cpp +++ b/libraries/libfc/src/crypto/ripemd160.cpp @@ -14,7 +14,7 @@ namespace fc { ripemd160::ripemd160() { memset( _hash, 0, sizeof(_hash) ); } -ripemd160::ripemd160( const std::string& hex_str ) { +ripemd160::ripemd160( std::string_view hex_str ) { auto bytes_written = fc::from_hex( hex_str, (char*)_hash, sizeof(_hash) ); if( bytes_written < sizeof(_hash) ) memset( (char*)_hash + bytes_written, 0, (sizeof(_hash) - bytes_written) ); @@ -98,19 +98,4 @@ bool operator == ( const ripemd160& h1, const ripemd160& h2 ) { return memcmp( h1._hash, h2._hash, sizeof(h1._hash) ) == 0; } - void to_variant( const ripemd160& bi, variant& v ) - { - v = std::vector( (const char*)&bi, ((const char*)&bi) + sizeof(bi) ); - } - void from_variant( const variant& v, ripemd160& bi ) - { - std::vector ve = v.as< std::vector >(); - if( ve.size() ) - { - memcpy(bi.data(), ve.data(), fc::min(ve.size(),sizeof(bi)) ); - } - else - memset( bi.data(), char(0), sizeof(bi) ); - } - } // fc diff --git a/libraries/libfc/src/crypto/sha1.cpp b/libraries/libfc/src/crypto/sha1.cpp index f95a15a353..2dcab62e70 100644 --- a/libraries/libfc/src/crypto/sha1.cpp +++ b/libraries/libfc/src/crypto/sha1.cpp @@ -11,7 +11,7 @@ namespace fc { sha1::sha1() { memset( _hash, 0, sizeof(_hash) ); } -sha1::sha1( const std::string& hex_str ) { +sha1::sha1( std::string_view hex_str ) { auto bytes_written = fc::from_hex( hex_str, (char*)_hash, sizeof(_hash) ); if( bytes_written < sizeof(_hash) ) memset( (char*)_hash + bytes_written, 0, (sizeof(_hash) - bytes_written) ); @@ -86,19 +86,4 @@ bool operator == ( const sha1& h1, const sha1& h2 ) { return memcmp( h1._hash, h2._hash, sizeof(h1._hash) ) == 0; } - void to_variant( const sha1& bi, variant& v ) - { - v = std::vector( (const char*)&bi, ((const char*)&bi) + sizeof(bi) ); - } - void from_variant( const variant& v, sha1& bi ) - { - std::vector ve = v.as< std::vector >(); - if( ve.size() ) - { - memcpy(bi.data(), ve.data(), fc::min(ve.size(),sizeof(bi)) ); - } - else - memset( bi.data(), char(0), sizeof(bi) ); - } - } // fc diff --git a/libraries/libfc/src/crypto/sha224.cpp b/libraries/libfc/src/crypto/sha224.cpp index 8aa9ceaefc..6c8991c030 100644 --- a/libraries/libfc/src/crypto/sha224.cpp +++ b/libraries/libfc/src/crypto/sha224.cpp @@ -10,7 +10,7 @@ namespace fc { sha224::sha224() { memset( _hash, 0, sizeof(_hash) ); } - sha224::sha224( const std::string& hex_str ) { + sha224::sha224( std::string_view hex_str ) { auto bytes_written = fc::from_hex( hex_str, (char*)_hash, sizeof(_hash) ); if( bytes_written < sizeof(_hash) ) memset( (char*)_hash + bytes_written, 0, (sizeof(_hash) - bytes_written) ); @@ -81,21 +81,6 @@ namespace fc { return memcmp( h1._hash, h2._hash, sizeof(sha224) ) == 0; } - void to_variant( const sha224& bi, variant& v ) - { - v = std::vector( (const char*)&bi, ((const char*)&bi) + sizeof(bi) ); - } - void from_variant( const variant& v, sha224& bi ) - { - std::vector ve = v.as< std::vector >(); - if( ve.size() ) - { - memcpy(bi.data(), ve.data(), fc::min(ve.size(),sizeof(bi)) ); - } - else - memset( bi.data(), char(0), sizeof(bi) ); - } - template<> unsigned int hmac::internal_block_size() const { return 64; } } diff --git a/libraries/libfc/src/crypto/sha256.cpp b/libraries/libfc/src/crypto/sha256.cpp index 51160eeb7d..1ff206dda0 100644 --- a/libraries/libfc/src/crypto/sha256.cpp +++ b/libraries/libfc/src/crypto/sha256.cpp @@ -5,8 +5,6 @@ #include #include #include -#include -#include #include #include "_digest_common.hpp" @@ -18,7 +16,7 @@ namespace fc { FC_THROW_EXCEPTION( exception, "sha256: size mismatch" ); memcpy(_hash, data, size ); } - sha256::sha256( const std::string& hex_str ) { + sha256::sha256( std::string_view hex_str ) { auto bytes_written = fc::from_hex( hex_str, (char*)_hash, sizeof(_hash) ); if( bytes_written < sizeof(_hash) ) memset( (char*)_hash + bytes_written, 0, (sizeof(_hash) - bytes_written) ); @@ -199,28 +197,6 @@ namespace fc { return lzbits; } - void to_variant( const sha256& bi, variant& v ) - { - v = std::vector( (const char*)&bi, ((const char*)&bi) + sizeof(bi) ); - } - void from_variant( const variant& v, sha256& bi ) - { - std::vector ve = v.as< std::vector >(); - if( ve.size() ) - { - memcpy(bi.data(), ve.data(), fc::min(ve.size(),sizeof(bi)) ); - } - else - memset( bi.data(), char(0), sizeof(bi) ); - } - void to_json_stream( const sha256& bi, json_writer& w ) - { - // Emit the canonical lowercase hex form. The existing to_variant stores raw bytes - // and lets fc::json::to_string base16-encode the blob at serialize time; this path - // shortcuts that by writing the hex string directly into the output buffer. - w.value_string( bi.str() ); - } - uint64_t hash64(const char* buf, size_t len) { sha256 sha_value = sha256::hash(buf,len); diff --git a/libraries/libfc/src/crypto/sha3.cpp b/libraries/libfc/src/crypto/sha3.cpp index 2856ee75de..6957b7b9b0 100644 --- a/libraries/libfc/src/crypto/sha3.cpp +++ b/libraries/libfc/src/crypto/sha3.cpp @@ -191,7 +191,7 @@ sha3::sha3(const char *data, size_t size) FC_THROW_EXCEPTION(exception, "sha3: size mismatch"); memcpy(_hash, data, size); } -sha3::sha3(const std::string &hex_str) +sha3::sha3(std::string_view hex_str) { auto bytes_written = fc::from_hex(hex_str, (char *)_hash, sizeof(_hash)); if (bytes_written < sizeof(_hash)) @@ -281,16 +281,4 @@ bool operator==(const sha3 &h1, const sha3 &h2) h1._hash[3] == h2._hash[3]; } -void to_variant(const sha3 &bi, variant &v) -{ - v = std::vector((const char *)&bi, ((const char *)&bi) + sizeof(bi)); -} -void from_variant(const variant &v, sha3 &bi) -{ - const auto &ve = v.as>(); - if (ve.size()) - memcpy(bi.data(), ve.data(), fc::min(ve.size(), sizeof(bi))); - else - memset(bi.data(), char(0), sizeof(bi)); -} } // namespace fc diff --git a/libraries/libfc/src/crypto/sha512.cpp b/libraries/libfc/src/crypto/sha512.cpp index 2a87d68b46..a7980dbb19 100644 --- a/libraries/libfc/src/crypto/sha512.cpp +++ b/libraries/libfc/src/crypto/sha512.cpp @@ -10,7 +10,7 @@ namespace fc { sha512::sha512() { memset( _hash, 0, sizeof(_hash) ); } - sha512::sha512( const std::string& hex_str ) { + sha512::sha512( std::string_view hex_str ) { auto bytes_written = fc::from_hex( hex_str, (char*)_hash, sizeof(_hash) ); if( bytes_written < sizeof(_hash) ) memset( (char*)_hash + bytes_written, 0, (sizeof(_hash) - bytes_written) ); @@ -88,21 +88,6 @@ namespace fc { return memcmp( h1._hash, h2._hash, sizeof(h1._hash) ) == 0; } - void to_variant( const sha512& bi, variant& v ) - { - v = std::vector( (const char*)&bi, ((const char*)&bi) + sizeof(bi) ); - } - void from_variant( const variant& v, sha512& bi ) - { - std::vector ve = v.as< std::vector >(); - if( ve.size() ) - { - memcpy(bi.data(), ve.data(), fc::min(ve.size(),sizeof(bi)) ); - } - else - memset( bi.data(), char(0), sizeof(bi) ); - } - template<> unsigned int hmac::internal_block_size() const { return 128; } } diff --git a/libraries/libfc/test/crypto/test_hash_functions.cpp b/libraries/libfc/test/crypto/test_hash_functions.cpp index a77b927256..0bc43e0a88 100644 --- a/libraries/libfc/test/crypto/test_hash_functions.cpp +++ b/libraries/libfc/test/crypto/test_hash_functions.cpp @@ -1,11 +1,49 @@ #include #include +#include +#include +#include #include +#include +#include +#include +#include +#include +#include #include using namespace fc; +namespace { + // Round-trip exercise for FC_SERIALIZE_AS_STRING-trait hash types. Verifies that: + // - Construction from a hex string round-trips through str() / to_string(). + // - The static T::from_string(string_view) factory matches construction. + // - The trait-routed fc::to_variant + fc::from_variant round-trip preserves the value + // and the variant's payload is the expected hex form. + // - fc::to_json_string emits the hex inside JSON quotes. + template + void check_hash_string_roundtrip(std::string_view hex) { + const Hash h{ hex }; + BOOST_CHECK_EQUAL(h.str(), hex); + BOOST_CHECK_EQUAL(h.to_string(), hex); + + const Hash h2 = Hash::from_string(hex); + BOOST_CHECK(h == h2); + + fc::variant v; + fc::to_variant(h, v); + BOOST_CHECK_EQUAL(v.as_string(), hex); + + Hash h3; + fc::from_variant(v, h3); + BOOST_CHECK(h == h3); + + const std::string expected_json = std::string{ "\"" } + std::string{ hex } + "\""; + BOOST_CHECK_EQUAL(fc::to_json_string(h), expected_json); + } +} + BOOST_AUTO_TEST_SUITE(hash_functions) BOOST_AUTO_TEST_CASE(sha3) try { @@ -66,4 +104,51 @@ BOOST_AUTO_TEST_CASE(keccak256) try { } FC_LOG_AND_RETHROW(); +BOOST_AUTO_TEST_CASE(sha1_string_roundtrip) try { + // 20 bytes / 40 hex chars + check_hash_string_roundtrip("0123456789abcdef0123456789abcdef01234567"); +} FC_LOG_AND_RETHROW(); + +BOOST_AUTO_TEST_CASE(sha224_string_roundtrip) try { + // 28 bytes / 56 hex chars + check_hash_string_roundtrip( + "0123456789abcdef0123456789abcdef0123456789abcdef01234567"); +} FC_LOG_AND_RETHROW(); + +BOOST_AUTO_TEST_CASE(sha256_string_roundtrip) try { + // 32 bytes / 64 hex chars + check_hash_string_roundtrip( + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"); +} FC_LOG_AND_RETHROW(); + +BOOST_AUTO_TEST_CASE(sha512_string_roundtrip) try { + // 64 bytes / 128 hex chars + check_hash_string_roundtrip( + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + "fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210"); +} FC_LOG_AND_RETHROW(); + +BOOST_AUTO_TEST_CASE(sha3_string_roundtrip) try { + // 32 bytes / 64 hex chars + check_hash_string_roundtrip( + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"); +} FC_LOG_AND_RETHROW(); + +BOOST_AUTO_TEST_CASE(ripemd160_string_roundtrip) try { + // 20 bytes / 40 hex chars + check_hash_string_roundtrip("0123456789abcdef0123456789abcdef01234567"); +} FC_LOG_AND_RETHROW(); + +BOOST_AUTO_TEST_CASE(keccak256_string_roundtrip) try { + // 32 bytes / 64 hex chars + check_hash_string_roundtrip( + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"); +} FC_LOG_AND_RETHROW(); + +BOOST_AUTO_TEST_CASE(blake3_string_roundtrip) try { + // 32 bytes / 64 hex chars + check_hash_string_roundtrip( + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"); +} FC_LOG_AND_RETHROW(); + BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file From 7249313a0102f3a7de96650a5316d6855198bec2 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Mon, 27 Apr 2026 14:39:59 -0500 Subject: [PATCH 20/71] libfc: migrate bls public_key/signature/aggregate_signature/private_key to FC_SERIALIZE_AS_STRING Each BLS type already produced a string variant via its hand-rolled to_variant -> .to_string() path, so the trait migration is JSON-equivalent. Per type: - static T from_string(string_view) added (just calls the corresponding ctor; to_string() already exists with the trait shape). - The hand-rolled fc::to_variant / fc::from_variant overloads in headers and .cpps are dropped. - FC_SERIALIZE_AS_STRING(...) opt-in declared in each header. The four BLS string-form constructors (public_key, signature, aggregate_signature, private_key) widen from const std::string& to std::string_view to avoid a materialisation in trait dispatch. That cascades through the helpers they call - deserialize_bls_base64url, sig_parse_base64url, priv_parse_base64url, and the underlying deserialize_base64url<> template - all widened to string_view. The cascade terminates at fc::base64url_decode, which is widened to string_view alongside (mirroring fc::base64_decode's earlier widening on the variant-perf branch). --- libraries/libfc/include/fc/crypto/base64.hpp | 6 ++--- .../libfc/include/fc/crypto/bls_common.hpp | 6 ++--- .../include/fc/crypto/bls_private_key.hpp | 23 ++++------------- .../include/fc/crypto/bls_public_key.hpp | 9 +++---- .../libfc/include/fc/crypto/bls_signature.hpp | 17 ++++++------- .../libfc/src/crypto/bls_private_key.cpp | 17 +++---------- libraries/libfc/src/crypto/bls_public_key.cpp | 14 +---------- libraries/libfc/src/crypto/bls_signature.cpp | 25 +++---------------- 8 files changed, 29 insertions(+), 88 deletions(-) diff --git a/libraries/libfc/include/fc/crypto/base64.hpp b/libraries/libfc/include/fc/crypto/base64.hpp index 0091200445..115f4850f5 100644 --- a/libraries/libfc/include/fc/crypto/base64.hpp +++ b/libraries/libfc/include/fc/crypto/base64.hpp @@ -82,7 +82,7 @@ std::vector base64_decode( const std::string& s); std::vector base64_decode( std::string_view s); std::string base64url_encode(const char* s, size_t len); std::string base64url_encode(const std::string& s); -std::vector base64url_decode(const std::string& s); +std::vector base64url_decode(std::string_view s); namespace detail { // @@ -373,7 +373,7 @@ inline std::string base64url_encode(const std::string& s) { return base64_encode((unsigned char const*)s.data(), s.size(), true); } -inline std::vector base64url_decode(const std::string& s) { - return detail::decode>(s, false, true); +inline std::vector base64url_decode(std::string_view s) { + return detail::decode, std::string_view>(s, false, true); } } // namespace fc diff --git a/libraries/libfc/include/fc/crypto/bls_common.hpp b/libraries/libfc/include/fc/crypto/bls_common.hpp index df205092f9..bebeaf061f 100644 --- a/libraries/libfc/include/fc/crypto/bls_common.hpp +++ b/libraries/libfc/include/fc/crypto/bls_common.hpp @@ -13,7 +13,7 @@ constexpr std::string signature_prefix = "SIG_BLS_"; } // namespace constants template - Container deserialize_base64url(const std::string& data_str) { + Container deserialize_base64url(std::string_view data_str) { using wrapper = checksum_data; wrapper wrapped; @@ -40,7 +40,7 @@ constexpr std::string signature_prefix = "SIG_BLS_"; return data_str; } - inline public_key_data deserialize_bls_base64url(const std::string& base64urlstr) { + inline public_key_data deserialize_bls_base64url(std::string_view base64urlstr) { using namespace fc::crypto::bls::constants; auto res = std::mismatch(bls_public_key_prefix.begin(), bls_public_key_prefix.end(), base64urlstr.begin()); FC_ASSERT(res.first == bls_public_key_prefix.end(), "BLS Public Key has invalid format : {}", base64urlstr); @@ -48,7 +48,7 @@ constexpr std::string signature_prefix = "SIG_BLS_"; return fc::crypto::bls::deserialize_base64url(data_str); } - inline signature_data sig_parse_base64url(const std::string& base64urlstr) { + inline signature_data sig_parse_base64url(std::string_view base64urlstr) { auto res = std::mismatch(constants::signature_prefix.begin(), bls::constants::signature_prefix.end(), base64urlstr.begin()); FC_ASSERT(res.first == bls::constants::signature_prefix.end(), "BLS Signature has invalid format : {}", base64urlstr); auto data_str = base64urlstr.substr(bls::constants::signature_prefix.size()); diff --git a/libraries/libfc/include/fc/crypto/bls_private_key.hpp b/libraries/libfc/include/fc/crypto/bls_private_key.hpp index 1a0cac3e40..78579d2226 100644 --- a/libraries/libfc/include/fc/crypto/bls_private_key.hpp +++ b/libraries/libfc/include/fc/crypto/bls_private_key.hpp @@ -39,13 +39,14 @@ class private_key { * @brief Constructs a private key from a base64url encoded string * @param base64urlstr The base64url encoded string */ - explicit private_key(const std::string& base64urlstr); + explicit private_key(std::string_view base64urlstr); private_key& operator=(const private_key& pk) = default; private_key& operator=(private_key&& pk) noexcept = default; private_key_secret get_secret() const { return _sk; }; std::string to_string() const; + static private_key from_string(std::string_view s) { return private_key(s); } public_key get_public_key() const; @@ -218,28 +219,14 @@ struct private_key_shim : crypto::shim { } // namespace fc::crypto::bls -namespace fc { -/** - * @brief Converts a BLS private key to a variant - * @param var The private key to convert - * @param vo The output variant - */ -void to_variant(const crypto::bls::private_key& var, variant& vo); - -/** - * @brief Converts a variant to a BLS private key - * @param var The variant to convert - * @param vo The output private key - */ -void from_variant(const variant& var, crypto::bls::private_key& vo); -} // namespace fc - - #include FC_REFLECT(fc::crypto::bls::private_key, (_sk)) FC_REFLECT_TYPENAME(fc::crypto::bls::public_key) +#include +FC_SERIALIZE_AS_STRING(fc::crypto::bls::private_key) + FC_REFLECT(fc::crypto::bls::public_key_shim, (shim_ptr)) FC_REFLECT(fc::crypto::bls::signature_shim, (shim_ptr)) diff --git a/libraries/libfc/include/fc/crypto/bls_public_key.hpp b/libraries/libfc/include/fc/crypto/bls_public_key.hpp index da3f9812bf..e5d5c44cc6 100644 --- a/libraries/libfc/include/fc/crypto/bls_public_key.hpp +++ b/libraries/libfc/include/fc/crypto/bls_public_key.hpp @@ -42,13 +42,14 @@ namespace fc::crypto::bls { explicit public_key(const public_key_data& affine_non_montgomery_le); // affine non-montgomery base64url with bls_public_key_prefix - explicit public_key(const std::string& base64urlstr); + explicit public_key(std::string_view base64urlstr); bool valid()const; public_key_data serialize()const; // affine non-montgomery base64url with bls_public_key_prefix std::string to_string() const; static std::string to_string(const public_key_data& key); + static public_key from_string(std::string_view s) { return public_key(s); } const bls12_381::g1& jacobian_montgomery_le() const { return _jacobian_montgomery_le; } const public_key_data& affine_non_montgomery_le() const { return _affine_non_montgomery_le; } @@ -99,7 +100,5 @@ namespace fc::crypto::bls { } // fc::crypto::bls -namespace fc { - void to_variant(const crypto::bls::public_key& var, variant& vo); - void from_variant(const variant& var, crypto::bls::public_key& vo); -} // namespace fc +#include +FC_SERIALIZE_AS_STRING(fc::crypto::bls::public_key) diff --git a/libraries/libfc/include/fc/crypto/bls_signature.hpp b/libraries/libfc/include/fc/crypto/bls_signature.hpp index 2d2a12dcf7..38ef4a4a87 100644 --- a/libraries/libfc/include/fc/crypto/bls_signature.hpp +++ b/libraries/libfc/include/fc/crypto/bls_signature.hpp @@ -30,11 +30,12 @@ namespace fc::crypto::bls { explicit signature(bls::signature_data_span affine_non_montgomery_le); // affine non-montgomery base64url with signature_prefix - explicit signature(const std::string& base64urlstr); + explicit signature(std::string_view base64urlstr); // affine non-montgomery base64url with signature_prefix std::string to_string() const; static std::string to_string(const bls::signature_data& sig); + static signature from_string(std::string_view s) { return signature(s); } const bls12_381::g2& jacobian_montgomery_le() const { return _jacobian_montgomery_le; } const bls::signature_data& affine_non_montgomery_le() const { return _affine_non_montgomery_le; } @@ -93,7 +94,7 @@ namespace fc::crypto::bls { aggregate_signature& operator=(aggregate_signature&&) = default; // affine non-montgomery base64url with signature_prefix - explicit aggregate_signature(const std::string& base64_url_str); + explicit aggregate_signature(std::string_view base64_url_str); explicit aggregate_signature(const signature& sig) : _jacobian_montgomery_le(sig.jacobian_montgomery_le()) {} @@ -110,6 +111,7 @@ namespace fc::crypto::bls { // affine non-montgomery base64url with signature_prefix // Expensive as conversion from Jacobian Montgomery to Affine Non-Montgomery needed std::string to_string() const; + static aggregate_signature from_string(std::string_view s) { return aggregate_signature(s); } const bls12_381::g2& jacobian_montgomery_le() const { return _jacobian_montgomery_le; } @@ -148,11 +150,6 @@ namespace fc::crypto::bls { } // fc::crypto::bls -namespace fc { - - void to_variant(const crypto::bls::signature& var, variant& vo); - void from_variant(const variant& var, crypto::bls::signature& vo); - void to_variant(const crypto::bls::aggregate_signature& var, variant& vo); - void from_variant(const variant& var, crypto::bls::aggregate_signature& vo); - -} // namespace fc +#include +FC_SERIALIZE_AS_STRING(fc::crypto::bls::signature) +FC_SERIALIZE_AS_STRING(fc::crypto::bls::aggregate_signature) diff --git a/libraries/libfc/src/crypto/bls_private_key.cpp b/libraries/libfc/src/crypto/bls_private_key.cpp index b64d491449..ffd1881291 100644 --- a/libraries/libfc/src/crypto/bls_private_key.cpp +++ b/libraries/libfc/src/crypto/bls_private_key.cpp @@ -38,7 +38,7 @@ signature_shim::public_key_type signature_shim::recover(const sha256&) const { FC_THROW_EXCEPTION(fc::unsupported_exception, "BLS Signature Recovery is not supported"); } -static fc::sha256::uint64_array_type priv_parse_base64url(const std::string& base64urlstr) { +static fc::sha256::uint64_array_type priv_parse_base64url(std::string_view base64urlstr) { auto res = std::mismatch(bls::constants::bls_private_key_prefix.begin(), bls::constants::bls_private_key_prefix.end(), base64urlstr.begin()); @@ -49,7 +49,7 @@ static fc::sha256::uint64_array_type priv_parse_base64url(const std::string& bas return fc::crypto::bls::deserialize_base64url(data_str); } -private_key::private_key(const std::string& base64urlstr) +private_key::private_key(std::string_view base64urlstr) : _sk(priv_parse_base64url(base64urlstr)) {} std::string private_key::to_string() const { @@ -62,15 +62,4 @@ bool operator ==(const private_key& pk1, const private_key& pk2) { return pk1._sk == pk2._sk; } -} // fc::crypto::bls - -namespace fc { -void to_variant(const crypto::bls::private_key& var, variant& vo) { - vo = var.to_string(); -} - -void from_variant(const variant& var, crypto::bls::private_key& vo) { - vo = crypto::bls::private_key(var.as_string()); -} - -} // fc \ No newline at end of file +} // fc::crypto::bls \ No newline at end of file diff --git a/libraries/libfc/src/crypto/bls_public_key.cpp b/libraries/libfc/src/crypto/bls_public_key.cpp index d08ce4f7b3..2b492a28fe 100644 --- a/libraries/libfc/src/crypto/bls_public_key.cpp +++ b/libraries/libfc/src/crypto/bls_public_key.cpp @@ -31,7 +31,7 @@ namespace fc::crypto::bls { , _jacobian_montgomery_le(from_affine_bytes_le(_affine_non_montgomery_le)) { } - public_key::public_key(const std::string& base64urlstr) + public_key::public_key(std::string_view base64urlstr) : _affine_non_montgomery_le(deserialize_bls_base64url(base64urlstr)) , _jacobian_montgomery_le(from_affine_bytes_le(_affine_non_montgomery_le)) { } @@ -53,15 +53,3 @@ namespace fc::crypto::bls { } } // fc::crypto::bls - -namespace fc { - - void to_variant(const crypto::bls::public_key& var, variant& vo) { - vo = var.to_string(); - } - - void from_variant(const variant& var, crypto::bls::public_key& vo) { - vo = crypto::bls::public_key(var.as_string()); - } - -} // fc diff --git a/libraries/libfc/src/crypto/bls_signature.cpp b/libraries/libfc/src/crypto/bls_signature.cpp index 94f89ccfed..47e59e58bd 100644 --- a/libraries/libfc/src/crypto/bls_signature.cpp +++ b/libraries/libfc/src/crypto/bls_signature.cpp @@ -36,7 +36,7 @@ signature::signature(bls::signature_data_span affine_non_montgomery_le) , _jacobian_montgomery_le(to_jacobian_montgomery_le(_affine_non_montgomery_le)) {} -signature::signature(const std::string& base64urlstr) +signature::signature(std::string_view base64urlstr) : _affine_non_montgomery_le(sig_parse_base64url(base64urlstr)) , _jacobian_montgomery_le(to_jacobian_montgomery_le(_affine_non_montgomery_le)) {} @@ -50,7 +50,7 @@ std::string signature::to_string() const { return to_string(_affine_non_montgomery_le); } -aggregate_signature::aggregate_signature(const std::string& base64_url_str) +aggregate_signature::aggregate_signature(std::string_view base64_url_str) : _jacobian_montgomery_le(signature::to_jacobian_montgomery_le(sig_parse_base64url(base64_url_str))) {} @@ -60,23 +60,4 @@ std::string aggregate_signature::to_string() const { return bls::constants::signature_prefix + data_str; } -} // fc::crypto::bls - -namespace fc { - -void to_variant(const crypto::bls::signature& var, variant& vo) { - vo = var.to_string(); -} - -void from_variant(const variant& var, crypto::bls::signature& vo) { - vo = crypto::bls::signature(var.as_string()); -} - -void to_variant(const crypto::bls::aggregate_signature& var, variant& vo) { - vo = var.to_string(); -} - -void from_variant(const variant& var, crypto::bls::aggregate_signature& vo) { - vo = crypto::bls::aggregate_signature(var.as_string()); -} -} // fc \ No newline at end of file +} // fc::crypto::bls \ No newline at end of file From 4cd3b0dfca92105417b522628572cea993d1c687 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Mon, 27 Apr 2026 15:14:47 -0500 Subject: [PATCH 21/71] libfc: remove unused fc::fixed_string The header was only included transitively via chain/types.hpp; zero actual users in libraries/, plugins/, or unittests/. Deleted along with the matching pack / unpack template declarations in fc/io/raw_fwd.hpp. --- libraries/chain/include/sysio/chain/types.hpp | 1 - libraries/libfc/include/fc/fixed_string.hpp | 171 ------------------ libraries/libfc/include/fc/io/raw_fwd.hpp | 3 - 3 files changed, 175 deletions(-) delete mode 100644 libraries/libfc/include/fc/fixed_string.hpp diff --git a/libraries/chain/include/sysio/chain/types.hpp b/libraries/chain/include/sysio/chain/types.hpp index 58ff620f42..61071c0240 100644 --- a/libraries/chain/include/sysio/chain/types.hpp +++ b/libraries/chain/include/sysio/chain/types.hpp @@ -13,7 +13,6 @@ #include #include #include -#include #include #include diff --git a/libraries/libfc/include/fc/fixed_string.hpp b/libraries/libfc/include/fc/fixed_string.hpp deleted file mode 100644 index 3c087fe284..0000000000 --- a/libraries/libfc/include/fc/fixed_string.hpp +++ /dev/null @@ -1,171 +0,0 @@ -#pragma once -#include - - -namespace fc { - - - /** - * This class is designed to offer in-place memory allocation of a string up to Length equal to - * sizeof(Storage). - * - * The string will serialize the same way as std::string for variant and raw formats - * The string will sort according to the comparison operators defined for Storage, this enables effecient - * sorting. - */ - template > - class fixed_string { - public: - fixed_string(){ - memset( (char*)&data, 0, sizeof(data) ); - } - fixed_string( const fixed_string& c ):data(c.data){} - - fixed_string( const std::string& str ) { - if( str.size() < sizeof(data) ) { - memset( (char*)&data, 0, sizeof(data) ); - memcpy( (char*)&data, str.c_str(), str.size() ); - } else { - memcpy( (char*)&data, str.c_str(), sizeof(data) ); - } - } - fixed_string( const char* str ) { - memset( (char*)&data, 0, sizeof(data) ); - auto l = strlen(str); - if( l < sizeof(data) ) { - memset( (char*)&data, 0, sizeof(data) ); - memcpy( (char*)&data, str, l ); - } - else { - memcpy( (char*)&data, str, sizeof(data) ); - } - } - - operator std::string()const { - const char* self = (const char*)&data; - return std::string( self, self + size() ); - } - - uint32_t size()const { - if( *(((const char*)&data)+sizeof(data) - 1) ) - return sizeof(data); - return strnlen( (const char*)&data, sizeof(data) ); - } - uint32_t length()const { return size(); } - - fixed_string& operator=( const fixed_string& str ) { - data = str.data; - return *this; - } - fixed_string& operator=( const char* str ) { - return *this = fixed_string(str); - } - - fixed_string& operator=( const std::string& str ) { - if( str.size() < sizeof(data) ) { - memset( (char*)&data, 0, sizeof(data) ); - memcpy( (char*)&data, str.c_str(), str.size() ); - } - else { - memcpy( (char*)&data, str.c_str(), sizeof(data) ); - } - return *this; - } - - friend std::string operator + ( const fixed_string& a, const std::string& b ) { - return std::string(a) + b; - } - friend std::string operator + ( const std::string& a, const fixed_string& b ) { - return a + std::string(b); - } - - friend bool operator < ( const fixed_string& a, const fixed_string& b ) { - return a.data < b.data; - } - friend bool operator <= ( const fixed_string& a, const fixed_string& b ) { - return a.data <= b.data; - } - friend bool operator > ( const fixed_string& a, const fixed_string& b ) { - return a.data > b.data; - } - friend bool operator >= ( const fixed_string& a, const fixed_string& b ) { - return a.data >= b.data; - } - friend bool operator == ( const fixed_string& a, const fixed_string& b ) { - return a.data == b.data; - } - friend bool operator != ( const fixed_string& a, const fixed_string& b ) { - return a.data != b.data; - } - - friend std::ostream& operator << ( std::ostream& out, const fixed_string& str ) { - return out << std::string(str); - } - //private: - Storage data; - }; - - namespace raw - { - template - inline void pack( Stream& s, const fc::fixed_string& u ) { - unsigned_int size = u.size(); - pack( s, size ); - s.write( (const char*)&u.data, size ); - } - - template - inline void unpack( Stream& s, fc::fixed_string& u ) { - unsigned_int size; - fc::raw::unpack( s, size ); - if( size.value > 0 ) { - if( size.value > sizeof(Storage) ) { - s.read( (char*)&u.data, sizeof(Storage) ); - char buf[1024]; - size_t left = size.value - sizeof(Storage); - while( left >= 1024 ) - { - s.read( buf, 1024 ); - left -= 1024; - } - s.read( buf, left ); - - /* - s.seekp( s.tellp() + (size.value - sizeof(Storage)) ); - char tmp; - size.value -= sizeof(storage); - while( size.value ){ s.read( &tmp, 1 ); --size.value; } - */ - // s.skip( size.value - sizeof(Storage) ); - } else { - s.read( (char*)&u.data, size.value ); - } - } - } - - /* - template - inline void pack( Stream& s, const boost::multiprecision::number& d ) { - s.write( (const char*)&d, sizeof(d) ); - } - - template - inline void unpack( Stream& s, boost::multiprecision::number& u ) { - s.read( (const char*)&u, sizeof(u) ); - } - */ - } -} - -#include -namespace fc { - template - void to_variant( const fixed_string& s, variant& v ) { - v = std::string(s); - } - - template - void from_variant( const variant& v, fixed_string& s ) { - s = v.as_string(); - } -} diff --git a/libraries/libfc/include/fc/io/raw_fwd.hpp b/libraries/libfc/include/fc/io/raw_fwd.hpp index ce9257a0fc..6a81239572 100644 --- a/libraries/libfc/include/fc/io/raw_fwd.hpp +++ b/libraries/libfc/include/fc/io/raw_fwd.hpp @@ -41,9 +41,6 @@ namespace fc { template void unpack(Stream& s, sysio::chain::signed_block& v); - template inline void pack( Stream& s, const fc::fixed_string& u ); - template inline void unpack( Stream& s, fc::fixed_string& u ); - template inline void pack( Stream& s, const fc::enum_type& tp ); template From 067ef4823541f644c0391cbd889e93ab73bb5865 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Mon, 27 Apr 2026 15:15:18 -0500 Subject: [PATCH 22/71] libfc: migrate bitset + url to FC_SERIALIZE_AS_STRING bitset already had a trait-shaped to_string() / static from_string(string_view); migration is just dropping the hand-rolled fc::to_variant / fc::from_variant and applying the macro. The previous to_variant had a defensive num_blocks > MAX_NUM_ARRAY_ELEMENTS guard that no longer fires - the trait path produces a 1-char-per-bit string, so a pathological bitset throws std::bad_alloc on .resize() rather than std::range_error. Theoretical only; real bitsets in chain code are O(100) bits. url gains a thin to_string() (alias for the existing operator std::string()) and static from_string(const std::string&) (alias for the existing string ctor). url::url(const std::string&) stays as-is - its body parses through std::stringstream(s), so widening to string_view would force a materialisation inside parse anyway. At the trait dispatch site, fc::from_variant(...) will compiler-implicitly materialise one std::string per call. --- libraries/libfc/include/fc/bitset.hpp | 18 +++--------------- libraries/libfc/include/fc/network/url.hpp | 8 +++++--- libraries/libfc/src/network/url.cpp | 9 --------- 3 files changed, 8 insertions(+), 27 deletions(-) diff --git a/libraries/libfc/include/fc/bitset.hpp b/libraries/libfc/include/fc/bitset.hpp index d1e80a8cf8..c503c0c322 100644 --- a/libraries/libfc/include/fc/bitset.hpp +++ b/libraries/libfc/include/fc/bitset.hpp @@ -6,7 +6,7 @@ #include #include #include -#include +#include namespace fc { @@ -202,18 +202,6 @@ struct bitset { }; -// ---------------- to_variant / from_variant conversions ---------------------------------------------- - -inline void to_variant(const fc::bitset& bs, fc::variant& v) { - auto num_blocks = bs.num_blocks(); - if (num_blocks > MAX_NUM_ARRAY_ELEMENTS) - throw std::range_error("number of blocks of bitset cannot be greather than MAX_NUM_ARRAY_ELEMENTS"); - - v = bs.to_string(); -} - -inline void from_variant(const fc::variant& v, fc::bitset& bs) { - bs = fc::bitset(v.get_string()); -} - } // namespace fc + +FC_SERIALIZE_AS_STRING(fc::bitset) diff --git a/libraries/libfc/include/fc/network/url.hpp b/libraries/libfc/include/fc/network/url.hpp index a4270f03ff..9a7ff4eb81 100644 --- a/libraries/libfc/include/fc/network/url.hpp +++ b/libraries/libfc/include/fc/network/url.hpp @@ -3,6 +3,7 @@ #include #include #include +#include #include namespace fc { @@ -39,6 +40,8 @@ namespace fc { bool operator==( const url& cmp )const; operator std::string()const; + std::string to_string() const { return std::string(*this); } + static url from_string(const std::string& s) { return url(s); } //// file, ssh, tcp, http, ssl, etc... std::string proto()const; @@ -55,7 +58,6 @@ namespace fc { std::shared_ptr my; }; - void to_variant( const url& u, fc::variant& v ); - void from_variant( const fc::variant& v, url& u ); - } // namespace fc + +FC_SERIALIZE_AS_STRING(fc::url) diff --git a/libraries/libfc/src/network/url.cpp b/libraries/libfc/src/network/url.cpp index b96c9a1938..376c80064a 100644 --- a/libraries/libfc/src/network/url.cpp +++ b/libraries/libfc/src/network/url.cpp @@ -77,15 +77,6 @@ namespace fc }; } - void to_variant( const url& u, fc::variant& v ) - { - v = std::string(u); - } - void from_variant( const fc::variant& v, url& u ) - { - u = url( v.as_string() ); - } - url::operator std::string()const { std::stringstream ss; From b08be8574e81711eb9cc1df7d3cc582c7a55a4e2 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Mon, 27 Apr 2026 19:40:12 -0500 Subject: [PATCH 23/71] libfc: extract json yield/escape into json_yield.hpp; co-locate to_json_stream container overloads with to_variant The streaming-JSON container overloads (std::vector / std::map / std::set / std::pair / std::deque / std::optional / etc.) used to live in fc/reflect/json_stream.hpp, away from their fc::to_variant siblings in fc/variant.hpp - easy for the two paths to drift, and the streaming versions had no MAX_NUM_ARRAY_ELEMENTS guards while the variant ones did. Putting them next to each other in variant.hpp means fc/variant.hpp needs fc/io/json_stream.hpp at the top, but fc/io/json.hpp's transitive #include of fc/variant.hpp creates a cycle that leaves class json incomplete inside json_stream.hpp's body. The cycle is broken by extracting the bits of json.hpp that json_stream.hpp actually uses - fc::json::yield_function_t and fc::escape_string - into a new fc/io/json_yield.hpp. json::yield_function_t becomes an alias for fc::json_yield_function_t so existing callers continue to compile. json_stream.hpp now includes json_yield.hpp instead of json.hpp, breaking its dependency on class json. With the cycle resolved: - variant.hpp adds to_json_stream for std::optional / std::unordered_set / std::unordered_map / std::map / std::multimap / std::set / std::deque / boost::container::deque / std::vector / std::array / std::pair - each immediately above its to_variant sibling, all with MAX_NUM_ARRAY_ELEMENTS guards. - flat.hpp adds to_json_stream for flat_set / flat_multiset above each to_variant, with the same guard. - reflect/json_stream.hpp loses the container overloads; keeps scalar overloads, reflector visitor, and the primary dispatch template. --- libraries/libfc/include/fc/container/flat.hpp | 17 ++- libraries/libfc/include/fc/io/json.hpp | 7 +- libraries/libfc/include/fc/io/json_stream.hpp | 6 +- libraries/libfc/include/fc/io/json_yield.hpp | 26 ++++ .../libfc/include/fc/reflect/json_stream.hpp | 70 +-------- libraries/libfc/include/fc/variant.hpp | 144 +++++++++++++++--- 6 files changed, 172 insertions(+), 98 deletions(-) create mode 100644 libraries/libfc/include/fc/io/json_yield.hpp diff --git a/libraries/libfc/include/fc/container/flat.hpp b/libraries/libfc/include/fc/container/flat.hpp index d9569c3daa..09d3eb569d 100644 --- a/libraries/libfc/include/fc/container/flat.hpp +++ b/libraries/libfc/include/fc/container/flat.hpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace fc { @@ -143,21 +144,33 @@ namespace fc { } } + template + void to_json_stream( const flat_set< T, U... >& s, json_writer& w ) { + FC_ASSERT( s.size() <= MAX_NUM_ARRAY_ELEMENTS ); + w.begin_array(); + for( const auto& e : s ) to_json_stream( e, w ); + w.end_array(); + } template void to_variant( const flat_set< T, U... >& s, fc::variant& vo ) { detail::to_variant_from_set( s, vo ); } - template void from_variant( const fc::variant& v, flat_set< T, U... >& s ) { detail::from_variant_to_flat_set( v, s ); } + template + void to_json_stream( const flat_multiset< T, U... >& s, json_writer& w ) { + FC_ASSERT( s.size() <= MAX_NUM_ARRAY_ELEMENTS ); + w.begin_array(); + for( const auto& e : s ) to_json_stream( e, w ); + w.end_array(); + } template void to_variant( const flat_multiset< T, U... >& s, fc::variant& vo ) { detail::to_variant_from_set( s, vo ); } - template void from_variant( const fc::variant& v, flat_multiset< T, U... >& s ) { detail::from_variant_to_flat_set( v, s ); diff --git a/libraries/libfc/include/fc/io/json.hpp b/libraries/libfc/include/fc/io/json.hpp index 53d93a919b..65cd0521c8 100644 --- a/libraries/libfc/include/fc/io/json.hpp +++ b/libraries/libfc/include/fc/io/json.hpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #define DEFAULT_MAX_RECURSION_DEPTH 200 @@ -26,7 +27,7 @@ namespace fc relaxed_parser = 2, legacy_parser_with_string_doubles = 3 }; - using yield_function_t = fc::optional_delegate; + using yield_function_t = fc::json_yield_function_t; static constexpr uint64_t max_length_limit = std::numeric_limits::max(); static constexpr size_t escape_string_yield_check_count = 128; static variant from_string( const std::string& utf8_str, const parse_type ptype = parse_type::legacy_parser, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ); @@ -91,10 +92,6 @@ namespace fc } }; - std::string escape_string( const std::string_view& str, const json::yield_function_t& yield, bool escape_control_chars = true ); - /// Returns true if any characters were escaped or invalid UTF-8 was pruned. - bool escape_string( const std::string_view& str, std::string& out, const json::yield_function_t& yield, bool escape_control_chars = true ); - } // fc #undef DEFAULT_MAX_RECURSION_DEPTH diff --git a/libraries/libfc/include/fc/io/json_stream.hpp b/libraries/libfc/include/fc/io/json_stream.hpp index 2afa42e2bb..cb704950ee 100644 --- a/libraries/libfc/include/fc/io/json_stream.hpp +++ b/libraries/libfc/include/fc/io/json_stream.hpp @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include @@ -74,7 +74,7 @@ class json_writer { out_.push_back('"'); // key strings must also be escaped (eg field names containing special chars are rare, // but correctness matters on untrusted input echoed into error messages). - fc::escape_string(k, out_, json::yield_function_t(), true); + fc::escape_string(k, out_, json_yield_function_t(), true); out_.push_back('"'); out_.push_back(':'); // A key mandates a value follows; suppress the comma from value_prefix for that value. @@ -84,7 +84,7 @@ class json_writer { void value_string(std::string_view s) { value_prefix(); out_.push_back('"'); - fc::escape_string(s, out_, json::yield_function_t(), true); + fc::escape_string(s, out_, json_yield_function_t(), true); out_.push_back('"'); } diff --git a/libraries/libfc/include/fc/io/json_yield.hpp b/libraries/libfc/include/fc/io/json_yield.hpp new file mode 100644 index 0000000000..396b613104 --- /dev/null +++ b/libraries/libfc/include/fc/io/json_yield.hpp @@ -0,0 +1,26 @@ +#pragma once + +// Lightweight header carrying just the bits of the JSON layer that fc/io/json_stream.hpp +// (and downstream variant.hpp / flat.hpp container streaming overloads) need to compile, +// without dragging in fc/io/json.hpp's class json. Pulling in class json would create +// a cycle: variant.hpp -> json_stream.hpp -> json.hpp -> variant.hpp. + +#include +#include +#include + +namespace fc { + + // The yield callback used by streaming serialisers to check deadlines and bail out + // of long-running JSON conversions. class json::yield_function_t (in fc/io/json.hpp) + // is an alias to this type so existing callers continue to compile. + using json_yield_function_t = fc::optional_delegate; + + /// Escape a UTF-8 string for inclusion in a JSON value. Returns the new string. + std::string escape_string( const std::string_view& str, const json_yield_function_t& yield, bool escape_control_chars = true ); + + /// Append the escaped form of `str` to `out`. Returns true if any characters were + /// escaped or invalid UTF-8 was pruned. + bool escape_string( const std::string_view& str, std::string& out, const json_yield_function_t& yield, bool escape_control_chars = true ); + +} // namespace fc diff --git a/libraries/libfc/include/fc/reflect/json_stream.hpp b/libraries/libfc/include/fc/reflect/json_stream.hpp index b3269d6922..d46771756b 100644 --- a/libraries/libfc/include/fc/reflect/json_stream.hpp +++ b/libraries/libfc/include/fc/reflect/json_stream.hpp @@ -3,15 +3,10 @@ #include #include -#include #include -#include -#include #include #include #include -#include -#include namespace fc { @@ -34,6 +29,7 @@ void to_json_stream(const T& o, json_writer& w); // -- Scalar overloads --------------------------------------------------------------------- inline void to_json_stream(bool b, json_writer& w) { w.value_bool(b); } +inline void to_json_stream(char c, json_writer& w) { w.value_int8(static_cast(c)); } inline void to_json_stream(int8_t n, json_writer& w) { w.value_int8(n); } inline void to_json_stream(uint8_t n, json_writer& w) { w.value_uint8(n); } inline void to_json_stream(int16_t n, json_writer& w) { w.value_int16(n); } @@ -48,65 +44,11 @@ inline void to_json_stream(std::string_view s, json_writer& w) { w.value_str inline void to_json_stream(const std::string& s, json_writer& w) { w.value_string(s); } inline void to_json_stream(const char* s, json_writer& w) { w.value_string(s ? std::string_view(s) : std::string_view()); } -// -- Container overloads ------------------------------------------------------------------ - -template -void to_json_stream(const std::vector& v, json_writer& w) { - w.begin_array(); - for (const auto& e : v) to_json_stream(e, w); - w.end_array(); -} - -template -void to_json_stream(const std::array& a, json_writer& w) { - w.begin_array(); - for (const auto& e : a) to_json_stream(e, w); - w.end_array(); -} - -template -void to_json_stream(const std::optional& o, json_writer& w) { - if (o) to_json_stream(*o, w); - else w.value_null(); -} - -template -void to_json_stream(const std::map& m, json_writer& w) { - static_assert(std::is_convertible_v || std::is_integral_v, - "JSON object keys must be string- or integral-convertible"); - w.begin_object(); - for (const auto& kv : m) { - if constexpr (std::is_convertible_v) { - w.key(std::string_view(kv.first)); - } else { - // Integral keys are emitted as the numeric literal surrounded by quotes so the - // result is a valid JSON object (keys must be strings per RFC 8259). - char buf[24]; - int n = std::snprintf(buf, sizeof(buf), "%lld", static_cast(kv.first)); - w.key(std::string_view(buf, n > 0 ? static_cast(n) : 0)); - } - to_json_stream(kv.second, w); - } - w.end_object(); -} - -template -void to_json_stream(const std::unordered_map& m, json_writer& w) { - static_assert(std::is_convertible_v || std::is_integral_v, - "JSON object keys must be string- or integral-convertible"); - w.begin_object(); - for (const auto& kv : m) { - if constexpr (std::is_convertible_v) { - w.key(std::string_view(kv.first)); - } else { - char buf[24]; - int n = std::snprintf(buf, sizeof(buf), "%lld", static_cast(kv.first)); - w.key(std::string_view(buf, n > 0 ? static_cast(n) : 0)); - } - to_json_stream(kv.second, w); - } - w.end_object(); -} +// Container overloads (std::vector, std::array, std::optional, std::map, std::set, +// std::pair, std::deque, std::unordered_map, std::unordered_set, std::multimap) live +// alongside their to_variant siblings in fc/variant.hpp. flat_set / flat_multiset live +// in fc/container/flat.hpp. Co-locating with to_variant keeps the two paths' shape +// and MAX_NUM_ARRAY_ELEMENTS guards in lock-step. // -- Reflector visitor -------------------------------------------------------------------- diff --git a/libraries/libfc/include/fc/variant.hpp b/libraries/libfc/include/fc/variant.hpp index 97d5423499..e5d0ff7ad9 100644 --- a/libraries/libfc/include/fc/variant.hpp +++ b/libraries/libfc/include/fc/variant.hpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -454,7 +455,17 @@ namespace fc void from_variant( const fc::variant& var, int32_t& vo ); /** @ingroup Serializable */ void from_variant( const fc::variant& var, uint32_t& vo ); + /** @ingroup Serializable */ + template + void to_json_stream( const std::optional& o, json_writer& w ) + { + // Top-level std::optional emits null when unset; reflected struct fields use the + // visitor's emit() overload that omits the field instead, mirroring to_variant_visitor::add. + if( o ) to_json_stream( *o, w ); + else w.value_null(); + } + template void from_variant( const variant& var, std::optional& vo ) { @@ -465,6 +476,15 @@ namespace fc from_variant( var, *vo ); } } + + template + void to_json_stream( const std::unordered_set& var, json_writer& w ) + { + if( var.size() > MAX_NUM_ARRAY_ELEMENTS ) throw std::range_error( "too large" ); + w.begin_array(); + for( const auto& e : var ) to_json_stream( e, w ); + w.end_array(); + } template void to_variant( const std::unordered_set& var, fc::variant& vo ) { @@ -487,6 +507,16 @@ namespace fc } + template + void to_json_stream( const std::unordered_map& var, json_writer& w ) + { + // Mirrors fc::to_variant: emits as an array of [key, value] + // pairs (NOT a JSON object) so non-string keys round-trip cleanly. + if( var.size() > MAX_NUM_ARRAY_ELEMENTS ) throw std::range_error( "too large" ); + w.begin_array(); + for( const auto& kv : var ) to_json_stream( kv, w ); + w.end_array(); + } template void to_variant( const std::unordered_map& var, fc::variant& vo ) { @@ -508,6 +538,14 @@ namespace fc } template + void to_json_stream( const std::map& var, json_writer& w ) + { + if( var.size() > MAX_NUM_ARRAY_ELEMENTS ) throw std::range_error( "too large" ); + w.begin_array(); + for( const auto& kv : var ) to_json_stream( kv, w ); + w.end_array(); + } + template void to_variant( const std::map& var, fc::variant& vo ) { if( var.size() > MAX_NUM_ARRAY_ELEMENTS ) throw std::range_error( "too large" ); @@ -527,6 +565,14 @@ namespace fc vo.insert( itr->as< std::pair >() ); } + template + void to_json_stream( const std::multimap& var, json_writer& w ) + { + if( var.size() > MAX_NUM_ARRAY_ELEMENTS ) throw std::range_error( "too large" ); + w.begin_array(); + for( const auto& kv : var ) to_json_stream( kv, w ); + w.end_array(); + } template void to_variant( const std::multimap& var, fc::variant& vo ) { @@ -548,6 +594,14 @@ namespace fc } + template + void to_json_stream( const std::set& var, json_writer& w ) + { + if( var.size() > MAX_NUM_ARRAY_ELEMENTS ) throw std::range_error( "too large" ); + w.begin_array(); + for( const auto& e : var ) to_json_stream( e, w ); + w.end_array(); + } template void to_variant( const std::set& var, fc::variant& vo ) { @@ -571,15 +625,13 @@ namespace fc /** @ingroup Serializable */ template - void from_variant( const fc::variant& var, std::deque& tmp ) + void to_json_stream( const std::deque& t, json_writer& w ) { - const variants& vars = var.get_array(); - if( vars.size() > MAX_NUM_ARRAY_ELEMENTS ) throw std::range_error( "too large" ); - tmp.clear(); - for( auto itr = vars.begin(); itr != vars.end(); ++itr ) - tmp.push_back( itr->as() ); + if( t.size() > MAX_NUM_ARRAY_ELEMENTS ) throw std::range_error( "too large" ); + w.begin_array(); + for( const auto& e : t ) to_json_stream( e, w ); + w.end_array(); } - /** @ingroup Serializable */ template void to_variant( const std::deque& t, fc::variant& v ) @@ -590,20 +642,26 @@ namespace fc vars[i] = fc::variant(t[i]); v = std::move(vars); } - /** @ingroup Serializable */ - template - void from_variant( const fc::variant& v, boost::container::deque& d ) + template + void from_variant( const fc::variant& var, std::deque& tmp ) { - const variants& vars = v.get_array(); + const variants& vars = var.get_array(); if( vars.size() > MAX_NUM_ARRAY_ELEMENTS ) throw std::range_error( "too large" ); - d.clear(); - d.resize( vars.size() ); - for( uint32_t i = 0; i < vars.size(); ++i ) { - from_variant( vars[i], d[i] ); - } + tmp.clear(); + for( auto itr = vars.begin(); itr != vars.end(); ++itr ) + tmp.push_back( itr->as() ); } + /** @ingroup Serializable */ + template + void to_json_stream( const boost::container::deque& d, json_writer& w ) + { + if( d.size() > MAX_NUM_ARRAY_ELEMENTS ) throw std::range_error( "too large" ); + w.begin_array(); + for( const auto& e : d ) to_json_stream( e, w ); + w.end_array(); + } /** @ingroup Serializable */ template void to_variant( const boost::container::deque& d, fc::variant& vo ) @@ -615,19 +673,28 @@ namespace fc } vo = std::move( vars ); } - /** @ingroup Serializable */ - template - void from_variant( const fc::variant& var, std::vector& tmp ) + template + void from_variant( const fc::variant& v, boost::container::deque& d ) { - const variants& vars = var.get_array(); + const variants& vars = v.get_array(); if( vars.size() > MAX_NUM_ARRAY_ELEMENTS ) throw std::range_error( "too large" ); - tmp.clear(); - tmp.reserve( vars.size() ); - for( auto itr = vars.begin(); itr != vars.end(); ++itr ) - tmp.push_back( itr->as() ); + d.clear(); + d.resize( vars.size() ); + for( uint32_t i = 0; i < vars.size(); ++i ) { + from_variant( vars[i], d[i] ); + } } + /** @ingroup Serializable */ + template + void to_json_stream( const std::vector& v, json_writer& w ) + { + if( v.size() > MAX_NUM_ARRAY_ELEMENTS ) throw std::range_error( "too large" ); + w.begin_array(); + for( const auto& e : v ) to_json_stream( e, w ); + w.end_array(); + } /** @ingroup Serializable */ template void to_variant( const std::vector& t, fc::variant& v ) @@ -638,7 +705,26 @@ namespace fc vars[i] = fc::variant(t[i]); v = std::move(vars); } + /** @ingroup Serializable */ + template + void from_variant( const fc::variant& var, std::vector& tmp ) + { + const variants& vars = var.get_array(); + if( vars.size() > MAX_NUM_ARRAY_ELEMENTS ) throw std::range_error( "too large" ); + tmp.clear(); + tmp.reserve( vars.size() ); + for( auto itr = vars.begin(); itr != vars.end(); ++itr ) + tmp.push_back( itr->as() ); + } + /** @ingroup Serializable */ + template + void to_json_stream( const std::array& t, json_writer& w ) + { + w.begin_array(); + for( const auto& e : t ) to_json_stream( e, w ); + w.end_array(); + } /** @ingroup Serializable */ template void from_variant( const fc::variant& var, std::array& tmp ) @@ -690,6 +776,16 @@ namespace fc v = std::move(vars); } + /** @ingroup Serializable */ + template + void to_json_stream( const std::pair& t, json_writer& w ) + { + // Mirrors to_variant: emits a 2-element array [first, second]. + w.begin_array(); + to_json_stream( t.first, w ); + to_json_stream( t.second, w ); + w.end_array(); + } /** @ingroup Serializable */ template void to_variant( const std::pair& t, fc::variant& v ) From 5dc9c26e2e94a809932586b8f94a585b9aa5bb0f Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Mon, 27 Apr 2026 19:40:42 -0500 Subject: [PATCH 24/71] chain_api_plugin: migrate sync read-only endpoints to streaming-cb path Fifteen sync read-only chain_api endpoints flip from CHAIN_RO_CALL (variant-cb) to CHAIN_RO_CALL_STREAM / CHAIN_RO_CALL_STREAM_POST (streaming-cb): get_activated_protocol_features, get_block, get_code, get_code_hash, get_consensus_parameters, get_abi, get_raw_code_and_abi, get_raw_abi, get_finalizer_info, get_table_by_scope, get_currency_balance, get_producers, get_producer_schedule, get_required_keys, get_transaction_id. Joins get_account / get_table_rows in add_api_stream. The new CALL_WITH_400_STREAM macro mirrors CALL_WITH_400 but delivers the response as a json_writer-emitting closure to the http thread pool, no fc::variant tree. chain/abi_def.hpp gains a to_json_stream for sysio::chain::may_not_exist co-located with its to_variant - get_abi's response embeds may_not_exist wrappers around its variants / action_results / enums fields. get_block_info, get_block_header_state, and get_currency_stats stay on the variant-cb path: their chain_plugin methods return fc::variant directly, and to_json_stream(variant, w) currently bridges through fc::json::to_string. Migrating them today would just splice the same JSON string into the writer with no perf win. They migrate naturally once the streaming variant walker (rewriting to_json_stream(variant, w) to walk tokens directly) and the abi_serializer streaming path land - those are the real perf steps; this commit is registration uniformity for the endpoints whose response struct doesn't trip the variant bridge. Async endpoints (compute_transaction, push_transaction*, send_transaction*) and _WITH_400 endpoints (get_accounts_by_authorizers, get_raw_block, get_block_header, get_transaction_status, send_read_only_transaction) remain on variant-cb pending follow-up phases. --- .../chain/include/sysio/chain/abi_def.hpp | 5 +++ .../chain_api_plugin/src/chain_api_plugin.cpp | 35 +++++++++++-------- .../include/sysio/http_plugin/macros.hpp | 22 ++++++++++++ 3 files changed, 47 insertions(+), 15 deletions(-) diff --git a/libraries/chain/include/sysio/chain/abi_def.hpp b/libraries/chain/include/sysio/chain/abi_def.hpp index 8c15ead50d..42418f1bf6 100644 --- a/libraries/chain/include/sysio/chain/abi_def.hpp +++ b/libraries/chain/include/sysio/chain/abi_def.hpp @@ -176,6 +176,11 @@ datastream& operator >> (datastream& s, sysio::chain::may_not_exist& return s; } +template +void to_json_stream(const sysio::chain::may_not_exist& e, fc::json_writer& w) { + to_json_stream( e.value, w ); +} + template void to_variant(const sysio::chain::may_not_exist& e, fc::variant& v) { to_variant( e.value, v); diff --git a/plugins/chain_api_plugin/src/chain_api_plugin.cpp b/plugins/chain_api_plugin/src/chain_api_plugin.cpp index 1e44d3fa85..819091b143 100644 --- a/plugins/chain_api_plugin/src/chain_api_plugin.cpp +++ b/plugins/chain_api_plugin/src/chain_api_plugin.cpp @@ -115,6 +115,7 @@ parse_params, 200, http_params_types::params_required), + CHAIN_RO_CALL_STREAM(get_producers, chain_apis::read_only::get_producers_result, 200, http_params_types::params_required), + CHAIN_RO_CALL_STREAM(get_producer_schedule, chain_apis::read_only::get_producer_schedule_result, 200, http_params_types::no_params), + CHAIN_RO_CALL_STREAM(get_required_keys, chain_apis::read_only::get_required_keys_result, 200, http_params_types::params_required), + CHAIN_RO_CALL_STREAM(get_transaction_id, chain_apis::read_only::get_transaction_id_result, 200, http_params_types::params_required), CHAIN_RO_CALL_STREAM_POST(get_account, chain_apis::read_only::get_account_results, 200, http_params_types::params_required), CHAIN_RO_CALL_STREAM_POST(get_table_rows, chain_apis::read_only::get_table_rows_result, 200, http_params_types::params_required), }, appbase::exec_queue::read_only); diff --git a/plugins/http_plugin/include/sysio/http_plugin/macros.hpp b/plugins/http_plugin/include/sysio/http_plugin/macros.hpp index 3aa7a98735..81ce48569f 100644 --- a/plugins/http_plugin/include/sysio/http_plugin/macros.hpp +++ b/plugins/http_plugin/include/sysio/http_plugin/macros.hpp @@ -85,6 +85,28 @@ }} +// Streaming-cb counterpart of CALL_WITH_400. Sync endpoint: call runs on the +// calling queue (typically read_only), result is captured into a json_writer- +// emitting closure that the cb forwards to the http thread pool for emission. +// No fc::variant tree on the response path. +// ------------------------------------------------------------------------------------------------------ +#define CALL_WITH_400_STREAM(api_name, category, api_handle, api_namespace, call_name, call_result, http_resp_code, params_type) \ +{std::string("/v1/" #api_name "/" #call_name), \ + api_category::category, \ + [api_handle](string&&, string&& body, url_response_stream_callback&& cb) mutable { \ + auto deadline = api_handle.start(); \ + try { \ + auto params = parse_params(body); \ + call_result result = api_handle.call_name(std::move(params), deadline); \ + cb(http_resp_code, [r = std::move(result)](fc::json_writer& w) mutable { \ + fc::to_json_stream(r, w); \ + }); \ + } catch (...) { \ + http_plugin::handle_exception_stream(#api_name, #call_name, body, cb); \ + } \ + }} + + // Streaming-cb counterpart of CALL_WITH_400_POST. Same Phase 1 (api thread) / // Phase 2 (http thread pool) split as the variant version, but Phase 2 hands // the typed call_result to cb as a json_writer-emitting closure - no fc::variant From a452c0e3e5d9cd08e8becea1106093acd9c35737 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Mon, 27 Apr 2026 19:47:27 -0500 Subject: [PATCH 25/71] libfc: streaming variant walker - to_json_stream(variant, w) emits tokens directly Rewrites to_json_stream(variant, w), to_json_stream(variant_object, w), and to_json_stream(mutable_variant_object, w) to walk their inputs token-by-token directly into json_writer instead of calling fc::json::to_string and splicing the result via raw_value. Mirrors the compact (indent=0) form of fc::json::to_string for byte-identical output. Per variant case, the dispatch maps to: - null -> w.value_null() - int64 in 32-bit range -> w.value_int64() (same numeric form) - int64 out of range -> w.value_string(as_string()) (quoted, JS-precision-safe; matches existing convention) - uint64 same pattern - int128 / uint128 / int256 / uint256 -> w.value_string(as_string()) (quoted decimal) - double -> w.value_string(as_string()) (quoted, matches fc::json::to_string) - bool -> w.value_bool() - string / string_sso -> w.value_string() (handles its own JSON escape via fc::escape_string) - blob -> w.value_string(as_string()) (base64 form) - array -> w.begin_array() + recurse on each element + w.end_array() - object -> recurse into to_json_stream(variant_object, w) which walks key/value pairs directly variant_object / mutable_variant_object iterate kv pairs and call w.key(kv.key()) + recurse on the value. No intermediate variant or std::string build at any level. This is the variant-walker step in the streaming-JSON roadmap: it makes the existing to_json_stream(variant) bridge a real streaming path. The bigger remaining win is on abi_serializer's binary_to_variant - that's where the per-row variant allocation cost comes from on get_table_rows / get_account, and the next step is a binary_to_json_stream variant alongside it. --- libraries/libfc/src/variant.cpp | 69 +++++++++++++++++++++++--- libraries/libfc/src/variant_object.cpp | 25 +++++++--- 2 files changed, 80 insertions(+), 14 deletions(-) diff --git a/libraries/libfc/src/variant.cpp b/libraries/libfc/src/variant.cpp index 53490bd401..5ec1c73b3c 100644 --- a/libraries/libfc/src/variant.cpp +++ b/libraries/libfc/src/variant.cpp @@ -1200,11 +1200,68 @@ std::string format_string( const std::string& frmt, const variant_object& args, void to_json_stream( const variant& v, json_writer& w ) { - // variants hold arbitrary nested structures; emit via the existing JSON serializer - // and splice the result as a raw value. No intermediate variant allocation is - // saved here (the variant already exists); the win is that reflected structs with - // a variant field participate in the streaming path instead of falling back to a - // full mutable_variant_object rebuild at the caller. - w.raw_value( fc::json::to_string( v, fc::json::yield_function_t() ) ); + // Walk the variant tree token-by-token directly into json_writer instead of + // building a JSON string via fc::json::to_string and splicing it back in. Mirrors + // the compact (indent=0) form of fc::json::to_string for byte-identical output. + switch( v.get_type() ) { + case variant::null_type: + w.value_null(); + return; + case variant::int64_type: { + int64_t i = v.as_int64(); + constexpr int64_t max_value(0xffffffff); + if( i > max_value || i < -max_value ) { + // Quote out-of-range integers so 64-bit JS clients don't lose precision. + w.value_string( v.as_string() ); + } else { + w.value_int64( i ); + } + return; + } + case variant::uint64_type: { + uint64_t i = v.as_uint64(); + if( i > 0xffffffff ) { + w.value_string( v.as_string() ); + } else { + w.value_uint64( i ); + } + return; + } + case variant::int128_type: + case variant::uint128_type: + case variant::int256_type: + case variant::uint256_type: + // Always quoted; representation is a decimal string already. + w.value_string( v.as_string() ); + return; + case variant::double_type: + // Quoted form matches fc::json::to_string's existing convention. + w.value_string( v.as_string() ); + return; + case variant::bool_type: + w.value_bool( v.as_bool() ); + return; + case variant::string_type: + case variant::string_sso_type: + w.value_string( v.get_string() ); + return; + case variant::blob_type: + // as_string() returns the base64-encoded form for blobs; matches the existing + // fc::json::to_string blob handling. + w.value_string( v.as_string() ); + return; + case variant::array_type: { + const variants& a = v.get_array(); + w.begin_array(); + for( const auto& e : a ) to_json_stream( e, w ); + w.end_array(); + return; + } + case variant::object_type: + to_json_stream( v.get_object(), w ); + return; + default: + FC_THROW_EXCEPTION( fc::invalid_arg_exception, "Unsupported variant type: {}", std::to_string( v.get_type() ) ); + } } } // namespace fc diff --git a/libraries/libfc/src/variant_object.cpp b/libraries/libfc/src/variant_object.cpp index 908c39a10c..6ce81f20c0 100644 --- a/libraries/libfc/src/variant_object.cpp +++ b/libraries/libfc/src/variant_object.cpp @@ -499,18 +499,27 @@ namespace fc void to_json_stream( const variant_object& vo, json_writer& w ) { - // variant_object already is an fc::variant's payload; defer to fc::json::to_string - // and splice the serialized form. Avoids rebuilding the object field-by-field on - // the stream path when the caller is handing us a prebuilt object (eg legacy code - // that produced a variant_object it wants to embed into a streaming response). - variant tmp = variant(vo); - w.raw_value( fc::json::to_string( tmp, fc::json::yield_function_t() ) ); + // Walk the object's key/value pairs directly into json_writer. Mirrors the + // compact (indent=0) form of fc::json::to_string for byte-identical output; + // values recurse into to_json_stream(variant, w) which handles nested arrays / + // objects without going through fc::json::to_string at any level. + w.begin_object(); + for( const auto& kv : vo ) { + w.key( kv.key() ); + to_json_stream( kv.value(), w ); + } + w.end_object(); } void to_json_stream( const mutable_variant_object& vo, json_writer& w ) { - variant tmp = variant(vo); - w.raw_value( fc::json::to_string( tmp, fc::json::yield_function_t() ) ); + // mutable_variant_object iterates the same pair shape as variant_object. + w.begin_object(); + for( const auto& kv : vo ) { + w.key( kv.key() ); + to_json_stream( kv.value(), w ); + } + w.end_object(); } void from_variant( const variant& var, mutable_variant_object& vo ) From 357f9134a559c5db5a8b1cd6f0c3bad751f266e8 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Mon, 27 Apr 2026 19:54:05 -0500 Subject: [PATCH 26/71] chain_api_plugin: migrate get_block_info / get_block_header_state / get_currency_stats to streaming-cb path Now that to_json_stream(variant, w) walks the variant tree directly into json_writer (no fc::json::to_string bridge), the three fc::variant-returning endpoints get real streaming on the response path. Their chain_plugin methods still build a fc::variant first (same as the variant-cb path), but the streaming-cb path now avoids the per-response JSON string allocation that the variant-cb path pays. Simple add_api -> add_api_stream move; no other change. --- plugins/chain_api_plugin/src/chain_api_plugin.cpp | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/plugins/chain_api_plugin/src/chain_api_plugin.cpp b/plugins/chain_api_plugin/src/chain_api_plugin.cpp index 819091b143..cd9c2105f0 100644 --- a/plugins/chain_api_plugin/src/chain_api_plugin.cpp +++ b/plugins/chain_api_plugin/src/chain_api_plugin.cpp @@ -136,13 +136,6 @@ void chain_api_plugin::plugin_startup() { }); _http_plugin.add_api({ - // get_block_info / get_block_header_state / get_currency_stats return fc::variant - // directly. Streaming them would just splice fc::json::to_string(v) - no real win - // until the streaming variant walker (rewrite of to_json_stream(variant, w)) lands, - // so they stay on the variant-cb path for now. - CHAIN_RO_CALL(get_block_info, 200, http_params_types::params_required), - CHAIN_RO_CALL(get_block_header_state, 200, http_params_types::params_required), - CHAIN_RO_CALL(get_currency_stats, 200, http_params_types::params_required), // transaction related APIs will be posted to read_write queue after keys are recovered, they are safe to run in parallel until they post to the read_write queue CHAIN_RO_CALL_ASYNC(compute_transaction, chain_apis::read_only::compute_transaction_results, 200, http_params_types::params_required), CHAIN_RW_CALL_ASYNC(push_transaction, chain_apis::read_write::push_transaction_results, 202, http_params_types::params_required), @@ -157,6 +150,8 @@ void chain_api_plugin::plugin_startup() { _http_plugin.add_api_stream({ CHAIN_RO_CALL_STREAM(get_activated_protocol_features, chain_apis::read_only::get_activated_protocol_features_results, 200, http_params_types::possible_no_params), CHAIN_RO_CALL_STREAM_POST(get_block, fc::variant, 200, http_params_types::params_required), // _POST because get_block() returns a lambda to be executed on the http thread pool + CHAIN_RO_CALL_STREAM(get_block_info, fc::variant, 200, http_params_types::params_required), + CHAIN_RO_CALL_STREAM(get_block_header_state, fc::variant, 200, http_params_types::params_required), CHAIN_RO_CALL_STREAM(get_code, chain_apis::read_only::get_code_results, 200, http_params_types::params_required), CHAIN_RO_CALL_STREAM(get_code_hash, chain_apis::read_only::get_code_hash_results, 200, http_params_types::params_required), CHAIN_RO_CALL_STREAM(get_consensus_parameters, chain_apis::read_only::get_consensus_parameters_results, 200, http_params_types::no_params), @@ -166,6 +161,7 @@ void chain_api_plugin::plugin_startup() { CHAIN_RO_CALL_STREAM(get_finalizer_info, chain_apis::read_only::get_finalizer_info_result, 200, http_params_types::no_params), CHAIN_RO_CALL_STREAM(get_table_by_scope, chain_apis::read_only::get_table_by_scope_result, 200, http_params_types::params_required), CHAIN_RO_CALL_STREAM(get_currency_balance, std::vector, 200, http_params_types::params_required), + CHAIN_RO_CALL_STREAM(get_currency_stats, fc::variant, 200, http_params_types::params_required), CHAIN_RO_CALL_STREAM(get_producers, chain_apis::read_only::get_producers_result, 200, http_params_types::params_required), CHAIN_RO_CALL_STREAM(get_producer_schedule, chain_apis::read_only::get_producer_schedule_result, 200, http_params_types::no_params), CHAIN_RO_CALL_STREAM(get_required_keys, chain_apis::read_only::get_required_keys_result, 200, http_params_types::params_required), From 14b1400da5c33d8e02ea021e70fe20eb3aa9449c Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Tue, 28 Apr 2026 09:38:31 -0500 Subject: [PATCH 27/71] abi_serializer: add streaming JSON path (binary_to_json_stream) Adds a templated _binary_walk alongside two sinks that share the same recursion shape: - variant_sink builds an fc::variant tree and replaces the previous _binary_to_variant recursion (parity gate; output byte-identical). - stream_sink emits JSON tokens directly into an fc::json_writer via a static per-type unpack table that mirrors built_in_types. Adds the missing to_json_stream overloads the streaming path needs: int128/uint128 (always quoted decimal), fc::signed_int / fc::unsigned_int, float128, std::vector (hex string instead of array), and the abi_serializer-internal quoting policy for int64/uint64 above 0xffffffff and the digits10+2 fixed-precision form for float32/float64. Public surface: - abi_serializer::binary_to_json_stream paralleling binary_to_variant (bytes/datastream x yield/microseconds), feeding stream_sink. - abi_serializer::find_built_in lookup helper used by the sinks. - abi_serializer::pb_binary_to_json_string streaming bridge for protobuf types (MessageToJsonString -> json_writer::raw_value). abi_tests/binary_to_json_stream_parity asserts byte-identical output between to_json_string(binary_to_variant(...)) and binary_to_json_stream(...) over the same fixture the existing general round-trip test exercises. --- libraries/chain/abi_serializer.cpp | 792 ++++++++++++++---- .../include/sysio/chain/abi_serializer.hpp | 58 +- .../chain/include/sysio/chain/abi_sinks.hpp | 150 ++++ .../include/sysio/chain/database_utils.hpp | 15 +- libraries/libfc/include/fc/int128.hpp | 6 + libraries/libfc/include/fc/io/varint.hpp | 6 + libraries/libfc/include/fc/variant.hpp | 4 + libraries/libfc/src/int128.cpp | 9 + libraries/libfc/src/io/varint.cpp | 4 + libraries/libfc/src/variant.cpp | 6 + unittests/abi_tests.cpp | 47 +- 11 files changed, 930 insertions(+), 167 deletions(-) create mode 100644 libraries/chain/include/sysio/chain/abi_sinks.hpp diff --git a/libraries/chain/abi_serializer.cpp b/libraries/chain/abi_serializer.cpp index f999e37af4..9442bbb7cd 100644 --- a/libraries/chain/abi_serializer.cpp +++ b/libraries/chain/abi_serializer.cpp @@ -1,5 +1,7 @@ #include +#include #include +#include #include #include #include @@ -10,6 +12,10 @@ #include #include +#include +#include +#include + namespace sysio::chain { const size_t abi_serializer::max_recursion_depth; @@ -73,6 +79,168 @@ namespace sysio::chain { ); } + // Stream-side counterpart to `pack_unpack`. The lambda reads a single value of type + // `T` from the datastream and forwards to the per-type `emit(value, json_writer&)` + // callback, which is responsible for matching the variant path's exact JSON shape + // (quote conventions for large 64-bit ints, decimal strings for 128-bit, etc). + template + auto pack_unpack_stream(Emit emit) { + return [emit]( fc::datastream& stream, bool is_array, bool is_optional, + const abi_serializer::yield_function_t& yield, fc::json_writer& w ) { + if( is_array ) { + fc::unsigned_int sz; + fc::raw::unpack(stream, sz); + w.begin_array(); + for( fc::unsigned_int::base_uint i = 0; i < sz.value; ++i ) { + T val; + fc::raw::unpack(stream, val); + emit(val, w); + } + w.end_array(); + } else if( is_optional ) { + char flag; + fc::raw::unpack(stream, flag); + if( flag ) { + T val; + fc::raw::unpack(stream, val); + emit(val, w); + } else { + w.value_null(); + } + } else { + T val; + fc::raw::unpack(stream, val); + emit(val, w); + } + }; + } + + // Same as `pack_unpack_stream`, but invokes `yield(0)` on each value read so a + // long signature/public_key decode can bail out at the configured deadline. + template + auto pack_unpack_stream_deadline(Emit emit) { + return [emit]( fc::datastream& stream, bool is_array, bool is_optional, + const abi_serializer::yield_function_t& yield, fc::json_writer& w ) { + if( is_array ) { + fc::unsigned_int sz; + fc::raw::unpack(stream, sz); + w.begin_array(); + for( fc::unsigned_int::base_uint i = 0; i < sz.value; ++i ) { + T val; + fc::raw::unpack(stream, val); + yield(0); + emit(val, w); + } + w.end_array(); + } else if( is_optional ) { + char flag; + fc::raw::unpack(stream, flag); + if( flag ) { + T val; + fc::raw::unpack(stream, val); + yield(0); + emit(val, w); + } else { + w.value_null(); + } + } else { + T val; + fc::raw::unpack(stream, val); + yield(0); + emit(val, w); + } + }; + } + + using built_in_stream_unpack_fn = std::function&, bool, bool, + const abi_serializer::yield_function_t&, + fc::json_writer&)>; + + // Format double with the same fixed-precision spelling used by `variant::as_string()` + // for double_type variants (digits10 + 2, std::fixed). The variant path emits this + // form quoted via to_json_stream(variant) -> w.value_string(v.as_string()). + inline std::string double_as_string_for_abi(double d) { + std::stringstream ss; + ss << std::setprecision(std::numeric_limits::digits10 + 2) << std::fixed << d; + return ss.str(); + } + + // Static dispatch table for the streaming side of `unpack_built_in`. Built once on + // first access and then read-only. Variant-side overrides via + // `add_specialized_unpack_pack` are not reflected here; the streaming consumers + // (chain_api endpoints) do not register overrides today. + const std::map>& get_built_in_stream_unpacks() { + static const auto map = []() { + std::map> m; + + auto generic = [](const auto& v, fc::json_writer& w) { fc::to_json_stream(v, w); }; + + // int64/uint64: variant path quotes when |v| > 0xffffffff (JS precision guard). + constexpr int64_t i64_quote_above = 0xffffffffLL; + constexpr int64_t i64_quote_below = -0xffffffffLL; + constexpr uint64_t u64_quote_above = 0xffffffffULL; + auto emit_int64 = [=](int64_t v, fc::json_writer& w) { + if( v > i64_quote_above || v < i64_quote_below ) w.value_string(std::to_string(v)); + else w.value_int64(v); + }; + auto emit_uint64 = [=](uint64_t v, fc::json_writer& w) { + if( v > u64_quote_above ) w.value_string(std::to_string(v)); + else w.value_uint64(v); + }; + + // float / double: variant path always quotes with digits10+2 fixed precision. + auto emit_double = [](double v, fc::json_writer& w) { + w.value_string(double_as_string_for_abi(v)); + }; + auto emit_float = [](float v, fc::json_writer& w) { + w.value_string(double_as_string_for_abi(static_cast(v))); + }; + + m.emplace("bool", pack_unpack_stream(generic)); + m.emplace("int8", pack_unpack_stream(generic)); + m.emplace("uint8", pack_unpack_stream(generic)); + m.emplace("int16", pack_unpack_stream(generic)); + m.emplace("uint16", pack_unpack_stream(generic)); + m.emplace("int32", pack_unpack_stream(generic)); + m.emplace("uint32", pack_unpack_stream(generic)); + m.emplace("int64", pack_unpack_stream(emit_int64)); + m.emplace("uint64", pack_unpack_stream(emit_uint64)); + m.emplace("int128", pack_unpack_stream(generic)); + m.emplace("uint128", pack_unpack_stream(generic)); + m.emplace("varint32", pack_unpack_stream(generic)); + m.emplace("varuint32", pack_unpack_stream(generic)); + + m.emplace("float32", pack_unpack_stream(emit_float)); + m.emplace("float64", pack_unpack_stream(emit_double)); + m.emplace("float128", pack_unpack_stream(generic)); + + m.emplace("time_point", pack_unpack_stream(generic)); + m.emplace("time_point_sec", pack_unpack_stream(generic)); + m.emplace("block_timestamp_type", pack_unpack_stream(generic)); + + m.emplace("name", pack_unpack_stream(generic)); + + m.emplace("bytes", pack_unpack_stream(generic)); + m.emplace("string", pack_unpack_stream(generic)); + + m.emplace("checksum160", pack_unpack_stream(generic)); + m.emplace("checksum256", pack_unpack_stream(generic)); + m.emplace("checksum512", pack_unpack_stream(generic)); + + m.emplace("public_key", pack_unpack_stream_deadline(generic)); + m.emplace("signature", pack_unpack_stream_deadline(generic)); + + m.emplace("symbol", pack_unpack_stream(generic)); + m.emplace("symbol_code", pack_unpack_stream(generic)); + m.emplace("asset", pack_unpack_stream(generic)); + m.emplace("extended_asset", pack_unpack_stream(generic)); + m.emplace("bitset", pack_unpack_stream(generic)); + + return m; + }(); + return map; + } + abi_serializer::abi_serializer( abi_def abi, const yield_function_t& yield ) { configure_built_in_types(); set_abi(std::move(abi), yield); @@ -88,6 +256,12 @@ namespace sysio::chain { built_in_types[name] = std::move( unpack_pack ); } + const std::pair* + abi_serializer::find_built_in(std::string_view type) const noexcept { + auto it = built_in_types.find(type); + return it == built_in_types.end() ? nullptr : &it->second; + } + void abi_serializer::configure_built_in_types() { built_in_types.emplace("bool", pack_unpack()); @@ -254,38 +428,63 @@ namespace sysio::chain { SYS_ASSERT( status.ok(), pack_exception, "Failed to convert JSON to protobuf message: {}", std::string(status.message()) ); } + namespace { + // Shared length-prefix decode used by both pb_binary_to_variant and + // pb_binary_to_json_string. Returns the decoded protobuf message, having + // already advanced `stream` past the consumed bytes. + std::unique_ptr pb_decode_message( + const std::string_view& type, fc::datastream& stream, + const google::protobuf::Descriptor* desc, + google::protobuf::DynamicMessageFactory& factory ) { + SYS_ASSERT( desc, unpack_exception, "Unknown protobuf type '{}'", impl::limit_size(type) ); + + // Outer varuint32 length prefix covers inner length prefix + protobuf bytes. + fc::unsigned_int outer_len; + fc::raw::unpack(stream, outer_len); + SYS_ASSERT( stream.remaining() >= outer_len.value, unpack_exception, + "Not enough data for protobuf message '{}': need {} bytes, have {}", + impl::limit_size(type), outer_len.value, stream.remaining() ); + + // Inner varuint32 length prefix (zpp_bits size_varint format). + fc::datastream inner_stream(stream.pos(), outer_len.value); + fc::unsigned_int pb_len; + fc::raw::unpack(inner_stream, pb_len); + + SYS_ASSERT( pb_len.value <= inner_stream.remaining(), unpack_exception, + "Protobuf message '{}': inner length {} exceeds available data {}", + impl::limit_size(type), pb_len.value, inner_stream.remaining() ); + SYS_ASSERT( pb_len.value == inner_stream.remaining(), unpack_exception, + "Protobuf message '{}': trailing data detected (inner length {} but {} bytes remain)", + impl::limit_size(type), pb_len.value, inner_stream.remaining() ); + + auto prototype = factory.GetPrototype(desc); + std::unique_ptr msg(prototype->New()); + SYS_ASSERT( msg->ParseFromArray(inner_stream.pos(), pb_len.value), unpack_exception, + "Failed to parse protobuf message '{}'", impl::limit_size(type) ); + stream.skip(outer_len.value); + return msg; + } + } + fc::variant abi_serializer::pb_binary_to_variant(const std::string_view& type, fc::datastream& stream) const { auto desc = get_pb_descriptor(type); - SYS_ASSERT( desc, unpack_exception, "Unknown protobuf type '{}'", impl::limit_size(type) ); - - // Read outer varuint32 length prefix (covers inner length prefix + protobuf bytes) - fc::unsigned_int outer_len; - fc::raw::unpack(stream, outer_len); - SYS_ASSERT( stream.remaining() >= outer_len.value, unpack_exception, - "Not enough data for protobuf message '{}': need {} bytes, have {}", - impl::limit_size(type), outer_len.value, stream.remaining() ); - - // Read inner varuint32 length prefix (zpp_bits size_varint format) - fc::datastream inner_stream(stream.pos(), outer_len.value); - fc::unsigned_int pb_len; - fc::raw::unpack(inner_stream, pb_len); - - SYS_ASSERT( pb_len.value <= inner_stream.remaining(), unpack_exception, - "Protobuf message '{}': inner length {} exceeds available data {}", - impl::limit_size(type), pb_len.value, inner_stream.remaining() ); - SYS_ASSERT( pb_len.value == inner_stream.remaining(), unpack_exception, - "Protobuf message '{}': trailing data detected (inner length {} but {} bytes remain)", - impl::limit_size(type), pb_len.value, inner_stream.remaining() ); - - auto prototype = pb_factory->GetPrototype(desc); - std::unique_ptr msg(prototype->New()); - SYS_ASSERT( msg->ParseFromArray(inner_stream.pos(), pb_len.value), unpack_exception, - "Failed to parse protobuf message '{}'", impl::limit_size(type) ); - stream.skip(outer_len.value); - + auto msg = pb_decode_message(type, stream, desc, *pb_factory); return pb_message_to_variant(*msg); } + std::string abi_serializer::pb_binary_to_json_string(const std::string_view& type, fc::datastream& stream) const { + namespace gpb = google::protobuf; + auto desc = get_pb_descriptor(type); + auto msg = pb_decode_message(type, stream, desc, *pb_factory); + std::string json; + gpb::util::JsonPrintOptions opts; + opts.preserve_proto_field_names = true; + auto status = gpb::util::MessageToJsonString(*msg, &json, opts); + SYS_ASSERT( status.ok(), unpack_exception, "Failed to convert protobuf message to JSON: {}", + std::string(status.message()) ); + return json; + } + void abi_serializer::pb_variant_to_binary(const std::string_view& type, const fc::variant& var, fc::datastream& ds) const { auto desc = get_pb_descriptor(type); SYS_ASSERT( desc, pack_exception, "Unknown protobuf type '{}'", impl::limit_size(type) ); @@ -503,136 +702,12 @@ namespace sysio::chain { return type; } - void abi_serializer::_binary_to_variant( const std::string_view& type, fc::datastream& stream, - fc::mutable_variant_object& obj, impl::binary_to_variant_context& ctx )const - { - auto h = ctx.enter_scope(); - auto s_itr = structs.find(type); - SYS_ASSERT( s_itr != structs.end(), invalid_type_inside_abi, "Unknown type {}", ctx.maybe_shorten(type) ); - ctx.hint_struct_type_if_in_array( s_itr ); - const auto& st = s_itr->second; - if( st.base != type_name() ) { - _binary_to_variant(resolve_type(st.base), stream, obj, ctx); - } - bool encountered_extension = false; - for( uint32_t i = 0; i < st.fields.size(); ++i ) { - const auto& field = st.fields[i]; - bool extension = field.type.ends_with("$"); - encountered_extension |= extension; - if( !stream.remaining() ) { - if( extension ) { - continue; - } - if( encountered_extension ) { - SYS_THROW( abi_exception, "Encountered field '{}' without binary extension designation while processing struct '{}'", - ctx.maybe_shorten(field.name), ctx.get_path_string() ); - } - SYS_THROW( unpack_exception, "Stream unexpectedly ended; unable to unpack field '{}' of struct '{}'", - ctx.maybe_shorten(field.name), ctx.get_path_string() ); - - } - auto h1 = ctx.push_to_path( impl::field_path_item{ .parent_struct_itr = s_itr, .field_ordinal = i } ); - auto field_type = resolve_type( extension ? _remove_bin_extension(field.type) : field.type ); - auto v = _binary_to_variant(field_type, stream, ctx); - if( ctx.is_logging() && v.is_string() && field_type == "bytes" ) { - fc::mutable_variant_object sub_obj; - auto size = v.get_string().size() / 2; // half because it is in hex - sub_obj( "size", size ); - if( size > impl::hex_log_max_size ) { - sub_obj( "trimmed_hex", v.get_string().substr( 0, impl::hex_log_max_size*2 ) ); - } else { - sub_obj( "hex", std::move( v ) ); - } - obj( field.name, std::move(sub_obj) ); - } else { - obj( field.name, std::move(v) ); - } - } - } - fc::variant abi_serializer::_binary_to_variant( const std::string_view& type, fc::datastream& stream, impl::binary_to_variant_context& ctx )const { - auto h = ctx.enter_scope(); - auto rtype = resolve_type(type); - auto ftype = fundamental_type(rtype); - auto fixed_array_sz = is_szarray(rtype); - - auto read_array = [&](fc::unsigned_int::base_uint sz) { - ctx.hint_array_type_if_in_array(); - fc::variants vars; - vars.reserve(std::min(sz, 1024u)); // limit the maximum size that can be reserved before data is read - auto h1 = ctx.push_to_path( impl::array_index_path_item{} ); - for( fc::unsigned_int::base_uint i = 0; i < sz; ++i ) { - ctx.set_array_index_of_path_back(i); - auto v = _binary_to_variant(ftype, stream, ctx); - // The exception below is commented out to allow array of optional as input data - //SYS_ASSERT( !v.is_null(), unpack_exception, "Invalid packed array '${p}'", ("p", ctx.get_path_string()) ); - vars.emplace_back(std::move(v)); - } - return fc::variant(std::move(vars)); - }; - - if (fixed_array_sz) { - return read_array(*fixed_array_sz); - } else if( auto btype = built_in_types.find(ftype ); btype != built_in_types.end() ) { - try { - return btype->second.first(stream, is_array(rtype), is_optional(rtype), ctx.get_yield_function()); - } SYS_RETHROW_EXCEPTIONS( unpack_exception, "Unable to unpack {} type '{}' while processing '{}'", - is_array(rtype) ? "array of built-in" : is_optional(rtype) ? "optional of built-in" : "built-in", - impl::limit_size(ftype), ctx.get_path_string() ) - } else if( is_protobuf_type(ftype) ) { - try { - return pb_binary_to_variant(ftype, stream); - } SYS_RETHROW_EXCEPTIONS( unpack_exception, "Unable to unpack protobuf type '{}' while processing '{}'", - impl::limit_size(ftype), ctx.get_path_string() ) - } - if ( is_array(rtype) ) { - fc::unsigned_int size; - try { - fc::raw::unpack(stream, size); - } SYS_RETHROW_EXCEPTIONS( unpack_exception, "Unable to unpack size of array '{}'", ctx.get_path_string() ) - return read_array(size.value); - } else if ( is_optional(rtype) ) { - char flag; - try { - fc::raw::unpack(stream, flag); - } SYS_RETHROW_EXCEPTIONS( unpack_exception, "Unable to unpack presence flag of optional '{}'", ctx.get_path_string() ) - return flag ? _binary_to_variant(ftype, stream, ctx) : fc::variant(); - } else if( auto e_itr = enums.find(rtype); e_itr != enums.end() ) { - // Enum type: read as underlying integer, return member name string. - // Falls back to integer for values not in the enum definition. - auto btype = built_in_types.find(e_itr->second.type); - SYS_ASSERT( btype != built_in_types.end(), invalid_type_inside_abi, - "Enum '{}' has unknown underlying type '{}'", impl::limit_size(rtype), impl::limit_size(e_itr->second.type) ); - auto int_var = btype->second.first(stream, false, false, ctx.get_yield_function()); - auto int_val = int_var.as_int64(); - for( const auto& ev : e_itr->second.values ) { - if( ev.value == int_val ) { - return fc::variant(ev.name); - } - } - return int_var; // Unknown value — return as integer - } else { - auto v_itr = variants.find(rtype); - if( v_itr != variants.end() ) { - ctx.hint_variant_type_if_in_array( v_itr ); - fc::unsigned_int select; - try { - fc::raw::unpack(stream, select); - } SYS_RETHROW_EXCEPTIONS( unpack_exception, "Unable to unpack tag of variant '{}'", ctx.get_path_string() ) - SYS_ASSERT( (size_t)select < v_itr->second.types.size(), unpack_exception, - "Unpacked invalid tag ({}) for variant '{}'", select.value, ctx.get_path_string() ); - auto h1 = ctx.push_to_path( impl::variant_path_item{ .variant_itr = v_itr, .variant_ordinal = static_cast(select) } ); - return fc::variants{v_itr->second.types[select], _binary_to_variant(v_itr->second.types[select], stream, ctx)}; - } - } - - fc::mutable_variant_object mvo; - _binary_to_variant(rtype, stream, mvo, ctx); - // QUESTION: Is this assert actually desired? It disallows unpacking empty structs from datastream. - SYS_ASSERT( mvo.size() > 0, unpack_exception, "Unable to unpack '{}' from stream", ctx.get_path_string() ); - return fc::variant( std::move(mvo) ); + impl::variant_sink sink(*this); + _binary_walk(type, stream, sink, ctx); + return std::move(sink).take_result(); } fc::variant abi_serializer::_binary_to_variant( const std::string_view& type, const bytes& binary, impl::binary_to_variant_context& ctx )const @@ -666,6 +741,34 @@ namespace sysio::chain { return _binary_to_variant(type, binary, ctx); } + void abi_serializer::binary_to_json_stream( const std::string_view& type, fc::datastream& binary, fc::json_writer& w, + const yield_function_t& yield, bool short_path )const { + impl::binary_to_variant_context ctx(*this, yield, fc::microseconds{}, type); + ctx.short_path = short_path; + impl::stream_sink sink(*this, w); + _binary_walk(type, binary, sink, ctx); + } + + void abi_serializer::binary_to_json_stream( const std::string_view& type, fc::datastream& binary, fc::json_writer& w, + const fc::microseconds& max_action_data_serialization_time, bool short_path )const { + impl::binary_to_variant_context ctx(*this, create_depth_yield_function(), max_action_data_serialization_time, type); + ctx.short_path = short_path; + impl::stream_sink sink(*this, w); + _binary_walk(type, binary, sink, ctx); + } + + void abi_serializer::binary_to_json_stream( const std::string_view& type, const bytes& binary, fc::json_writer& w, + const yield_function_t& yield, bool short_path )const { + fc::datastream ds(binary.data(), binary.size()); + binary_to_json_stream(type, ds, w, yield, short_path); + } + + void abi_serializer::binary_to_json_stream( const std::string_view& type, const bytes& binary, fc::json_writer& w, + const fc::microseconds& max_action_data_serialization_time, bool short_path )const { + fc::datastream ds(binary.data(), binary.size()); + binary_to_json_stream(type, ds, w, max_action_data_serialization_time, short_path); + } + void abi_serializer::_variant_to_binary( const std::string_view& type, const fc::variant& var, fc::datastream& ds, impl::variant_to_binary_context& ctx )const { try { auto h = ctx.enter_scope(); @@ -1199,3 +1302,384 @@ void from_variant(const fc::variant& v, sysio::chain::abi_def& abi) { } } // namespace fc + +namespace sysio::chain::impl { + +// -- variant_sink ------------------------------------------------------------------------ + +void variant_sink::begin_object() { + stack_.emplace_back(); + stack_.back().kind = frame_kind::object; +} + +void variant_sink::end_object() { + FC_ASSERT(!stack_.empty() && stack_.back().kind == frame_kind::object, + "variant_sink: end_object without matching begin_object"); + auto popped = std::move(stack_.back()); + stack_.pop_back(); + emit_value(fc::variant(std::move(popped.mvo))); +} + +void variant_sink::begin_array() { + stack_.emplace_back(); + stack_.back().kind = frame_kind::array; +} + +void variant_sink::end_array() { + FC_ASSERT(!stack_.empty() && stack_.back().kind == frame_kind::array, + "variant_sink: end_array without matching begin_array"); + auto popped = std::move(stack_.back()); + stack_.pop_back(); + emit_value(fc::variant(std::move(popped.arr))); +} + +void variant_sink::key(std::string_view k) { + FC_ASSERT(!stack_.empty() && stack_.back().kind == frame_kind::object, + "variant_sink: key emitted outside an object frame"); + auto& f = stack_.back(); + f.pending_key.assign(k.data(), k.size()); + f.has_pending_key = true; +} + +void variant_sink::value_string(std::string_view s) { emit_value(fc::variant(std::string(s))); } +void variant_sink::value_int64(int64_t n) { emit_value(fc::variant(n)); } +void variant_sink::value_uint64(uint64_t n) { emit_value(fc::variant(n)); } +void variant_sink::value_double(double d) { emit_value(fc::variant(d)); } +void variant_sink::value_bool(bool b) { emit_value(fc::variant(b)); } +void variant_sink::value_null() { emit_value(fc::variant()); } + +void variant_sink::value_hex(const char* data, size_t size) { + if (size == 0) { + emit_value(fc::variant(std::string{})); + return; + } + emit_value(fc::variant(fc::to_hex(data, size))); +} + +void variant_sink::raw_value(std::string_view raw_json) { + emit_value(fc::json::from_string(std::string(raw_json))); +} + +bool variant_sink::frame_has_items() const noexcept { + return !stack_.empty() && stack_.back().item_count > 0; +} + +void variant_sink::emit_value(fc::variant v) { + if (stack_.empty()) { + result_ = std::move(v); + return; + } + auto& f = stack_.back(); + if (f.kind == frame_kind::object) { + FC_ASSERT(f.has_pending_key, "variant_sink: value without preceding key in object frame"); + f.mvo(std::move(f.pending_key), std::move(v)); + f.pending_key.clear(); + f.has_pending_key = false; + } else { + f.arr.emplace_back(std::move(v)); + } + ++f.item_count; +} + +void variant_sink::unpack_built_in(std::string_view ftype, fc::datastream& stream, + bool is_array, bool is_optional, abi_traverse_context& ctx) { + auto bv = abi_.find_built_in(ftype); + FC_ASSERT(bv, "variant_sink::unpack_built_in: '{}' is not a built-in", std::string(ftype)); + emit_value(bv->first(stream, is_array, is_optional, ctx.get_yield_function())); +} + +void variant_sink::unpack_protobuf(std::string_view ftype, fc::datastream& stream) { + emit_value(abi_.pb_binary_to_variant(ftype, stream)); +} + +// -- stream_sink ------------------------------------------------------------------------- + +void stream_sink::begin_object() { + w_.begin_object(); + frame_items_.emplace_back(0); +} + +void stream_sink::end_object() { + w_.end_object(); + FC_ASSERT(!frame_items_.empty(), "stream_sink: end_object without matching begin_object"); + frame_items_.pop_back(); + on_value_emitted(); +} + +void stream_sink::begin_array() { + w_.begin_array(); + frame_items_.emplace_back(0); +} + +void stream_sink::end_array() { + w_.end_array(); + FC_ASSERT(!frame_items_.empty(), "stream_sink: end_array without matching begin_array"); + frame_items_.pop_back(); + on_value_emitted(); +} + +void stream_sink::key(std::string_view k) { w_.key(k); } + +void stream_sink::value_string(std::string_view s) { w_.value_string(s); on_value_emitted(); } +void stream_sink::value_int64(int64_t n) { w_.value_int64(n); on_value_emitted(); } +void stream_sink::value_uint64(uint64_t n) { w_.value_uint64(n); on_value_emitted(); } +void stream_sink::value_double(double d) { w_.value_double(d); on_value_emitted(); } +void stream_sink::value_bool(bool b) { w_.value_bool(b); on_value_emitted(); } +void stream_sink::value_null() { w_.value_null(); on_value_emitted(); } + +void stream_sink::value_hex(const char* data, size_t size) { + w_.value_hex(data, size); + on_value_emitted(); +} + +void stream_sink::raw_value(std::string_view raw_json) { + w_.raw_value(raw_json); + on_value_emitted(); +} + +void stream_sink::value_variant(const fc::variant& v) { + fc::to_json_stream(v, w_); + on_value_emitted(); +} + +bool stream_sink::frame_has_items() const noexcept { + return !frame_items_.empty() && frame_items_.back() > 0; +} + +void stream_sink::on_value_emitted() noexcept { + if (!frame_items_.empty()) ++frame_items_.back(); +} + +void stream_sink::unpack_built_in(std::string_view ftype, fc::datastream& stream, + bool is_array, bool is_optional, abi_traverse_context& ctx) { + const auto& dispatch = get_built_in_stream_unpacks(); + auto it = dispatch.find(ftype); + if( it != dispatch.end() ) { + it->second(stream, is_array, is_optional, ctx.get_yield_function(), w_); + on_value_emitted(); + return; + } + // Variant-side may carry user overrides via add_specialized_unpack_pack that the + // streaming dispatch does not mirror today. Fall back to the variant unpack + + // to_json_stream(variant, w) bridge so those overrides remain effective. + auto bv = abi_.find_built_in(ftype); + FC_ASSERT(bv, "stream_sink::unpack_built_in: '{}' is not a built-in", std::string(ftype)); + auto v = bv->first(stream, is_array, is_optional, ctx.get_yield_function()); + value_variant(v); +} + +void stream_sink::unpack_protobuf(std::string_view ftype, fc::datastream& stream) { + // MessageToJsonString already produces JSON; splice it directly into the writer. + raw_value(abi_.pb_binary_to_json_string(ftype, stream)); +} + +} // namespace sysio::chain::impl + +// -- _binary_walk et al ------------------------------------------------------------ + +namespace sysio::chain { + +template +void abi_serializer::_unpack_enum( const enum_def& e_def, fc::datastream& stream, + Sink& sink, impl::binary_to_variant_context& ctx )const +{ + auto btype = built_in_types.find(e_def.type); + SYS_ASSERT( btype != built_in_types.end(), invalid_type_inside_abi, + "Enum has unknown underlying type '{}'", impl::limit_size(e_def.type) ); + // Read the underlying integer through the variant-side functor. Both sinks emit the + // same shape (string name on hit, raw integer on miss) — the parity is structural. + auto int_var = btype->second.first(stream, false, false, ctx.get_yield_function()); + auto int_val = int_var.as_int64(); + for( const auto& ev : e_def.values ) { + if( ev.value == int_val ) { + sink.value_string(ev.name); + return; + } + } + // Unknown value — emit as the underlying integer. variant_sink reuses the just-read + // variant; stream_sink rebuilds it via to_json_stream(variant, w) inside value_variant. + if constexpr (std::is_same_v) { + sink.value_variant(std::move(int_var)); + } else { + sink.value_variant(int_var); + } +} + +template +void abi_serializer::_binary_walk_struct_fields( + const map>::const_iterator& s_itr, + fc::datastream& stream, + Sink& sink, impl::binary_to_variant_context& ctx )const +{ + const auto& st = s_itr->second; + if( st.base != type_name() ) { + auto base_itr = structs.find(resolve_type(st.base)); + SYS_ASSERT( base_itr != structs.end(), invalid_type_inside_abi, + "Unknown base type {}", impl::limit_size(st.base) ); + _binary_walk_struct_fields(base_itr, stream, sink, ctx); + } + bool encountered_extension = false; + for( uint32_t i = 0; i < st.fields.size(); ++i ) { + const auto& field = st.fields[i]; + bool extension = field.type.ends_with("$"); + encountered_extension |= extension; + if( !stream.remaining() ) { + if( extension ) { + continue; + } + if( encountered_extension ) { + SYS_THROW( abi_exception, "Encountered field '{}' without binary extension designation while processing struct '{}'", + ctx.maybe_shorten(field.name), ctx.get_path_string() ); + } + SYS_THROW( unpack_exception, "Stream unexpectedly ended; unable to unpack field '{}' of struct '{}'", + ctx.maybe_shorten(field.name), ctx.get_path_string() ); + } + auto h1 = ctx.push_to_path( impl::field_path_item{ .parent_struct_itr = s_itr, .field_ordinal = i } ); + auto field_type = resolve_type( extension ? _remove_bin_extension(field.type) : field.type ); + sink.key(field.name); + if( ctx.is_logging() && field_type == "bytes" ) { + // Logging-bytes special case: variant path replaces the hex string with + // {size: N, hex|trimmed_hex: ...}. Read raw bytes here and emit the + // sub-object directly so the streaming sink doesn't need to retroactively + // rewrite a previously-emitted value. + std::vector data; + try { + fc::raw::unpack(stream, data); + } SYS_RETHROW_EXCEPTIONS( unpack_exception, "Unable to unpack 'bytes' for field '{}' of struct '{}'", + ctx.maybe_shorten(field.name), ctx.get_path_string() ) + sink.begin_object(); + sink.key("size"); + sink.value_uint64(data.size()); + if( data.size() > impl::hex_log_max_size ) { + sink.key("trimmed_hex"); + sink.value_hex(data.data(), impl::hex_log_max_size); + } else { + sink.key("hex"); + sink.value_hex(data.data(), data.size()); + } + sink.end_object(); + } else { + _binary_walk(field_type, stream, sink, ctx); + } + } +} + +template +void abi_serializer::_binary_walk( const std::string_view& type, fc::datastream& stream, + Sink& sink, impl::binary_to_variant_context& ctx )const +{ + auto h = ctx.enter_scope(); + auto rtype = resolve_type(type); + auto ftype = fundamental_type(rtype); + auto fixed_array_sz = is_szarray(rtype); + + auto walk_array = [&](fc::unsigned_int::base_uint sz) { + ctx.hint_array_type_if_in_array(); + sink.begin_array(); + auto h1 = ctx.push_to_path( impl::array_index_path_item{} ); + for( fc::unsigned_int::base_uint i = 0; i < sz; ++i ) { + ctx.set_array_index_of_path_back(i); + _binary_walk(ftype, stream, sink, ctx); + } + sink.end_array(); + }; + + if( fixed_array_sz ) { + walk_array(*fixed_array_sz); + return; + } + + if( auto btype = built_in_types.find(ftype); btype != built_in_types.end() ) { + try { + sink.unpack_built_in(ftype, stream, is_array(rtype), is_optional(rtype), ctx); + return; + } SYS_RETHROW_EXCEPTIONS( unpack_exception, "Unable to unpack {} type '{}' while processing '{}'", + is_array(rtype) ? "array of built-in" : is_optional(rtype) ? "optional of built-in" : "built-in", + impl::limit_size(ftype), ctx.get_path_string() ) + } + + if( is_protobuf_type(ftype) ) { + try { + sink.unpack_protobuf(ftype, stream); + return; + } SYS_RETHROW_EXCEPTIONS( unpack_exception, "Unable to unpack protobuf type '{}' while processing '{}'", + impl::limit_size(ftype), ctx.get_path_string() ) + } + + if( is_array(rtype) ) { + fc::unsigned_int size; + try { + fc::raw::unpack(stream, size); + } SYS_RETHROW_EXCEPTIONS( unpack_exception, "Unable to unpack size of array '{}'", ctx.get_path_string() ) + walk_array(size.value); + return; + } + + if( is_optional(rtype) ) { + char flag; + try { + fc::raw::unpack(stream, flag); + } SYS_RETHROW_EXCEPTIONS( unpack_exception, "Unable to unpack presence flag of optional '{}'", ctx.get_path_string() ) + if( flag ) { + _binary_walk(ftype, stream, sink, ctx); + } else { + sink.value_null(); + } + return; + } + + if( auto e_itr = enums.find(rtype); e_itr != enums.end() ) { + _unpack_enum(e_itr->second, stream, sink, ctx); + return; + } + + if( auto v_itr = variants.find(rtype); v_itr != variants.end() ) { + ctx.hint_variant_type_if_in_array( v_itr ); + fc::unsigned_int select; + try { + fc::raw::unpack(stream, select); + } SYS_RETHROW_EXCEPTIONS( unpack_exception, "Unable to unpack tag of variant '{}'", ctx.get_path_string() ) + SYS_ASSERT( (size_t)select < v_itr->second.types.size(), unpack_exception, + "Unpacked invalid tag ({}) for variant '{}'", select.value, ctx.get_path_string() ); + auto h1 = ctx.push_to_path( impl::variant_path_item{ .variant_itr = v_itr, .variant_ordinal = static_cast(select) } ); + sink.begin_array(); + sink.value_string(v_itr->second.types[select]); + _binary_walk(v_itr->second.types[select], stream, sink, ctx); + sink.end_array(); + return; + } + + auto s_itr = structs.find(rtype); + SYS_ASSERT( s_itr != structs.end(), invalid_type_inside_abi, "Unknown type {}", ctx.maybe_shorten(rtype) ); + ctx.hint_struct_type_if_in_array( s_itr ); + sink.begin_object(); + _binary_walk_struct_fields(s_itr, stream, sink, ctx); + const bool emitted_any = sink.frame_has_items(); + sink.end_object(); + SYS_ASSERT( emitted_any, unpack_exception, "Unable to unpack '{}' from stream", ctx.get_path_string() ); +} + +// Explicit instantiations for the two sinks defined in abi_sinks.hpp. Keeps the +// member-template definitions out of headers without sacrificing the link surface. +template void abi_serializer::_binary_walk( + const std::string_view&, fc::datastream&, impl::variant_sink&, + impl::binary_to_variant_context&) const; +template void abi_serializer::_binary_walk( + const std::string_view&, fc::datastream&, impl::stream_sink&, + impl::binary_to_variant_context&) const; + +template void abi_serializer::_binary_walk_struct_fields( + const map>::const_iterator&, + fc::datastream&, impl::variant_sink&, impl::binary_to_variant_context&) const; +template void abi_serializer::_binary_walk_struct_fields( + const map>::const_iterator&, + fc::datastream&, impl::stream_sink&, impl::binary_to_variant_context&) const; + +template void abi_serializer::_unpack_enum( + const enum_def&, fc::datastream&, impl::variant_sink&, + impl::binary_to_variant_context&) const; +template void abi_serializer::_unpack_enum( + const enum_def&, fc::datastream&, impl::stream_sink&, + impl::binary_to_variant_context&) const; + +} // namespace sysio::chain diff --git a/libraries/chain/include/sysio/chain/abi_serializer.hpp b/libraries/chain/include/sysio/chain/abi_serializer.hpp index 5fd4791e8d..58f4236e94 100644 --- a/libraries/chain/include/sysio/chain/abi_serializer.hpp +++ b/libraries/chain/include/sysio/chain/abi_serializer.hpp @@ -8,6 +8,7 @@ #include #include #include +#include namespace google::protobuf { class DescriptorPool; @@ -32,6 +33,9 @@ namespace impl { struct binary_to_variant_context; struct variant_to_binary_context; struct action_data_to_variant_context; + + class variant_sink; + class stream_sink; } /** @@ -80,6 +84,21 @@ struct abi_serializer { fc::variant binary_to_variant( const std::string_view& type, fc::datastream& binary, const yield_function_t& yield, bool short_path = false )const; fc::variant binary_to_variant( const std::string_view& type, fc::datastream& binary, const fc::microseconds& max_action_data_serialization_time, bool short_path = false )const; + /// Streaming counterpart to `binary_to_variant`. Decodes the binary blob and + /// emits its JSON representation directly into `w` without constructing a + /// `fc::variant` tree. The output is byte-identical to + /// `fc::json::to_string(binary_to_variant(...))` for all built-in types and + /// any reflected user struct that has a corresponding `to_json_stream` overload + /// (which `FC_REFLECT` provides automatically). + void binary_to_json_stream( const std::string_view& type, const bytes& binary, fc::json_writer& w, + const yield_function_t& yield, bool short_path = false )const; + void binary_to_json_stream( const std::string_view& type, const bytes& binary, fc::json_writer& w, + const fc::microseconds& max_action_data_serialization_time, bool short_path = false )const; + void binary_to_json_stream( const std::string_view& type, fc::datastream& binary, fc::json_writer& w, + const yield_function_t& yield, bool short_path = false )const; + void binary_to_json_stream( const std::string_view& type, fc::datastream& binary, fc::json_writer& w, + const fc::microseconds& max_action_data_serialization_time, bool short_path = false )const; + bytes variant_to_binary( const std::string_view& type, const fc::variant& var, const fc::microseconds& max_action_data_serialization_time, bool short_path = false )const; bytes variant_to_binary( const std::string_view& type, const fc::variant& var, const yield_function_t& yield, bool short_path = false )const; void variant_to_binary( const std::string_view& type, const fc::variant& var, fc::datastream& ds, const fc::microseconds& max_action_data_serialization_time, bool short_path = false )const; @@ -122,6 +141,12 @@ struct abi_serializer { void add_specialized_unpack_pack( const string& name, std::pair unpack_pack ); + /// Lookup helper used by the streaming walker sinks. Returns a pointer to the + /// (unpack, pack) entry registered for `type`, or `nullptr` if it is not a + /// built-in. Const view; the underlying map is shared with `binary_to_variant` + /// dispatch so any `add_specialized_unpack_pack` overrides are reflected. + const std::pair* find_built_in(std::string_view type) const noexcept; + static constexpr size_t max_recursion_depth = 32; // arbitrary depth to prevent infinite recursion // create standard yield function that checks for max_serialization_time and max_recursion_depth. @@ -167,12 +192,39 @@ struct abi_serializer { bool is_protobuf_type(const std::string_view& type) const; const google::protobuf::Descriptor* get_pb_descriptor(const std::string_view& type) const; fc::variant pb_binary_to_variant(const std::string_view& type, fc::datastream& stream) const; + /// Streaming protobuf bridge: decodes the protobuf blob and returns the JSON + /// string produced by `MessageToJsonString` (which `pb_binary_to_variant` + /// otherwise round-trips through `fc::json::from_string`). Used by + /// `stream_sink::unpack_protobuf` to splice via `raw_value` without building + /// an intermediate variant tree. + std::string pb_binary_to_json_string(const std::string_view& type, fc::datastream& stream) const; void pb_variant_to_binary(const std::string_view& type, const fc::variant& var, fc::datastream& ds) const; fc::variant _binary_to_variant( const std::string_view& type, const bytes& binary, impl::binary_to_variant_context& ctx )const; fc::variant _binary_to_variant( const std::string_view& type, fc::datastream& binary, impl::binary_to_variant_context& ctx )const; - void _binary_to_variant( const std::string_view& type, fc::datastream& stream, - fc::mutable_variant_object& obj, impl::binary_to_variant_context& ctx )const; + + /// Templated recursive walker that emits a value of `type` from `stream` into `sink`. + /// Mirrors the recursion shape of `_binary_to_variant`: fixed-array, built-in, + /// protobuf, dynamic array, optional, enum, variant, struct. Two explicit + /// instantiations (variant_sink, stream_sink) live in `abi_serializer.cpp`. + template + void _binary_walk( const std::string_view& type, fc::datastream& stream, + Sink& sink, impl::binary_to_variant_context& ctx )const; + + /// Walk struct fields without emitting `begin_object`/`end_object`. Recurses + /// into the base struct (if any) so its fields land in the same object frame + /// the caller already opened. + template + void _binary_walk_struct_fields( const map>::const_iterator& s_itr, + fc::datastream& stream, + Sink& sink, impl::binary_to_variant_context& ctx )const; + + /// Read an enum's underlying integer from `stream` and emit either the matching + /// member name as a string, or the raw integer for unknown values. Templated + /// over sink to avoid a `fc::variant` round-trip on the streaming path. + template + void _unpack_enum( const enum_def& e_def, fc::datastream& stream, + Sink& sink, impl::binary_to_variant_context& ctx )const; bytes _variant_to_binary( const std::string_view& type, const fc::variant& var, impl::variant_to_binary_context& ctx )const; void _variant_to_binary( const std::string_view& type, const fc::variant& var, @@ -186,6 +238,8 @@ struct abi_serializer { friend struct impl::abi_from_variant; friend struct impl::abi_to_variant; friend struct impl::abi_traverse_context_with_path; + friend class impl::variant_sink; + friend class impl::stream_sink; }; namespace impl { diff --git a/libraries/chain/include/sysio/chain/abi_sinks.hpp b/libraries/chain/include/sysio/chain/abi_sinks.hpp new file mode 100644 index 0000000000..0bb89a771c --- /dev/null +++ b/libraries/chain/include/sysio/chain/abi_sinks.hpp @@ -0,0 +1,150 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace sysio::chain::impl { + +/** + * Sink consumed by `abi_serializer::_binary_walk` while it walks an ABI-typed + * binary blob. Two implementations exist in lock-step: + * + * - `variant_sink`: assembles an `fc::variant` tree. Drives the legacy + * `binary_to_variant` API after step 3 of the streaming migration. + * - `stream_sink`: emits JSON tokens directly into an `fc::json_writer` (no + * intermediate variant allocation). Drives the new `binary_to_json_stream` API. + * + * Both sinks expose the same write-side surface (`begin_object`, `value_string`, ...). + * The walker calls `unpack_built_in` / `unpack_protobuf` to delegate the type-specific + * read+emit step; each sink owns its own per-type unpack table. + */ + +/** + * Builds an `fc::variant` tree token-by-token. Object frames accumulate fields into + * a `mutable_variant_object`; array frames accumulate elements into an `fc::variants`. + * When the outermost frame closes (or a single scalar value is emitted at top level), + * `take_result()` yields the assembled variant. + */ +class variant_sink { +public: + explicit variant_sink(const abi_serializer& abi) noexcept : abi_(abi) {} + + void begin_object(); + void end_object(); + void begin_array(); + void end_array(); + void key(std::string_view k); + + void value_string(std::string_view s); + void value_int64(int64_t n); + void value_uint64(uint64_t n); + void value_int32(int32_t n) { value_int64(n); } + void value_uint32(uint32_t n) { value_uint64(n); } + void value_int16(int16_t n) { value_int64(n); } + void value_uint16(uint16_t n) { value_uint64(n); } + void value_int8(int8_t n) { value_int64(n); } + void value_uint8(uint8_t n) { value_uint64(n); } + void value_double(double d); + void value_bool(bool b); + void value_null(); + void value_hex(const char* data, size_t size); + void raw_value(std::string_view raw_json); + + /// Inject an already-built `fc::variant` at the current value position. Used by + /// `unpack_built_in` and `unpack_protobuf` to avoid re-tokenising values that the + /// existing variant-side unpack functions produce as a single `fc::variant`. + void value_variant(fc::variant v) { emit_value(std::move(v)); } + + /// True if at least one item has been added to the current frame. Used by the walker + /// to enforce the legacy "Unable to unpack '...' from stream" guard on empty structs. + bool frame_has_items() const noexcept; + + void unpack_built_in(std::string_view ftype, fc::datastream& stream, + bool is_array, bool is_optional, abi_traverse_context& ctx); + void unpack_protobuf(std::string_view ftype, fc::datastream& stream); + + fc::variant take_result() && { return std::move(result_); } + +private: + void emit_value(fc::variant v); + + const abi_serializer& abi_; + + enum class frame_kind : uint8_t { object, array }; + struct frame { + frame_kind kind; + fc::mutable_variant_object mvo; + fc::variants arr; + std::string pending_key; + bool has_pending_key = false; + uint32_t item_count = 0; + }; + std::vector stack_; + fc::variant result_; +}; + +/** + * Emits JSON tokens directly into an `fc::json_writer`. No `fc::variant` is + * constructed for built-in scalars; container framing forwards directly to the + * writer. Per-frame item counts are tracked locally so the walker can run the + * same empty-struct guard the variant path enforces. + * + * In the initial scaffolding commit, `unpack_built_in` falls back to the variant-side + * unpack and pipes the result through `to_json_stream(variant, w)`. Subsequent commits + * replace those entries with direct streaming unpacks. + */ +class stream_sink { +public: + stream_sink(const abi_serializer& abi, fc::json_writer& w) noexcept : abi_(abi), w_(w) {} + + void begin_object(); + void end_object(); + void begin_array(); + void end_array(); + void key(std::string_view k); + + void value_string(std::string_view s); + void value_int64(int64_t n); + void value_uint64(uint64_t n); + void value_int32(int32_t n) { value_int64(n); } + void value_uint32(uint32_t n) { value_uint64(n); } + void value_int16(int16_t n) { value_int64(n); } + void value_uint16(uint16_t n) { value_uint64(n); } + void value_int8(int8_t n) { value_int64(n); } + void value_uint8(uint8_t n) { value_uint64(n); } + void value_double(double d); + void value_bool(bool b); + void value_null(); + void value_hex(const char* data, size_t size); + void raw_value(std::string_view raw_json); + + /// Splice an already-built variant. Internally serialises via `to_json_stream` + /// so the writer state stays balanced. Used by the protobuf bridge today and + /// by the variant-fallback built-in path until commit 2 lands the direct unpacks. + void value_variant(const fc::variant& v); + + bool frame_has_items() const noexcept; + + void unpack_built_in(std::string_view ftype, fc::datastream& stream, + bool is_array, bool is_optional, abi_traverse_context& ctx); + void unpack_protobuf(std::string_view ftype, fc::datastream& stream); + +private: + void on_value_emitted() noexcept; + + const abi_serializer& abi_; + fc::json_writer& w_; + std::vector frame_items_; +}; + +} // namespace sysio::chain::impl diff --git a/libraries/chain/include/sysio/chain/database_utils.hpp b/libraries/chain/include/sysio/chain/database_utils.hpp index fee01386ef..8a43253c4b 100644 --- a/libraries/chain/include/sysio/chain/database_utils.hpp +++ b/libraries/chain/include/sysio/chain/database_utils.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -369,14 +370,24 @@ namespace fc { inline void to_variant( const float128_t& f, variant& v ) { - // Assumes platform is little endian and hex representation of 128-bit integer is in little endian order. + // Assumes platform is little endian and hex representation of 128-bit integer is in little endian order. char as_bytes[sizeof(sysio::chain::uint128_t)]; memcpy(as_bytes, &f, sizeof(as_bytes)); - std::string s = "0x"; + std::string s = "0x"; s.append( to_hex( as_bytes, sizeof(as_bytes) ) ); v = s; } + /// JSON shape mirrors to_variant: "0x" with little-endian bytes. + inline + void to_json_stream( const float128_t& f, json_writer& w ) { + char as_bytes[sizeof(sysio::chain::uint128_t)]; + memcpy(as_bytes, &f, sizeof(as_bytes)); + std::string s = "0x"; + s.append( to_hex( as_bytes, sizeof(as_bytes) ) ); + w.value_string(s); + } + inline void from_variant( const variant& v, float128_t& f ) { // Temporarily hold the binary in uint128_t before casting it to float128_t diff --git a/libraries/libfc/include/fc/int128.hpp b/libraries/libfc/include/fc/int128.hpp index 8a28c7d3e8..a0a0c5aebb 100644 --- a/libraries/libfc/include/fc/int128.hpp +++ b/libraries/libfc/include/fc/int128.hpp @@ -18,6 +18,7 @@ namespace fc __int128 int128_from_string( const std::string& s ); class variant; + class json_writer; fc::uint128 to_uint128(std::uint64_t hi, uint64_t lo); fc::uint128 to_uint128(const fc::variant& v); @@ -29,6 +30,11 @@ namespace fc void to_variant( const int128& var, variant& vo ); void from_variant( const variant& var, int128& vo ); + /// JSON shape: always-quoted decimal string, matching to_json_stream(variant, w) + /// for int128_type / uint128_type variants. + void to_json_stream( const uint128& var, json_writer& w ); + void to_json_stream( const int128& var, json_writer& w ); + namespace raw { template diff --git a/libraries/libfc/include/fc/io/varint.hpp b/libraries/libfc/include/fc/io/varint.hpp index 16dad3d8b6..916c5520bc 100644 --- a/libraries/libfc/include/fc/io/varint.hpp +++ b/libraries/libfc/include/fc/io/varint.hpp @@ -72,12 +72,18 @@ struct signed_int { constexpr auto format_as(const signed_int& si) { return si.value; } class variant; +class json_writer; void to_variant( const signed_int& var, variant& vo ); void from_variant( const variant& var, signed_int& vo ); void to_variant( const unsigned_int& var, variant& vo ); void from_variant( const variant& var, unsigned_int& vo ); +/// JSON shape: bare integer. Matches to_json_stream(variant) for variants built +/// from `signed_int::value` / `unsigned_int::value` (int32 / uint32 fit-range). +void to_json_stream( const signed_int& var, json_writer& w ); +void to_json_stream( const unsigned_int& var, json_writer& w ); + } // namespace fc #include diff --git a/libraries/libfc/include/fc/variant.hpp b/libraries/libfc/include/fc/variant.hpp index e5d0ff7ad9..b4d020f0c8 100644 --- a/libraries/libfc/include/fc/variant.hpp +++ b/libraries/libfc/include/fc/variant.hpp @@ -97,6 +97,10 @@ namespace fc void from_variant( const fc::variant& var, mutable_variant_object& vo ); void to_variant( const std::vector& var, fc::variant& vo ); void from_variant( const fc::variant& var, std::vector& vo ); + /// JSON shape: lowercase-hex string (matches to_variant which produces a string variant). + /// Concretely overrides the generic `to_json_stream(vector)` template (which emits an + /// array of int8s). Empty vector emits `""` to match the to_variant(empty_vec) result. + void to_json_stream( const std::vector& var, json_writer& w ); template void to_variant( const std::unordered_map& var, fc::variant& vo ); diff --git a/libraries/libfc/src/int128.cpp b/libraries/libfc/src/int128.cpp index b27e7945d4..9b2bb0a94d 100644 --- a/libraries/libfc/src/int128.cpp +++ b/libraries/libfc/src/int128.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -83,6 +84,14 @@ void from_variant(const variant& var, fc::uint128& vo) { void to_variant(const fc::int128& var, fc::variant& vo) { vo = fc::to_string(var); } + +void to_json_stream(const fc::uint128& var, fc::json_writer& w) { + w.value_string(fc::to_string(var)); +} + +void to_json_stream(const fc::int128& var, fc::json_writer& w) { + w.value_string(fc::to_string(var)); +} void from_variant(const variant& var, fc::int128& vo) { if( var.is_int128() ) { vo = var.as_int128(); diff --git a/libraries/libfc/src/io/varint.cpp b/libraries/libfc/src/io/varint.cpp index a3eaac22aa..51a5d80270 100644 --- a/libraries/libfc/src/io/varint.cpp +++ b/libraries/libfc/src/io/varint.cpp @@ -1,4 +1,5 @@ #include +#include #include namespace fc @@ -7,4 +8,7 @@ void to_variant( const signed_int& var, variant& vo ) { vo = var.value; } void from_variant( const variant& var, signed_int& vo ) { vo.value = static_cast(var.as_int64()); } void to_variant( const unsigned_int& var, variant& vo ) { vo = var.value; } void from_variant( const variant& var, unsigned_int& vo ) { vo.value = static_cast(var.as_uint64()); } + +void to_json_stream( const signed_int& var, json_writer& w ) { w.value_int32(var.value); } +void to_json_stream( const unsigned_int& var, json_writer& w ) { w.value_uint32(var.value); } } diff --git a/libraries/libfc/src/variant.cpp b/libraries/libfc/src/variant.cpp index 5ec1c73b3c..994c50d3c4 100644 --- a/libraries/libfc/src/variant.cpp +++ b/libraries/libfc/src/variant.cpp @@ -1005,6 +1005,12 @@ void to_variant( const std::vector& var, variant& vo ) vo = variant(to_hex(var.data(),var.size())); else vo = ""; } +void to_json_stream( const std::vector& var, json_writer& w ) +{ + FC_ASSERT( var.size() <= MAX_SIZE_OF_BYTE_ARRAYS ); + // value_hex with size 0 emits `""` already; no separate empty branch needed. + w.value_hex(var.data(), var.size()); +} void from_variant( const variant& var, std::vector& vo ) { std::string_view str = var.get_string(); diff --git a/unittests/abi_tests.cpp b/unittests/abi_tests.cpp index df0c559a7f..f54a94ec25 100644 --- a/unittests/abi_tests.cpp +++ b/unittests/abi_tests.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -732,14 +733,10 @@ BOOST_AUTO_TEST_CASE(uint_types) } FC_LOG_AND_RETHROW() } -BOOST_AUTO_TEST_CASE(general) -{ try { - - auto abi = sysio_contract_abi(fc::json::from_string(my_abi).as()); - - abi_serializer abis(abi_def(abi), yield_fn()); - - const char *my_other = R"=====( +// Shared fixture for `general` and the streaming-parity test. Top-level fields +// cover every built-in plus inherited base structs, large-int quoting boundaries, +// optional/array containers, and the ABI-described meta-types. +static const char* my_other_json = R"=====( { "publickey" : "SYS6MRyAjQq8ud7hVNYcfnVPJqcVpscN5So8BhtHuGYqET5GDW5CV", "publickey_arr" : ["SYS6MRyAjQq8ud7hVNYcfnVPJqcVpscN5So8BhtHuGYqET5GDW5CV","SYS6MRyAjQq8ud7hVNYcfnVPJqcVpscN5So8BhtHuGYqET5GDW5CV","SYS6MRyAjQq8ud7hVNYcfnVPJqcVpscN5So8BhtHuGYqET5GDW5CV"], @@ -932,7 +929,14 @@ BOOST_AUTO_TEST_CASE(general) } )====="; - auto var = fc::json::from_string(my_other); +BOOST_AUTO_TEST_CASE(general) +{ try { + + auto abi = sysio_contract_abi(fc::json::from_string(my_abi).as()); + + abi_serializer abis(abi_def(abi), yield_fn()); + + auto var = fc::json::from_string(my_other_json); verify_byte_round_trip_conversion(abi_serializer{std::move(abi), yield_fn()}, "A", var); } FC_LOG_AND_RETHROW() } @@ -4173,4 +4177,29 @@ BOOST_AUTO_TEST_CASE(abi_serializer_long_table_name) { try { } FC_LOG_AND_RETHROW() } +// Parity gate for `binary_to_json_stream`: the streaming path must produce +// byte-identical JSON output to `fc::json::to_string(binary_to_variant(...))`. +// Reuses `my_abi` + `my_other_json` from `general` so coverage stays in lock-step +// with the established round-trip suite. +BOOST_AUTO_TEST_CASE(binary_to_json_stream_parity) +{ try { + auto abi = sysio_contract_abi(fc::json::from_string(my_abi).as()); + abi_serializer abis(abi_def(abi), yield_fn()); + + auto fixture_var = fc::json::from_string(my_other_json); + auto packed = abis.variant_to_binary("A", fixture_var, yield_fn()); + + auto via_variant = abis.binary_to_variant("A", packed, yield_fn()); + std::string variant_json = fc::json::to_string(via_variant, get_deadline()); + + std::string streamed_json; + { + fc::json_writer w(streamed_json); + abis.binary_to_json_stream("A", packed, w, yield_fn()); + BOOST_CHECK( w.balanced() ); + } + + BOOST_CHECK_EQUAL( variant_json, streamed_json ); +} FC_LOG_AND_RETHROW() } + BOOST_AUTO_TEST_SUITE_END() From d313512a5f48fd0eec260fefb85216e9c20ca77e Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Tue, 28 Apr 2026 10:23:33 -0500 Subject: [PATCH 28/71] chain_plugin: stream get_table_rows directly into json_writer Phase 1 of get_table_rows (the chainbase row scan, scope handling, and bound parsing) is extracted into a private helper that returns a `table_rows_phase1` struct. Both the existing variant-cb method and a new `get_table_rows_stream` share the helper; only the Phase 2 closures differ. `get_table_rows_stream` returns a `std::function` that walks the collected rows on the http response thread and emits the JSON via `abi_serializer::binary_to_json_stream` per row. No per-row `mutable_variant_object` is constructed; the response is materialised straight into the writer's output buffer. A new `CALL_WITH_400_STREAM_POST_DIRECT` macro on http_plugin carries the typed emit closure end-to-end (no `call_result` template parameter) and chain_api_plugin flips `/v1/chain/get_table_rows` over to it. `filter` is intentionally not honored on the streaming path -- it is in-tree only and requires a `fc::variant` to evaluate. In-tree callers continue using `get_table_rows()`; a `SYS_ASSERT` guards against `filter` being threaded into `get_table_rows_stream` accidentally. `table_reader_tests/stream_direct_vs_variant_byte_identical` pins parity for three params shapes (full wrap + payer, paged with limit, values_only + all_rows) by comparing `fc::json::to_string(get_table_rows().result)` against `get_table_rows_stream().emit(json_writer)` byte-for-byte. --- .../chain_api_plugin/src/chain_api_plugin.cpp | 3 +- .../sysio/chain_plugin/chain_plugin.hpp | 37 +++ plugins/chain_plugin/src/chain_plugin.cpp | 219 ++++++++++-------- .../include/sysio/http_plugin/macros.hpp | 42 ++++ tests/test_get_table_rows_page.cpp | 47 ++++ 5 files changed, 250 insertions(+), 98 deletions(-) diff --git a/plugins/chain_api_plugin/src/chain_api_plugin.cpp b/plugins/chain_api_plugin/src/chain_api_plugin.cpp index cd9c2105f0..3def3dd4ee 100644 --- a/plugins/chain_api_plugin/src/chain_api_plugin.cpp +++ b/plugins/chain_api_plugin/src/chain_api_plugin.cpp @@ -117,6 +117,7 @@ parse_params key; + std::vector value; + chain::name payer; + }; + struct table_rows_phase1 { + chain::abi_def abi; + std::string tbl_name; + std::vector key_names; + std::vector key_types; + size_t scope_key_count = 0; + bool json = true; + bool show_payer = false; + bool more = false; + std::string next_key; + std::vector rows; + }; + public: static const string KEYi64; @@ -436,6 +458,15 @@ class read_only : public api_base { /// `values_only` (strip the `{key, value, payer?}` wrapper). get_table_rows_return_t get_table_rows( const get_table_rows_params& params, const fc::time_point& deadline )const; + /// Streaming counterpart to `get_table_rows`. Phase 1 (main thread) runs the same row-collection + /// logic; Phase 2 (http thread pool) returns a closure that emits the response JSON directly into + /// an `fc::json_writer` via `abi_serializer::binary_to_json_stream`, eliminating per-row variant + /// allocations. `filter` is intentionally not honored on this path since it is in-tree-only and + /// requires a `fc::variant` to evaluate; in-tree callers continue using `get_table_rows()`. + using get_table_rows_stream_emit_fn = std::function; + using get_table_rows_stream_return_t = std::function()>; + get_table_rows_stream_return_t get_table_rows_stream( const get_table_rows_params& params, const fc::time_point& deadline )const; + struct get_table_by_scope_params { name code; // mandatory name table; // optional, act as filter @@ -589,6 +620,12 @@ class read_only : public api_base { std::optional wasm_config; }; get_consensus_parameters_results get_consensus_parameters(const get_consensus_parameters_params&, const fc::time_point& deadline) const; + +private: + /// Phase 1 of `get_table_rows` / `get_table_rows_stream`: scans chainbase on the + /// main thread and returns the raw KV rows plus the ABI metadata Phase 2 needs. + /// Filter/post-pass options are applied by the caller's Phase 2 closure. + table_rows_phase1 collect_table_rows_phase1(const get_table_rows_params& params, const fc::time_point& deadline) const; }; class read_write : public api_base { diff --git a/plugins/chain_plugin/src/chain_plugin.cpp b/plugins/chain_plugin/src/chain_plugin.cpp index 0d7cc7acbe..ddde39bb4c 100644 --- a/plugins/chain_plugin/src/chain_plugin.cpp +++ b/plugins/chain_plugin/src/chain_plugin.cpp @@ -1676,8 +1676,8 @@ fc::variant strip_scope_fields(fc::variant&& full_key, size_t count) { return fc::variant(std::move(stripped)); } -read_only::get_table_rows_return_t -read_only::get_table_rows( const read_only::get_table_rows_params& p, const fc::time_point& deadline ) const { +read_only::table_rows_phase1 +read_only::collect_table_rows_phase1( const read_only::get_table_rows_params& p, const fc::time_point& deadline ) const { abi_def abi = sysio::chain_apis::get_abi( db, p.code ); const auto& tbl = get_kv_table_def( abi, p.table ); @@ -1749,23 +1749,15 @@ read_only::get_table_rows( const read_only::get_table_rows_params& p, const fc:: // else: scope is empty — iterate ALL scopes (no prefix filtering) } - // Phase 1: Collect raw rows on the main thread. - // Phase 2 (the returned lambda): ABI-decode on the http thread pool. - struct raw_row { - std::vector key; - std::vector value; - name payer; - }; - struct http_params_t { - bool json; - bool show_payer; - bool more; - std::string next_key; - std::vector rows; - }; + // Phase 1: Collect raw rows on the main thread; the caller's Phase 2 closure + // ABI-decodes on the http thread pool. + using raw_row = raw_table_row; + using http_params_t = table_rows_phase1; bool show_payer = p.show_payer.has_value() && *p.show_payer; - http_params_t hp{ p.json, show_payer, false, {}, {} }; + http_params_t hp; + hp.json = p.json; + hp.show_payer = show_payer; const auto& d = db.db(); const auto& kv_idx = d.get_index(); @@ -2040,72 +2032,12 @@ read_only::get_table_rows( const read_only::get_table_rows_params& p, const fc:: } } - // Phase 2: ABI decode on http thread pool - return [p = std::move(hp), abi = std::move(abi), table_name = p.table, - key_names = std::move(key_names), key_types = std::move(key_types), - scope_key_count, - abi_serializer_max_time = abi_serializer_max_time, - shorten_abi_errors = shorten_abi_errors, - all_rows = p.all_rows, - values_only = p.values_only.value_or(false), - filter = p.filter]() mutable -> - chain::t_or_exception { - get_table_rows_result result; - result.more = p.more; - result.next_key = std::move(p.next_key); - - abi_serializer abis; - abis.set_abi(std::move(abi), abi_serializer::create_yield_function(abi_serializer_max_time)); - auto table_type = abis.get_table_type(table_name); - - for (auto& row : p.rows) { - fc::mutable_variant_object obj; - // For secondary queries, decode the primary key as the key field - if (p.json) { - try { - auto full_key = chain::be_key_codec::decode_key( - row.key.data(), row.key.size(), key_names, key_types); - obj["key"] = strip_scope_fields(std::move(full_key), scope_key_count); - } catch (...) { - obj["key"] = fc::to_hex(row.key.data(), row.key.size()); - } - } else { - obj["key"] = fc::to_hex(row.key.data(), row.key.size()); - } - // Decode value - if (p.json && !table_type.empty() && !row.value.empty()) { - try { - obj["value"] = abis.binary_to_variant(table_type, row.value, - abi_serializer::create_yield_function(abi_serializer_max_time), - shorten_abi_errors); - } catch (...) { - obj["value"] = fc::to_hex(row.value.data(), row.value.size()); - } - } else { - obj["value"] = fc::to_hex(row.value.data(), row.value.size()); - } - if (p.show_payer) - obj["payer"] = row.payer.to_string(); - result.rows.push_back(std::move(obj)); - } - - // --- Post-pass: C++-only wrapper behaviors. Same shape as the primary-path Phase 2 below. - if (all_rows) { result.more = false; result.next_key.clear(); } - if (values_only) { - for (auto& row : result.rows) { - if (!row.is_object()) continue; - const auto& row_obj = row.get_object(); - if (!row_obj.contains("value")) continue; - fc::variant value{row_obj["value"]}; - row = std::move(value); - } - } - if (filter.has_value()) { - const auto& predicate = *filter; - std::erase_if(result.rows, [&](const fc::variant& row) { return !predicate(row); }); - } - return result; - }; + hp.abi = std::move(abi); + hp.tbl_name = p.table; + hp.key_names = std::move(key_names); + hp.key_types = std::move(key_types); + hp.scope_key_count = scope_key_count; + return hp; } // --- Primary key query path --- @@ -2216,20 +2148,29 @@ read_only::get_table_rows( const read_only::get_table_rows_params& p, const fc:: } } - return [hp = std::move(hp), abi = std::move(abi), tbl_name = p.table, - key_names = std::move(key_names), key_types = std::move(key_types), - scope_key_count, + hp.abi = std::move(abi); + hp.tbl_name = p.table; + hp.key_names = std::move(key_names); + hp.key_types = std::move(key_types); + hp.scope_key_count = scope_key_count; + return hp; +} + +read_only::get_table_rows_return_t +read_only::get_table_rows( const read_only::get_table_rows_params& p, const fc::time_point& deadline ) const { + auto hp = collect_table_rows_phase1(p, deadline); + return [hp = std::move(hp), abi_serializer_max_time = abi_serializer_max_time, - shorten_abi_errors = shorten_abi_errors, - all_rows = p.all_rows, - values_only = p.values_only.value_or(false), - filter = p.filter]() mutable + shorten_abi_errors = shorten_abi_errors, + all_rows = p.all_rows, + values_only = p.values_only.value_or(false), + filter = p.filter]() mutable -> chain::t_or_exception { read_only::get_table_rows_result result; abi_serializer abis; - abis.set_abi(std::move(abi), abi_serializer::create_yield_function(abi_serializer_max_time)); - auto value_type = abis.get_table_type(tbl_name); + abis.set_abi(std::move(hp.abi), abi_serializer::create_yield_function(abi_serializer_max_time)); + auto value_type = abis.get_table_type(hp.tbl_name); for (auto& row : hp.rows) { fc::mutable_variant_object obj; @@ -2238,8 +2179,8 @@ read_only::get_table_rows( const read_only::get_table_rows_params& p, const fc:: if (hp.json) { try { auto full_key = chain::be_key_codec::decode_key( - row.key.data(), row.key.size(), key_names, key_types); - obj("key", strip_scope_fields(std::move(full_key), scope_key_count)); + row.key.data(), row.key.size(), hp.key_names, hp.key_types); + obj("key", strip_scope_fields(std::move(full_key), hp.scope_key_count)); } catch (...) { obj("key", fc::to_hex(row.key.data(), static_cast(row.key.size()))); } @@ -2248,7 +2189,7 @@ read_only::get_table_rows( const read_only::get_table_rows_params& p, const fc:: } // Decode value -- fall back to hex if ABI decode fails - if (hp.json) { + if (hp.json && !value_type.empty() && !row.value.empty()) { try { obj("value", abis.binary_to_variant(value_type, row.value, abi_serializer::create_yield_function(abi_serializer_max_time), @@ -2266,8 +2207,8 @@ read_only::get_table_rows( const read_only::get_table_rows_params& p, const fc:: result.rows.emplace_back(std::move(obj)); } - result.more = hp.more; - result.next_key = hp.next_key; + result.more = hp.more; + result.next_key = std::move(hp.next_key); // --- Post-pass: C++-only wrapper behaviors --- // `all_rows` walked the whole scan; a resume cursor is meaningless on the way out. @@ -2301,6 +2242,90 @@ read_only::get_table_rows( const read_only::get_table_rows_params& p, const fc:: }; } +read_only::get_table_rows_stream_return_t +read_only::get_table_rows_stream( const read_only::get_table_rows_params& p, const fc::time_point& deadline ) const { + SYS_ASSERT( !p.filter.has_value(), chain::contract_table_query_exception, + "get_table_rows_stream does not support the 'filter' predicate; use get_table_rows() instead" ); + auto hp = collect_table_rows_phase1(p, deadline); + return [hp = std::move(hp), + abi_serializer_max_time = abi_serializer_max_time, + shorten_abi_errors = shorten_abi_errors, + all_rows = p.all_rows, + values_only = p.values_only.value_or(false)]() mutable + -> chain::t_or_exception { + // Build the abi_serializer on the http thread pool so the on-wire emit + // closure doesn't pay the abi_def move-construction cost during emission. + auto abis = std::make_shared(); + abis->set_abi(std::move(hp.abi), abi_serializer::create_yield_function(abi_serializer_max_time)); + auto value_type = abis->get_table_type(hp.tbl_name); + + const bool effective_more = !all_rows && hp.more; + std::string effective_next_key = all_rows ? std::string{} : std::move(hp.next_key); + + return [hp = std::move(hp), + abis = std::move(abis), + value_type = std::move(value_type), + abi_serializer_max_time, + shorten_abi_errors, + values_only, + effective_more, + effective_next_key = std::move(effective_next_key)] + (fc::json_writer& w) mutable { + auto emit_value = [&](const std::vector& data) { + if (hp.json && !value_type.empty() && !data.empty()) { + try { + abis->binary_to_json_stream(value_type, data, w, + abi_serializer::create_yield_function(abi_serializer_max_time), + shorten_abi_errors); + return; + } catch (...) { + // fall through to hex + } + } + // Empty value or non-json mode: emit hex (matches the variant path's + // fc::variant(vector) -> to_hex on emit). + w.value_hex(data.data(), data.size()); + }; + + w.begin_object(); + w.key("rows"); + w.begin_array(); + for (auto& row : hp.rows) { + if (values_only) { + emit_value(row.value); + continue; + } + w.begin_object(); + w.key("key"); + if (hp.json) { + try { + auto full_key = chain::be_key_codec::decode_key( + row.key.data(), row.key.size(), hp.key_names, hp.key_types); + fc::to_json_stream(strip_scope_fields(std::move(full_key), hp.scope_key_count), w); + } catch (...) { + w.value_hex(row.key.data(), row.key.size()); + } + } else { + w.value_hex(row.key.data(), row.key.size()); + } + w.key("value"); + emit_value(row.value); + if (hp.show_payer) { + w.key("payer"); + w.value_string(row.payer.to_string()); + } + w.end_object(); + } + w.end_array(); + w.key("more"); + w.value_bool(effective_more); + w.key("next_key"); + w.value_string(effective_next_key); + w.end_object(); + }; + }; +} + read_only::get_table_by_scope_result read_only::get_table_by_scope( const read_only::get_table_by_scope_params& p, const fc::time_point& deadline )const { diff --git a/plugins/http_plugin/include/sysio/http_plugin/macros.hpp b/plugins/http_plugin/include/sysio/http_plugin/macros.hpp index 81ce48569f..90463286f5 100644 --- a/plugins/http_plugin/include/sysio/http_plugin/macros.hpp +++ b/plugins/http_plugin/include/sysio/http_plugin/macros.hpp @@ -107,6 +107,48 @@ }} +// Direct-streaming counterpart of CALL_WITH_400_STREAM_POST. Phase 1 returns a +// closure that, when invoked on the http thread pool (Phase 2), produces the +// final json_writer-emitting closure directly -- no typed `call_result` struct +// on the response path. Used when the api builds and emits its JSON response +// without an intermediate variant or reflected struct (eg per-row binary decode). +// Method name on api_handle is `_stream` so the variant-cb method +// continues to compile alongside. +// ------------------------------------------------------------------------------------------------------ +#define CALL_WITH_400_STREAM_POST_DIRECT(api_name, category, api_handle, api_namespace, call_name, http_resp_code, params_type) \ +{std::string("/v1/" #api_name "/" #call_name), \ + api_category::category, \ + [api_handle, &_http_plugin](string&&, string&& body, url_response_stream_callback&& cb) { \ + auto deadline = api_handle.start(); \ + try { \ + auto params = parse_params(body); \ + using emit_fn_t = std::function; \ + using http_fwd_t = std::function()>; \ + http_fwd_t http_fwd(api_handle.call_name ## _stream(std::move(params), deadline)); \ + _http_plugin.post_http_thread_pool([resp_code=http_resp_code, cb=std::move(cb), \ + body=std::move(body), \ + http_fwd = std::move(http_fwd)]() mutable { \ + try { \ + chain::t_or_exception result = http_fwd(); \ + if (std::holds_alternative(result)) { \ + try { \ + throw *std::get(result); \ + } catch (...) { \ + http_plugin::handle_exception_stream(#api_name, #call_name, body, cb); \ + } \ + } else { \ + cb(resp_code, std::get(std::move(result))); \ + } \ + } catch (...) { \ + http_plugin::handle_exception_stream(#api_name, #call_name, body, cb); \ + } \ + }); \ + } catch (...) { \ + http_plugin::handle_exception_stream(#api_name, #call_name, body, cb); \ + } \ + }} + + // Streaming-cb counterpart of CALL_WITH_400_POST. Same Phase 1 (api thread) / // Phase 2 (http thread pool) split as the variant version, but Phase 2 hands // the typed call_result to cb as a json_writer-emitting closure - no fc::variant diff --git a/tests/test_get_table_rows_page.cpp b/tests/test_get_table_rows_page.cpp index 7e95377124..d120b55d41 100644 --- a/tests/test_get_table_rows_page.cpp +++ b/tests/test_get_table_rows_page.cpp @@ -301,4 +301,51 @@ BOOST_FIXTURE_TEST_CASE(streaming_vs_variant_byte_identical, validating_tester) BOOST_CHECK_EQUAL(variant_path, stream_path); } FC_LOG_AND_RETHROW() +namespace { + /// Run the new direct-streaming `get_table_rows_stream` to completion and return the + /// emitted JSON. Fails the test if the RPC produced an exception_ptr. + std::string run_stream(read_only& ro, const read_only::get_table_rows_params& p) { + auto outer = ro.get_table_rows_stream(p, fc::time_point::maximum())(); + BOOST_REQUIRE(!std::holds_alternative(outer)); + auto emit = std::get(std::move(outer)); + std::string out; + { + fc::json_writer w(out); + emit(w); + BOOST_REQUIRE(w.balanced()); + } + return out; + } +} + +// Byte-identical compat for the new direct-streaming path. `get_table_rows_stream` +// bypasses the per-row fc::variant assembly step; its emitted JSON must match +// `fc::json::to_string(fc::variant(get_table_rows().result))` for the same params. +// Any drift here would break HTTP clients that hit /v1/chain/get_table_rows after the +// chain_api_plugin migration to CHAIN_RO_CALL_STREAM_POST_DIRECT. +BOOST_FIXTURE_TEST_CASE(stream_direct_vs_variant_byte_identical, validating_tester) try { + deploy_contract(*this); + populate_numobjs(*this, 4); + + auto ro = make_read_only(*this); + + // Three shapes worth pinning: full wrapper + payer, all_rows escape hatch, and + // values_only stripped form. Each exercises a distinct branch in the stream + // emit closure (wrapped object vs bare value, more/next_key suppression). + for (auto setup : { +[](read_only::get_table_rows_params& q) { q.all_rows = true; q.show_payer = true; }, + +[](read_only::get_table_rows_params& q) { q.show_payer = true; q.limit = 2; }, + +[](read_only::get_table_rows_params& q) { q.values_only = true; q.all_rows = true; } }) { + auto p = numobjs_params(); + setup(p); + + auto via_variant = run(ro, p); + const std::string variant_json = + fc::json::to_string(fc::variant(via_variant), fc::time_point::maximum()); + + const std::string stream_json = run_stream(ro, p); + + BOOST_CHECK_EQUAL(variant_json, stream_json); + } +} FC_LOG_AND_RETHROW() + BOOST_AUTO_TEST_SUITE_END() From c432c3fd6d4aa88aa87cb055c3eafd088d0d117a Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Tue, 28 Apr 2026 15:12:49 -0500 Subject: [PATCH 29/71] abi_serializer: parameterise to_variant on Sink; migrate get_block to streaming The abi_serializer::to_variant template family now drives both the variant-building path and the new direct-streaming path through a single sink-templated body in impl::abi_to_variant. No logic is duplicated across the two output paths; variant_sink yields a fc::variant tree, stream_sink emits JSON tokens directly into a json_writer. Concretely: - impl::abi_to_variant's nine specialised overloads (action, action_trace, packed_transaction, transaction, signed_block, plus container shapes for vector/deque/shared_ptr/std::variant) now take Sink& instead of mutable_variant_object&. Each specialised add() factors its body into a sibling add_value() so top-level entry into a typed value can hit the specialisation rather than the generic reflector path. - variant_sink and stream_sink gain emit(v) (generic field emit; variant_sink builds fc::variant(v), stream_sink calls fc::to_json_stream(v, w)) and unpack_action_data(abi, type, data, ctx, short_path) (per-action ABI decode at the current frame position). - Both sinks now offer a default-constructed mode that doesn't bind to an abi_serializer; the bound-to-abi mode drives _binary_walk's built-in dispatch as before. - abi_serializer::to_json_stream public API mirrors to_variant in shape (yield-function and microseconds overloads). - read_only::get_block_stream and convert_block_stream emit get_block's response directly into json_writer via abi_serializer::to_json_stream; chain_api_plugin's /v1/chain/get_block flips to CHAIN_RO_CALL_STREAM_POST_DIRECT. Doc on get_block_stream walks the Phase 1 (main thread; chainbase abi snapshot) -> Hop 1 -> Phase 2 (http pool; build emit closure) -> Hop 2 -> Phase 3 (http pool; encode + send) sequence. Side-effects supporting the migration: - fc::enum_type::to_json_stream forwards through EnumType so FC_REFLECT_ENUM-named enums emit as their member-name strings (matching the variant path). - std::variant::to_json_stream emits a 2-element [index, value] array matching to_variant. - fc/reflect/json_stream.hpp now self-contains its fc::json::to_string dependency; abi_sinks.hpp drops its abi_serializer.hpp include in favour of forward declarations to break the circular include between sinks and the templates that drive them. verify_byte_round_trip_conversion, verify_round_trip_conversion, and verify_type_round_trip_conversion in abi_tests gain a verify_stream_matches_variant call so every type those helpers cover gets a byte-identical stream-vs-variant assertion -- linkauth, unlinkauth, updateauth, deleteauth, newaccount, setcode, setabi, packed_transaction, the full general-suite ABI, abi_std_optional, variants, aliased_variants, variant_of_aliases, extend, plus the std_array / optional / uint family. test_chain_plugin_tests/get_block_streaming_vs_variant_byte_identical pins the same parity for a populated block (newaccount + transfer) at the convert_block / convert_block_stream entry point. In-tree abi_tests call sites that previously poked impl::abi_to_variant::add with a fc::mutable_variant_object now route through the public abi_serializer::to_variant entry. --- libraries/chain/abi_serializer.cpp | 27 +- .../include/sysio/chain/abi_serializer.hpp | 605 ++++++++++-------- .../chain/include/sysio/chain/abi_sinks.hpp | 69 +- libraries/libfc/include/fc/io/enum_type.hpp | 11 + .../libfc/include/fc/reflect/json_stream.hpp | 1 + libraries/libfc/include/fc/static_variant.hpp | 10 + .../chain_api_plugin/src/chain_api_plugin.cpp | 2 +- .../sysio/chain_plugin/chain_plugin.hpp | 18 + plugins/chain_plugin/src/chain_plugin.cpp | 35 + tests/test_chain_plugin.cpp | 39 ++ unittests/abi_tests.cpp | 75 +-- 11 files changed, 583 insertions(+), 309 deletions(-) diff --git a/libraries/chain/abi_serializer.cpp b/libraries/chain/abi_serializer.cpp index 9442bbb7cd..1cc822be1f 100644 --- a/libraries/chain/abi_serializer.cpp +++ b/libraries/chain/abi_serializer.cpp @@ -1383,13 +1383,22 @@ void variant_sink::emit_value(fc::variant v) { void variant_sink::unpack_built_in(std::string_view ftype, fc::datastream& stream, bool is_array, bool is_optional, abi_traverse_context& ctx) { - auto bv = abi_.find_built_in(ftype); + FC_ASSERT(abi_, "variant_sink::unpack_built_in requires the abi_serializer-bound constructor"); + auto bv = abi_->find_built_in(ftype); FC_ASSERT(bv, "variant_sink::unpack_built_in: '{}' is not a built-in", std::string(ftype)); emit_value(bv->first(stream, is_array, is_optional, ctx.get_yield_function())); } void variant_sink::unpack_protobuf(std::string_view ftype, fc::datastream& stream) { - emit_value(abi_.pb_binary_to_variant(ftype, stream)); + FC_ASSERT(abi_, "variant_sink::unpack_protobuf requires the abi_serializer-bound constructor"); + emit_value(abi_->pb_binary_to_variant(ftype, stream)); +} + +void variant_sink::unpack_action_data(const abi_serializer& abi, std::string_view type, const bytes& data, + abi_traverse_context& ctx, bool short_path) { + action_data_to_variant_context inner(abi, ctx, type); + inner.short_path = short_path; + emit_value(abi._binary_to_variant(type, data, inner)); } // -- stream_sink ------------------------------------------------------------------------- @@ -1462,7 +1471,8 @@ void stream_sink::unpack_built_in(std::string_view ftype, fc::datastreamfind_built_in(ftype); FC_ASSERT(bv, "stream_sink::unpack_built_in: '{}' is not a built-in", std::string(ftype)); auto v = bv->first(stream, is_array, is_optional, ctx.get_yield_function()); value_variant(v); @@ -1470,7 +1480,16 @@ void stream_sink::unpack_built_in(std::string_view ftype, fc::datastream& stream) { // MessageToJsonString already produces JSON; splice it directly into the writer. - raw_value(abi_.pb_binary_to_json_string(ftype, stream)); + FC_ASSERT(abi_, "stream_sink::unpack_protobuf requires the abi_serializer-bound constructor"); + raw_value(abi_->pb_binary_to_json_string(ftype, stream)); +} + +void stream_sink::unpack_action_data(const abi_serializer& abi, std::string_view type, const bytes& data, + abi_traverse_context& ctx, bool short_path) { + action_data_to_variant_context inner(abi, ctx, type); + inner.short_path = short_path; + fc::datastream ds(data.data(), data.size()); + abi._binary_walk(type, ds, *this, inner); } } // namespace sysio::chain::impl diff --git a/libraries/chain/include/sysio/chain/abi_serializer.hpp b/libraries/chain/include/sysio/chain/abi_serializer.hpp index 58f4236e94..a87e2184df 100644 --- a/libraries/chain/include/sysio/chain/abi_serializer.hpp +++ b/libraries/chain/include/sysio/chain/abi_serializer.hpp @@ -1,5 +1,6 @@ #pragma once #include +#include #include #include #include @@ -114,6 +115,15 @@ struct abi_serializer { template static void to_log_variant( const T& o, fc::variant& vo, const Resolver& resolver, const fc::microseconds& max_action_data_serialization_time ); + /// Streaming counterpart to `to_variant`: emits the same JSON shape directly into + /// `w` without building an intermediate `fc::variant` tree. Reflected types route + /// through the same `impl::abi_to_variant` machinery, which is sink-templated -- the + /// only difference between this path and `to_variant` is the sink instance passed in. + template + static void to_json_stream( const T& o, fc::json_writer& w, const Resolver& resolver, const yield_function_t& yield ); + template + static void to_json_stream( const T& o, fc::json_writer& w, const Resolver& resolver, const fc::microseconds& max_action_data_serialization_time ); + template static void from_variant( const fc::variant& v, T& o, const Resolver& resolver, const yield_function_t& yield ); template @@ -421,161 +431,203 @@ namespace impl { template using require_abi_t = std::enable_if_t(), int>; + /// Sink-parameterised emitter for ABI-aware reflected types. The same template + /// body drives both `abi_serializer::to_variant` (variant_sink) and + /// `abi_serializer::to_json_stream` (stream_sink) -- no logic is duplicated + /// across the two output paths. Each `add` overload emits one named field at + /// the current sink frame; nested objects/arrays open a sub-frame. struct abi_to_variant { - /** - * template which overloads add for types which are not relevant to ABI information - * and can be degraded to the normal ::to_variant(...) processing - */ - template = 1> - static void add( mutable_variant_object &mvo, const char* name, const M& v, const Resolver&, abi_traverse_context& ctx ) + // Non-ABI types: bottom out into the sink's generic emit (which is + // fc::variant(v) for variant_sink and fc::to_json_stream(v, w) for stream_sink). + template = 1> + static void add( Sink& sink, std::string_view name, const M& v, const Resolver&, abi_traverse_context& ctx ) { auto h = ctx.enter_scope(); - mvo(name,v); + sink.key(name); + sink.template emit(v); } - /** - * template which overloads add for types which contain ABI information in their trees - * for these types we create new ABI aware visitors - */ - template = 1> - static void add( mutable_variant_object &mvo, const char* name, const M& v, const Resolver& resolver, abi_traverse_context& ctx ); + // ABI types: walk through fc::reflector via abi_to_variant_visitor. + template = 1> + static void add( Sink& sink, std::string_view name, const M& v, const Resolver& resolver, abi_traverse_context& ctx ); - /** - * template which overloads add for vectors of types which contain ABI information in their trees - * for these members we call ::add in order to trigger further processing - */ - template = 1> - static void add( mutable_variant_object &mvo, const char* name, const vector& v, const Resolver& resolver, abi_traverse_context& ctx ) + // vector of ABI-bearing M. + template = 1> + static void add( Sink& sink, std::string_view name, const vector& v, const Resolver& resolver, abi_traverse_context& ctx ) { auto h = ctx.enter_scope(); - fc::variants array; - array.reserve(v.size()); + sink.key(name); + sink.begin_array(); + for( const auto& iter : v ) { + add_value( sink, iter, resolver, ctx ); + } + sink.end_array(); + } - for (const auto& iter: v) { - mutable_variant_object elem_mvo; - add(elem_mvo, "_", iter, resolver, ctx); - array.emplace_back(std::move(elem_mvo["_"])); + // deque of ABI-bearing M. + template = 1> + static void add( Sink& sink, std::string_view name, const deque& v, const Resolver& resolver, abi_traverse_context& ctx ) + { + auto h = ctx.enter_scope(); + sink.key(name); + sink.begin_array(); + for( const auto& iter : v ) { + add_value( sink, iter, resolver, ctx ); } - mvo(name, std::move(array)); + sink.end_array(); } - /** - * template which overloads add for deques of types which contain ABI information in their trees - * for these members we call ::add in order to trigger further processing - */ - template = 1> - static void add( mutable_variant_object &mvo, const char* name, const deque& v, const Resolver& resolver, abi_traverse_context& ctx ) + // shared_ptr of ABI-bearing M. Null pointer is a no-op (matches the + // pre-Sink behaviour of skipping the field entirely). + template = 1> + static void add( Sink& sink, std::string_view name, const std::shared_ptr& v, const Resolver& resolver, abi_traverse_context& ctx ) { auto h = ctx.enter_scope(); - deque array; + if( !v ) return; + sink.key(name); + add_value( sink, *v, resolver, ctx ); + } - for (const auto& iter: v) { - mutable_variant_object elem_mvo; - add(elem_mvo, "_", iter, resolver, ctx); - array.emplace_back(std::move(elem_mvo["_"])); + // Element-position emitter used by container helpers above and by the + // top-level `to_variant` / `to_json_stream` entry points. Emits the + // value at the current (array or top-level) sink position with no leading + // key. Specialised for each container shape recognised by `add` so that + // recursion through nested containers does not bottom-out into the + // reflector path with a container type as `M`. + template = 1> + static void add_value( Sink& sink, const M& v, const Resolver&, abi_traverse_context& ctx ) + { + auto h = ctx.enter_scope(); + sink.template emit(v); + } + template = 1> + static void add_value( Sink& sink, const M& v, const Resolver& resolver, abi_traverse_context& ctx ); + + // Container-shape specialisations: walk through to the element add_value. + template = 1> + static void add_value( Sink& sink, const vector& v, const Resolver& resolver, abi_traverse_context& ctx ) + { + auto h = ctx.enter_scope(); + sink.begin_array(); + for( const auto& iter : v ) { + add_value( sink, iter, resolver, ctx ); } - mvo(name, std::move(array)); + sink.end_array(); } - /** - * template which overloads add for shared_ptr of types which contain ABI information in their trees - * for these members we call ::add in order to trigger further processing - */ - template = 1> - static void add( mutable_variant_object &mvo, const char* name, const std::shared_ptr& v, const Resolver& resolver, abi_traverse_context& ctx ) + template = 1> + static void add_value( Sink& sink, const deque& v, const Resolver& resolver, abi_traverse_context& ctx ) { auto h = ctx.enter_scope(); - if( !v ) return; - mutable_variant_object obj_mvo; - add(obj_mvo, "_", *v, resolver, ctx); - mvo(name, std::move(obj_mvo["_"])); + sink.begin_array(); + for( const auto& iter : v ) { + add_value( sink, iter, resolver, ctx ); + } + sink.end_array(); + } + template = 1> + static void add_value( Sink& sink, const std::shared_ptr& v, const Resolver& resolver, abi_traverse_context& ctx ) + { + auto h = ctx.enter_scope(); + if( !v ) { sink.value_null(); return; } + add_value( sink, *v, resolver, ctx ); + } + template + static void add_value( Sink& sink, const std::variant& v, const Resolver& resolver, abi_traverse_context& ctx ) + { + auto h = ctx.enter_scope(); + add_static_variant adder( sink, resolver, ctx ); + std::visit( adder, v ); } - template + // std::variant<...>: dispatch the active alternative through the visitor. + template struct add_static_variant { - mutable_variant_object& obj_mvo; + Sink& sink; const Resolver& resolver; abi_traverse_context& ctx; - - add_static_variant( mutable_variant_object& o, const Resolver& r, abi_traverse_context& ctx ) - :obj_mvo(o), resolver(r), ctx(ctx) {} + add_static_variant( Sink& s, const Resolver& r, abi_traverse_context& c ) : sink(s), resolver(r), ctx(c) {} typedef void result_type; - template void operator()( T& v )const + template void operator()( T& v ) const { - add(obj_mvo, "_", v, resolver, ctx); + add_value( sink, v, resolver, ctx ); } }; - template - static void add( mutable_variant_object &mvo, const char* name, const std::variant& v, const Resolver& resolver, abi_traverse_context& ctx ) + template + static void add( Sink& sink, std::string_view name, const std::variant& v, const Resolver& resolver, abi_traverse_context& ctx ) { auto h = ctx.enter_scope(); - mutable_variant_object obj_mvo; - add_static_variant adder(obj_mvo, resolver, ctx); - std::visit(adder, v); - mvo(name, std::move(obj_mvo["_"])); + sink.key(name); + add_static_variant adder( sink, resolver, ctx ); + std::visit( adder, v ); } - template - static bool add_special_logging( mutable_variant_object& mvo, const char* name, const action& act, const Resolver& resolver, abi_traverse_context& ctx ) { + // Logging-mode side-effect for sysio::setcode -- inject `code_hash` next to + // the existing `data`/`hex_data` fields. Returns false either way; callers + // still emit `data` afterwards. No-op outside logging mode. + template + static bool add_special_logging( Sink& sink, const action& act, const Resolver& /*resolver*/, abi_traverse_context& ctx ) { if( !ctx.is_logging() ) return false; - try { - if( act.account == sysio::chain::config::system_account_name && act.name == "setcode"_n ) { auto setcode_act = act.data_as(); if( setcode_act.code.size() > 0 ) { fc::sha256 code_hash = fc::sha256::hash(setcode_act.code.data(), (uint32_t) setcode_act.code.size()); - mvo("code_hash", code_hash); + sink.key("code_hash"); + sink.template emit(code_hash); } return false; // still want the hex data included } - } catch(...) {} // return false - return false; } - /** - * overload of to_variant_object for actions - * - * This matches the FC_REFLECT for this type, but this is provided to extract the contents of act.data - * @tparam Resolver - * @param act - * @param resolver - * @return - */ - template - static void add( mutable_variant_object &out, const char* name, const action& act, const Resolver& resolver, abi_traverse_context& ctx ) + // Emit `bytes` either as raw hex (non-logging) or as the + // `{size, hex|trimmed_hex}` log shape. Used for `data` and `hex_data` + // fields on `action`. + template + static void emit_action_bytes_field( Sink& sink, std::string_view name, const bytes& data, abi_traverse_context& ctx ) + { + sink.key(name); + if( !ctx.is_logging() ) { + sink.template emit(data); + return; + } + sink.begin_object(); + sink.key("size"); + sink.value_uint64(data.size()); + if( data.size() > impl::hex_log_max_size ) { + sink.key("trimmed_hex"); + sink.template emit( bytes(&data[0], &data[0] + impl::hex_log_max_size) ); + } else { + sink.key("hex"); + sink.template emit(data); + } + sink.end_object(); + } + + /// Action: matches the FC_REFLECT shape but routes `data` through the + /// resolver-located ABI to decode action arguments inline. The body lives + /// on `add_value` so the top-level `to_variant` entry hits the + /// special handling rather than the reflector-based generic path. + template + static void add_value( Sink& sink, const action& act, const Resolver& resolver, abi_traverse_context& ctx ) { static_assert(fc::reflector::total_member_count == 4); auto h = ctx.enter_scope(); - mutable_variant_object mvo; - mvo("account", act.account); - mvo("name", act.name); - mvo("authorization", act.authorization); + sink.begin_object(); + add( sink, "account", act.account, resolver, ctx ); + add( sink, "name", act.name, resolver, ctx ); + add( sink, "authorization", act.authorization, resolver, ctx ); - if( add_special_logging(mvo, name, act, resolver, ctx) ) { - out(name, std::move(mvo)); + if( add_special_logging( sink, act, resolver, ctx ) ) { + sink.end_object(); return; } - auto set_hex_data = [&](mutable_variant_object& mvo, const char* name, const bytes& data) { - if( !ctx.is_logging() ) { - mvo(name, data); - } else { - fc::mutable_variant_object sub_obj; - sub_obj( "size", data.size() ); - if( data.size() > impl::hex_log_max_size ) { - sub_obj( "trimmed_hex", std::vector(&data[0], &data[0] + impl::hex_log_max_size) ); - } else { - sub_obj( "hex", data ); - } - mvo(name, std::move(sub_obj)); - } - }; - + bool data_emitted = false; try { auto abi_optional = resolver(act.account); if (abi_optional) { @@ -583,211 +635,218 @@ namespace impl { auto type = abi.get_action_type(act.name); if (!type.empty()) { try { - action_data_to_variant_context _ctx(abi, ctx, type); - mvo( "data", abi._binary_to_variant( type, act.data, _ctx )); + sink.key("data"); + sink.unpack_action_data(abi, type, act.data, ctx, /*short_path*/ false); + data_emitted = true; } catch(...) { - // any failure to serialize data, then leave as not serialized - set_hex_data(mvo, "data", act.data); + // serialization failure -- fall back to hex below } - } else { - set_hex_data(mvo, "data", act.data); } - } else { - set_hex_data(mvo, "data", act.data); } - } catch(...) { - set_hex_data(mvo, "data", act.data); + } catch(...) { /* fall back to hex below */ } + if( !data_emitted ) { + emit_action_bytes_field( sink, "data", act.data, ctx ); } - set_hex_data(mvo, "hex_data", act.data); - out(name, std::move(mvo)); + emit_action_bytes_field( sink, "hex_data", act.data, ctx ); + sink.end_object(); } - /** - * overload of to_variant_object for action_trace - * - * This matches the FC_REFLECT for this type, but this is provided to extract the contents of action_trace.return_value - * @tparam Resolver - * @param action_trace - * @param resolver - * @return - */ - template - static void add( mutable_variant_object& out, const char* name, const action_trace& act_trace, const Resolver& resolver, abi_traverse_context& ctx ) + template + static void add( Sink& sink, std::string_view name, const action& act, const Resolver& resolver, abi_traverse_context& ctx ) + { + sink.key(name); + add_value( sink, act, resolver, ctx ); + } + + /// action_trace: matches FC_REFLECT shape; routes `return_value` through the + /// resolver-located ABI when an action_result type is registered. + template + static void add_value( Sink& sink, const action_trace& act_trace, const Resolver& resolver, abi_traverse_context& ctx ) { static_assert(fc::reflector::total_member_count == 19); auto h = ctx.enter_scope(); - mutable_variant_object mvo; - - mvo("action_ordinal", act_trace.action_ordinal); - mvo("creator_action_ordinal", act_trace.creator_action_ordinal); - mvo("closest_unnotified_ancestor_action_ordinal", act_trace.closest_unnotified_ancestor_action_ordinal); - mvo("receipt", act_trace.receipt); - mvo("receiver", act_trace.receiver); - add(mvo, "act", act_trace.act, resolver, ctx); - mvo("context_free", act_trace.context_free); - mvo("elapsed", act_trace.elapsed); - mvo("cpu_usage_us", act_trace.cpu_usage_us); - mvo("net_usage", act_trace.net_usage); - mvo("console", act_trace.console); - mvo("trx_id", act_trace.trx_id); - mvo("block_num", act_trace.block_num); - mvo("block_time", act_trace.block_time); - mvo("producer_block_id", act_trace.producer_block_id); - mvo("account_ram_deltas", act_trace.account_ram_deltas); - mvo("except", act_trace.except); - mvo("error_code", act_trace.error_code); - - mvo("return_value_hex_data", act_trace.return_value); - auto act = act_trace.act; + sink.begin_object(); + add( sink, "action_ordinal", act_trace.action_ordinal, resolver, ctx ); + add( sink, "creator_action_ordinal", act_trace.creator_action_ordinal, resolver, ctx ); + add( sink, "closest_unnotified_ancestor_action_ordinal", act_trace.closest_unnotified_ancestor_action_ordinal, resolver, ctx ); + add( sink, "receipt", act_trace.receipt, resolver, ctx ); + add( sink, "receiver", act_trace.receiver, resolver, ctx ); + add( sink, "act", act_trace.act, resolver, ctx ); + add( sink, "context_free", act_trace.context_free, resolver, ctx ); + add( sink, "elapsed", act_trace.elapsed, resolver, ctx ); + add( sink, "cpu_usage_us", act_trace.cpu_usage_us, resolver, ctx ); + add( sink, "net_usage", act_trace.net_usage, resolver, ctx ); + add( sink, "console", act_trace.console, resolver, ctx ); + add( sink, "trx_id", act_trace.trx_id, resolver, ctx ); + add( sink, "block_num", act_trace.block_num, resolver, ctx ); + add( sink, "block_time", act_trace.block_time, resolver, ctx ); + add( sink, "producer_block_id", act_trace.producer_block_id, resolver, ctx ); + add( sink, "account_ram_deltas", act_trace.account_ram_deltas, resolver, ctx ); + add( sink, "except", act_trace.except, resolver, ctx ); + add( sink, "error_code", act_trace.error_code, resolver, ctx ); + add( sink, "return_value_hex_data", act_trace.return_value, resolver, ctx ); + try { - auto abi_optional = resolver(act.account); + auto abi_optional = resolver(act_trace.act.account); if (abi_optional) { const abi_serializer& abi = *abi_optional; - auto type = abi.get_action_result_type(act.name); + auto type = abi.get_action_result_type(act_trace.act.name); if (!type.empty()) { - action_data_to_variant_context _ctx(abi, ctx, type); - mvo( "return_value_data", abi._binary_to_variant( type, act_trace.return_value, _ctx )); + sink.key("return_value_data"); + sink.unpack_action_data(abi, type, act_trace.return_value, ctx, /*short_path*/ false); } } } catch(...) {} - out(name, std::move(mvo)); + sink.end_object(); } - /** - * overload of to_variant_object for packed_transaction - * - * This matches the FC_REFLECT for this type, but this is provided to allow extracting the contents of ptrx.transaction - * @tparam Resolver - * @param act - * @param resolver - * @return - */ - template - static void add( mutable_variant_object &out, const char* name, const packed_transaction& ptrx, const Resolver& resolver, abi_traverse_context& ctx ) + template + static void add( Sink& sink, std::string_view name, const action_trace& act_trace, const Resolver& resolver, abi_traverse_context& ctx ) + { + sink.key(name); + add_value( sink, act_trace, resolver, ctx ); + } + + /// packed_transaction: extract the inner transaction so consumers see action + /// data fully decoded rather than a packed-bytes blob. + template + static void add_value( Sink& sink, const packed_transaction& ptrx, const Resolver& resolver, abi_traverse_context& ctx ) { static_assert(fc::reflector::total_member_count == 4); auto h = ctx.enter_scope(); - mutable_variant_object mvo; + sink.begin_object(); auto trx = ptrx.get_transaction(); - mvo("id", trx.id()); - mvo("signatures", ptrx.get_signatures()); - mvo("compression", ptrx.get_compression()); - mvo("packed_context_free_data", ptrx.get_packed_context_free_data()); - mvo("context_free_data", ptrx.get_context_free_data()); + add( sink, "id", trx.id(), resolver, ctx ); + add( sink, "signatures", ptrx.get_signatures(), resolver, ctx ); + add( sink, "compression", ptrx.get_compression(), resolver, ctx ); + add( sink, "packed_context_free_data", ptrx.get_packed_context_free_data(), resolver, ctx ); + add( sink, "context_free_data", ptrx.get_context_free_data(), resolver, ctx ); if( !ctx.is_logging() ) - mvo("packed_trx", ptrx.get_packed_transaction()); - add(mvo, "transaction", trx, resolver, ctx); + add( sink, "packed_trx", ptrx.get_packed_transaction(), resolver, ctx ); + add( sink, "transaction", trx, resolver, ctx ); + sink.end_object(); + } - out(name, std::move(mvo)); + template + static void add( Sink& sink, std::string_view name, const packed_transaction& ptrx, const Resolver& resolver, abi_traverse_context& ctx ) + { + sink.key(name); + add_value( sink, ptrx, resolver, ctx ); } - /** - * overload of to_variant_object for transaction - * - * This matches the FC_REFLECT for this type, but this is provided to allow extracting the contents of trx.transaction_extensions - */ - template - static void add( mutable_variant_object &out, const char* name, const transaction& trx, const Resolver& resolver, abi_traverse_context& ctx ) + /// transaction: per FC_REFLECT but recurses into context_free_actions and + /// actions through the ABI-aware action overload above. + template + static void add_value( Sink& sink, const transaction& trx, const Resolver& resolver, abi_traverse_context& ctx ) { static_assert(fc::reflector::total_member_count == 9); auto h = ctx.enter_scope(); - mutable_variant_object mvo; - mvo("expiration", trx.expiration); - mvo("ref_block_num", trx.ref_block_num); - mvo("ref_block_prefix", trx.ref_block_prefix); - mvo("max_net_usage_words", trx.max_net_usage_words); - mvo("max_cpu_usage_ms", trx.max_cpu_usage_ms); - mvo("delay_sec", trx.delay_sec); - add(mvo, "context_free_actions", trx.context_free_actions, resolver, ctx); - add(mvo, "actions", trx.actions, resolver, ctx); - + sink.begin_object(); + add( sink, "expiration", trx.expiration, resolver, ctx ); + add( sink, "ref_block_num", trx.ref_block_num, resolver, ctx ); + add( sink, "ref_block_prefix", trx.ref_block_prefix, resolver, ctx ); + add( sink, "max_net_usage_words", trx.max_net_usage_words, resolver, ctx ); + add( sink, "max_cpu_usage_ms", trx.max_cpu_usage_ms, resolver, ctx ); + add( sink, "delay_sec", trx.delay_sec, resolver, ctx ); + add( sink, "context_free_actions", trx.context_free_actions, resolver, ctx ); + add( sink, "actions", trx.actions, resolver, ctx ); // No transaction extensions are currently supported. - // When extensions are added in the future, deserialize them here. + sink.end_object(); + } - out(name, std::move(mvo)); + template + static void add( Sink& sink, std::string_view name, const transaction& trx, const Resolver& resolver, abi_traverse_context& ctx ) + { + sink.key(name); + add_value( sink, trx, resolver, ctx ); } - /** - * overload of to_variant_object for signed_block - * - * This matches the FC_REFLECT for this type, but this is provided to allow extracting the contents of - * block.header_extensions and block.block_extensions - */ - template - static void add( mutable_variant_object &out, const char* name, const signed_block& block, const Resolver& resolver, abi_traverse_context& ctx ) + /// Emit the inline (no begin/end_object) field set for `signed_block`. + /// Exposed so that callers with extra wrapper fields (eg `convert_block`'s + /// `id` / `block_num` / `ref_block_prefix`) can interleave them with the + /// reflected-block fields inside a single JSON object. + template + static void emit_signed_block_body( Sink& sink, const signed_block& block, const Resolver& resolver, abi_traverse_context& ctx ) { static_assert(fc::reflector::total_member_count == 13); - auto h = ctx.enter_scope(); - mutable_variant_object mvo; - mvo("timestamp", block.timestamp); - mvo("producer", block.producer); - mvo("previous", block.previous); - mvo("transaction_mroot", block.transaction_mroot); - mvo("finality_mroot", block.finality_mroot); - mvo("qc_claim", block.qc_claim); - mvo("new_finalizer_policy_diff", block.new_finalizer_policy_diff); - mvo("new_proposer_policy_diff", block.new_proposer_policy_diff); - - // process contents of block.header_extensions + add( sink, "timestamp", block.timestamp, resolver, ctx ); + add( sink, "producer", block.producer, resolver, ctx ); + add( sink, "previous", block.previous, resolver, ctx ); + add( sink, "transaction_mroot", block.transaction_mroot, resolver, ctx ); + add( sink, "finality_mroot", block.finality_mroot, resolver, ctx ); + add( sink, "qc_claim", block.qc_claim, resolver, ctx ); + add( sink, "new_finalizer_policy_diff", block.new_finalizer_policy_diff, resolver, ctx ); + add( sink, "new_proposer_policy_diff", block.new_proposer_policy_diff, resolver, ctx ); + flat_multimap header_exts = block.validate_and_extract_header_extensions(); if (auto it = header_exts.find(protocol_feature_activation::extension_id()); it != header_exts.end()) { const auto& new_protocol_features = std::get(it->second).protocol_features; - fc::variants pf_array; - pf_array.reserve(new_protocol_features.size()); + sink.key("new_protocol_features"); + sink.begin_array(); for (auto feature : new_protocol_features) { - mutable_variant_object feature_mvo; - add(feature_mvo, "feature_digest", feature, resolver, ctx); - pf_array.push_back(std::move(feature_mvo)); + sink.begin_object(); + add( sink, "feature_digest", feature, resolver, ctx ); + sink.end_object(); } - mvo("new_protocol_features", pf_array); + sink.end_array(); } - mvo("producer_signatures", block.producer_signatures); - add(mvo, "transactions", block.transactions, resolver, ctx); + add( sink, "producer_signatures", block.producer_signatures, resolver, ctx ); + add( sink, "transactions", block.transactions, resolver, ctx ); if (block.qc) { - mvo("qc", *block.qc); + add( sink, "qc", *block.qc, resolver, ctx ); } + } + + template + static void add_value( Sink& sink, const signed_block& block, const Resolver& resolver, abi_traverse_context& ctx ) + { + auto h = ctx.enter_scope(); + sink.begin_object(); + emit_signed_block_body( sink, block, resolver, ctx ); + sink.end_object(); + } - out(name, std::move(mvo)); + template + static void add( Sink& sink, std::string_view name, const signed_block& block, const Resolver& resolver, abi_traverse_context& ctx ) + { + sink.key(name); + add_value( sink, block, resolver, ctx ); } }; /** - * Reflection visitor that uses a resolver to resolve ABIs for nested types - * this will degrade to the common fc::to_variant as soon as the type no longer contains - * ABI related info + * Reflection visitor that drives `abi_to_variant::add` over each member of T, + * resolving ABIs for nested ABI-bearing types via the supplied Resolver. Now + * sink-templated so the same visitor walks reflected fields for both the + * variant-building and stream-emitting paths. * - * @tparam Resolver - callable with the signature (const name& code_account) -> std::optional + * @tparam Sink - emit target (variant_sink or stream_sink) + * @tparam T - the reflected type whose members are being visited + * @tparam Resolver - callable with signature (const name&) -> std::optional */ - template + template class abi_to_variant_visitor { public: - abi_to_variant_visitor( mutable_variant_object& _mvo, const T& _val, const Resolver& _resolver, abi_traverse_context& _ctx ) - :_vo(_mvo) - ,_val(_val) - ,_resolver(_resolver) - ,_ctx(_ctx) + abi_to_variant_visitor( Sink& s, const T& v, const Resolver& r, abi_traverse_context& c ) + : _sink(s) + , _val(v) + , _resolver(r) + , _ctx(c) {} - /** - * Visit a single member and add it to the variant object - * @tparam Member - the member to visit - * @tparam Class - the class we are traversing - * @tparam member - pointer to the member - * @param name - the name of the member - */ - template + template void operator()( const char* name )const { - abi_to_variant::add( _vo, name, (_val.*member), _resolver, _ctx ); + abi_to_variant::add( _sink, name, (_val.*member), _resolver, _ctx ); } private: - mutable_variant_object& _vo; - const T& _val; - const Resolver& _resolver; + Sink& _sink; + const T& _val; + const Resolver& _resolver; abi_traverse_context& _ctx; }; @@ -999,13 +1058,23 @@ namespace impl { abi_traverse_context& _ctx; }; - template> - void abi_to_variant::add( mutable_variant_object &mvo, const char* name, const M& v, const Resolver& resolver, abi_traverse_context& ctx ) + template> + void abi_to_variant::add( Sink& sink, std::string_view name, const M& v, const Resolver& resolver, abi_traverse_context& ctx ) + { + auto h = ctx.enter_scope(); + sink.key(name); + sink.begin_object(); + fc::reflector::visit( impl::abi_to_variant_visitor(sink, v, resolver, ctx) ); + sink.end_object(); + } + + template> + void abi_to_variant::add_value( Sink& sink, const M& v, const Resolver& resolver, abi_traverse_context& ctx ) { auto h = ctx.enter_scope(); - mutable_variant_object member_mvo; - fc::reflector::visit( impl::abi_to_variant_visitor( member_mvo, v, resolver, ctx) ); - mvo(name, std::move(member_mvo)); + sink.begin_object(); + fc::reflector::visit( impl::abi_to_variant_visitor(sink, v, resolver, ctx) ); + sink.end_object(); } template> @@ -1019,36 +1088,50 @@ namespace impl { template void abi_serializer::to_variant( const T& o, fc::variant& vo, const Resolver& resolver, const yield_function_t& yield ) try { - mutable_variant_object mvo; + impl::variant_sink sink; impl::abi_traverse_context ctx( yield, fc::microseconds{} ); - impl::abi_to_variant::add(mvo, "_", o, resolver, ctx); - vo = std::move(mvo["_"]); + impl::abi_to_variant::add_value(sink, o, resolver, ctx); + vo = std::move(sink).take_result(); } FC_RETHROW_EXCEPTIONS(error, "Failed to serialize: {}", boost::core::demangle( typeid(o).name() )) template void abi_serializer::to_variant( const T& o, fc::variant& vo, const Resolver& resolver, const fc::microseconds& max_action_data_serialization_time ) try { - mutable_variant_object mvo; + impl::variant_sink sink; impl::abi_traverse_context ctx( create_depth_yield_function(), max_action_data_serialization_time ); - impl::abi_to_variant::add(mvo, "_", o, resolver, ctx); - vo = std::move(mvo["_"]); + impl::abi_to_variant::add_value(sink, o, resolver, ctx); + vo = std::move(sink).take_result(); } FC_RETHROW_EXCEPTIONS(error, "Failed to serialize: {}", boost::core::demangle( typeid(o).name() )) template void abi_serializer::to_log_variant( const T& o, fc::variant& vo, const Resolver& resolver, const yield_function_t& yield ) try { - mutable_variant_object mvo; - impl::abi_traverse_context ctx( yield, fc::microseconds{} ); - ctx.logging(); - impl::abi_to_variant::add(mvo, "_", o, resolver, ctx); - vo = std::move(mvo["_"]); + impl::variant_sink sink; + impl::abi_traverse_context ctx( yield, fc::microseconds{} ); + ctx.logging(); + impl::abi_to_variant::add_value(sink, o, resolver, ctx); + vo = std::move(sink).take_result(); } FC_RETHROW_EXCEPTIONS(error, "Failed to serialize: {}", boost::core::demangle( typeid(o).name() )) template void abi_serializer::to_log_variant( const T& o, fc::variant& vo, const Resolver& resolver, const fc::microseconds& max_action_data_serialization_time ) try { - mutable_variant_object mvo; + impl::variant_sink sink; impl::abi_traverse_context ctx( create_depth_yield_function(), max_action_data_serialization_time ); ctx.logging(); - impl::abi_to_variant::add(mvo, "_", o, resolver, ctx); - vo = std::move(mvo["_"]); + impl::abi_to_variant::add_value(sink, o, resolver, ctx); + vo = std::move(sink).take_result(); +} FC_RETHROW_EXCEPTIONS(error, "Failed to serialize: {}", boost::core::demangle( typeid(o).name() )) + +template +void abi_serializer::to_json_stream( const T& o, fc::json_writer& w, const Resolver& resolver, const yield_function_t& yield ) try { + impl::stream_sink sink(w); + impl::abi_traverse_context ctx( yield, fc::microseconds{} ); + impl::abi_to_variant::add_value(sink, o, resolver, ctx); +} FC_RETHROW_EXCEPTIONS(error, "Failed to serialize: {}", boost::core::demangle( typeid(o).name() )) + +template +void abi_serializer::to_json_stream( const T& o, fc::json_writer& w, const Resolver& resolver, const fc::microseconds& max_action_data_serialization_time ) try { + impl::stream_sink sink(w); + impl::abi_traverse_context ctx( create_depth_yield_function(), max_action_data_serialization_time ); + impl::abi_to_variant::add_value(sink, o, resolver, ctx); } FC_RETHROW_EXCEPTIONS(error, "Failed to serialize: {}", boost::core::demangle( typeid(o).name() )) template diff --git a/libraries/chain/include/sysio/chain/abi_sinks.hpp b/libraries/chain/include/sysio/chain/abi_sinks.hpp index 0bb89a771c..c3047b9b17 100644 --- a/libraries/chain/include/sysio/chain/abi_sinks.hpp +++ b/libraries/chain/include/sysio/chain/abi_sinks.hpp @@ -1,8 +1,15 @@ #pragma once -#include +// Self-contained sink definitions consumed by abi_serializer's binary_walk and +// abi_to_variant templates. Deliberately forward-declares `abi_serializer` and +// `abi_traverse_context` instead of including `abi_serializer.hpp`, since +// abi_serializer.hpp includes this header before defining the templates that +// use Sink. Method bodies in abi_serializer.cpp see the full definitions. + +#include #include +#include #include #include #include @@ -11,7 +18,13 @@ #include #include #include -#include + +namespace sysio::chain { + struct abi_serializer; + namespace impl { + struct abi_traverse_context; + } +} namespace sysio::chain::impl { @@ -37,7 +50,14 @@ namespace sysio::chain::impl { */ class variant_sink { public: - explicit variant_sink(const abi_serializer& abi) noexcept : abi_(abi) {} + /// Construct without an abi_serializer reference -- used by `abi_to_emit` + /// callers (to_variant template family) that don't need the binary unpack helpers. + variant_sink() noexcept = default; + + /// Construct bound to an abi_serializer -- used by `_binary_walk` callers + /// that need `unpack_built_in` and `unpack_protobuf` to dispatch built-in types + /// and protobuf decoding respectively. + explicit variant_sink(const abi_serializer& abi) noexcept : abi_(&abi) {} void begin_object(); void end_object(); @@ -65,6 +85,19 @@ class variant_sink { /// existing variant-side unpack functions produce as a single `fc::variant`. void value_variant(fc::variant v) { emit_value(std::move(v)); } + /// Generic field emit used by `abi_to_emit` for non-ABI types: builds a + /// `fc::variant(v)` and emits it at the current value position. Symmetric with + /// `stream_sink::emit` which calls `fc::to_json_stream(v, w)` instead. + template + void emit(const T& v) { emit_value(fc::variant(v)); } + + /// ABI-aware action data emit: decode the binary payload to an `fc::variant` + /// via the per-account abi and inject it at the current value position. This + /// mirrors `stream_sink::unpack_action_data` which calls `binary_to_json_stream` + /// directly into the writer. + void unpack_action_data(const abi_serializer& abi, std::string_view type, const bytes& data, + abi_traverse_context& ctx, bool short_path); + /// True if at least one item has been added to the current frame. Used by the walker /// to enforce the legacy "Unable to unpack '...' from stream" guard on empty structs. bool frame_has_items() const noexcept; @@ -78,7 +111,7 @@ class variant_sink { private: void emit_value(fc::variant v); - const abi_serializer& abi_; + const abi_serializer* abi_ = nullptr; enum class frame_kind : uint8_t { object, array }; struct frame { @@ -105,7 +138,15 @@ class variant_sink { */ class stream_sink { public: - stream_sink(const abi_serializer& abi, fc::json_writer& w) noexcept : abi_(abi), w_(w) {} + /// Construct without an abi_serializer reference -- used by `abi_to_emit` + /// callers (to_json_stream template family) that don't need the binary unpack + /// helpers. + explicit stream_sink(fc::json_writer& w) noexcept : w_(w) {} + + /// Construct bound to an abi_serializer -- used by `_binary_walk` callers + /// that need `unpack_built_in` and `unpack_protobuf` to dispatch built-in types + /// and protobuf decoding respectively. + stream_sink(const abi_serializer& abi, fc::json_writer& w) noexcept : abi_(&abi), w_(w) {} void begin_object(); void end_object(); @@ -133,6 +174,22 @@ class stream_sink { /// by the variant-fallback built-in path until commit 2 lands the direct unpacks. void value_variant(const fc::variant& v); + /// Generic field emit used by `abi_to_emit` for non-ABI types: dispatches + /// straight to `fc::to_json_stream(v, w)` so reflected user structs and primitives + /// emit their tokens without an intermediate variant build. + template + void emit(const T& v) { + fc::to_json_stream(v, w_); + on_value_emitted(); + } + + /// ABI-aware action data emit: decode the binary payload via + /// `abi.binary_to_json_stream` directly into the writer at the current value + /// position. Mirrors `variant_sink::unpack_action_data` which assembles a + /// `fc::variant` instead. + void unpack_action_data(const abi_serializer& abi, std::string_view type, const bytes& data, + abi_traverse_context& ctx, bool short_path); + bool frame_has_items() const noexcept; void unpack_built_in(std::string_view ftype, fc::datastream& stream, @@ -142,7 +199,7 @@ class stream_sink { private: void on_value_emitted() noexcept; - const abi_serializer& abi_; + const abi_serializer* abi_ = nullptr; fc::json_writer& w_; std::vector frame_items_; }; diff --git a/libraries/libfc/include/fc/io/enum_type.hpp b/libraries/libfc/include/fc/io/enum_type.hpp index 63e8a5b4b4..6921a0ccf9 100644 --- a/libraries/libfc/include/fc/io/enum_type.hpp +++ b/libraries/libfc/include/fc/io/enum_type.hpp @@ -1,5 +1,6 @@ #pragma once #include +#include #include #include @@ -58,6 +59,16 @@ namespace fc vo.value = var.as(); } + /// JSON shape mirrors to_variant: forward to the EnumType's own to_json_stream + /// so FC_REFLECT_ENUM-reflected enums emit as their member-name strings (not + /// the underlying integer). Bare numeric enums fall through to the primitive + /// overloads via integer promotion. + template + void to_json_stream( const enum_type& var, json_writer& w ) + { + fc::to_json_stream(static_cast(var.value), w); + } + /** serializes like an IntType */ namespace raw diff --git a/libraries/libfc/include/fc/reflect/json_stream.hpp b/libraries/libfc/include/fc/reflect/json_stream.hpp index d46771756b..870ddabe7c 100644 --- a/libraries/libfc/include/fc/reflect/json_stream.hpp +++ b/libraries/libfc/include/fc/reflect/json_stream.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include diff --git a/libraries/libfc/include/fc/static_variant.hpp b/libraries/libfc/include/fc/static_variant.hpp index 37f004ac3d..f811edb50c 100644 --- a/libraries/libfc/include/fc/static_variant.hpp +++ b/libraries/libfc/include/fc/static_variant.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -80,6 +81,15 @@ template void to_variant( const std::variant& s, fc::varian v = std::move(vars); } +/// JSON shape: 2-element array `[index, value]` matching the to_variant form. +template void to_json_stream( const std::variant& s, json_writer& w ) +{ + w.begin_array(); + w.value_uint64(static_cast(s.index())); + std::visit([&w](const auto& v) { fc::to_json_stream(v, w); }, s); + w.end_array(); +} + template void from_variant( const fc::variant& v, std::variant& s ) { auto ar = v.get_array(); diff --git a/plugins/chain_api_plugin/src/chain_api_plugin.cpp b/plugins/chain_api_plugin/src/chain_api_plugin.cpp index 3def3dd4ee..fb022b6ec9 100644 --- a/plugins/chain_api_plugin/src/chain_api_plugin.cpp +++ b/plugins/chain_api_plugin/src/chain_api_plugin.cpp @@ -150,7 +150,7 @@ void chain_api_plugin::plugin_startup() { // delivered to the http thread pool (closure vs variant tree). _http_plugin.add_api_stream({ CHAIN_RO_CALL_STREAM(get_activated_protocol_features, chain_apis::read_only::get_activated_protocol_features_results, 200, http_params_types::possible_no_params), - CHAIN_RO_CALL_STREAM_POST(get_block, fc::variant, 200, http_params_types::params_required), // _POST because get_block() returns a lambda to be executed on the http thread pool + CHAIN_RO_CALL_STREAM_POST_DIRECT(get_block, 200, http_params_types::params_required), // _POST because get_block_stream() returns a lambda to be executed on the http thread pool CHAIN_RO_CALL_STREAM(get_block_info, fc::variant, 200, http_params_types::params_required), CHAIN_RO_CALL_STREAM(get_block_header_state, fc::variant, 200, http_params_types::params_required), CHAIN_RO_CALL_STREAM(get_code, chain_apis::read_only::get_code_results, 200, http_params_types::params_required), diff --git a/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp b/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp index 6fee63a17b..5b857c3a2a 100644 --- a/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp +++ b/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp @@ -385,12 +385,30 @@ class read_only : public api_base { using get_block_params = get_raw_block_params; std::function()> get_block(const get_block_params& params, const fc::time_point& deadline) const; + /// Direct-streaming counterpart to `get_block`. Thread sequence: + /// - Phase 1 (main app thread, read_only queue): this method's body. `get_raw_block` is itself thread-safe; + /// `get_serializers_cache` is what *requires* the main thread because it reads each referenced account's `abi` + /// field out of chainbase. Returns the outer http_fwd closure. + /// - Hop 1: `CALL_WITH_400_STREAM_POST_DIRECT` posts the outer closure to the http thread pool. + /// - Phase 2 (http thread pool): invokes the outer closure -- captured block + resolver snapshots, no chainbase + /// access -- and produces the inner `json_writer`-emitting closure, handed to `cb`. + /// - Hop 2: `make_http_stream_response_handler` re-dispatches onto the same http thread pool (inherited from the + /// variant-cb `cb` shape, where Phase 2 does the row-decode work). + /// - Phase 3 (http thread pool): walks the captured block via `abi_serializer::to_json_stream`, + /// decoding each action's `data` straight into the writer with `binary_to_json_stream`. No `fc::variant` tree + /// is built per action. + using get_block_stream_emit_fn = std::function; + using get_block_stream_return_t = std::function()>; + get_block_stream_return_t get_block_stream(const get_block_params& params, const fc::time_point& deadline) const; + // call from app() thread abi_resolver get_block_serializers( const chain::signed_block_ptr& block, const fc::microseconds& max_time ) const; // call from any thread fc::variant convert_block( const chain::signed_block_ptr& block, abi_resolver& resolver ) const; + void convert_block_stream( const chain::signed_block_ptr& block, + abi_resolver& resolver, fc::json_writer& w ) const; struct get_block_header_params { string block_num_or_id; diff --git a/plugins/chain_plugin/src/chain_plugin.cpp b/plugins/chain_plugin/src/chain_plugin.cpp index ddde39bb4c..e8a083baee 100644 --- a/plugins/chain_plugin/src/chain_plugin.cpp +++ b/plugins/chain_plugin/src/chain_plugin.cpp @@ -2788,6 +2788,23 @@ std::function()> read_only::get_block(const g }; } +read_only::get_block_stream_return_t +read_only::get_block_stream(const get_block_params& params, const fc::time_point& deadline) const { + chain::signed_block_ptr block = get_raw_block(params, deadline); + + using return_type = chain::t_or_exception; + return [this, + resolver = get_serializers_cache(db, block, abi_serializer_max_time), + block = std::move(block)]() mutable -> return_type { + try { + return get_block_stream_emit_fn{[this, resolver = std::move(resolver), block = std::move(block)] + (fc::json_writer& w) mutable { + convert_block_stream(block, resolver, w); + }}; + } CATCH_AND_RETURN(return_type); + }; +} + read_only::get_block_header_result read_only::get_block_header(const read_only::get_block_header_params& params, const fc::time_point& deadline) const{ std::optional block_num; @@ -2844,6 +2861,24 @@ fc::variant read_only::convert_block( const chain::signed_block_ptr& block, abi_ ( "ref_block_prefix", ref_block_prefix ); } +void read_only::convert_block_stream( const chain::signed_block_ptr& block, abi_resolver& resolver, fc::json_writer& w ) const { + chain::impl::stream_sink sink(w); + chain::impl::abi_traverse_context ctx(abi_serializer::create_depth_yield_function(), abi_serializer_max_time); + + const auto block_id = block->calculate_id(); + const uint32_t ref_block_pfx = block_id._hash[1]; + + sink.begin_object(); + chain::impl::abi_to_variant::emit_signed_block_body(sink, *block, resolver, ctx); + sink.key("id"); + sink.template emit(block_id); + sink.key("block_num"); + sink.value_uint64(block->block_num()); + sink.key("ref_block_prefix"); + sink.value_uint64(ref_block_pfx); + sink.end_object(); +} + fc::variant read_only::get_block_info(const read_only::get_block_info_params& params, const fc::time_point&) const { std::optional block; diff --git a/tests/test_chain_plugin.cpp b/tests/test_chain_plugin.cpp index ef9a9ea38d..0e9e99c440 100644 --- a/tests/test_chain_plugin.cpp +++ b/tests/test_chain_plugin.cpp @@ -159,4 +159,43 @@ BOOST_FIXTURE_TEST_CASE(get_account_streaming_vs_variant_byte_identical, chain_p BOOST_CHECK_EQUAL(variant_path, stream_path); } FC_LOG_AND_RETHROW() } +// Byte-identical compat for the new get_block_stream direct path. Builds a +// populated block (account creation + a transfer; both decode through the +// resolver-located sysio.token / system ABI), then compares +// `to_string(convert_block(block))` against the streaming +// `convert_block_stream(block, json_writer)` output. Drives the +// abi_serializer::to_json_stream template -- any drift in the +// ABI-aware stream path's token order, quoting, or action data form would +// surface here. +BOOST_FIXTURE_TEST_CASE(get_block_streaming_vs_variant_byte_identical, chain_plugin_tester) { try { + produce_blocks(10); + setup_system_accounts(); + produce_blocks(); + create_account("alice1111111"_n, config::system_account_name); + transfer(name("sysio"), name("alice1111111"), core_from_string("100.0000"), name("sysio")); + produce_block(); + + auto signed_block = control->fetch_block_by_number(control->head().block_num()); + BOOST_REQUIRE(signed_block); + + std::optional _tracked_votes; + read_only ro(*control, {}, {}, _tracked_votes, + fc::microseconds::maximum(), fc::microseconds::maximum(), {}); + + auto resolver_v = ro.get_block_serializers(signed_block, fc::microseconds::maximum()); + auto resolver_s = ro.get_block_serializers(signed_block, fc::microseconds::maximum()); + + const fc::variant variant_block = ro.convert_block(signed_block, resolver_v); + const std::string variant_path = fc::json::to_string(variant_block, fc::time_point::maximum()); + + std::string stream_path; + { + fc::json_writer w(stream_path); + ro.convert_block_stream(signed_block, resolver_s, w); + BOOST_REQUIRE(w.balanced()); + } + + BOOST_CHECK_EQUAL(variant_path, stream_path); +} FC_LOG_AND_RETHROW() } + BOOST_AUTO_TEST_SUITE_END() diff --git a/unittests/abi_tests.cpp b/unittests/abi_tests.cpp index f54a94ec25..869321023b 100644 --- a/unittests/abi_tests.cpp +++ b/unittests/abi_tests.cpp @@ -53,6 +53,21 @@ static fc::time_point get_deadline() { static auto yield_fn() { return abi_serializer::create_yield_function( max_serialization_time ); } +// Verify the streaming `binary_to_json_stream` path emits byte-identical JSON to +// `to_string(binary_to_variant(...))` for the same packed bytes. Called from the +// round-trip helpers below so every type already exercised by abi_tests gets a +// parity check on the streaming path for free. +void verify_stream_matches_variant( const abi_serializer& abis, const type_name& type, const bytes& packed, const std::string& variant_json ) +{ + std::string stream_json; + { + fc::json_writer w(stream_json); + abis.binary_to_json_stream(type, packed, w, yield_fn()); + BOOST_REQUIRE( w.balanced() ); + } + BOOST_TEST( stream_json == variant_json ); +} + // verify that round trip conversion, via bytes, reproduces the exact same data fc::variant verify_byte_round_trip_conversion( const abi_serializer& abis, const type_name& type, const fc::variant& var ) { @@ -67,6 +82,8 @@ fc::variant verify_byte_round_trip_conversion( const abi_serializer& abis, const std::string r3 = fc::json::to_string(var3, get_deadline()); BOOST_TEST( r2 == r3 ); + verify_stream_matches_variant( abis, type, bytes, r2 ); + auto bytes2 = abis.variant_to_binary(type, var2, yield_fn()); auto bytes3 = abis.variant_to_binary(type, var3, max_serialization_time); BOOST_TEST( bytes2 == bytes3 ); @@ -87,6 +104,7 @@ void verify_round_trip_conversion( const abi_serializer& abis, const type_name& BOOST_REQUIRE_EQUAL(fc::json::to_string(var2, get_deadline()), expected_json); auto var3 = abis.binary_to_variant(type, b, max_serialization_time ); BOOST_REQUIRE_EQUAL(fc::json::to_string(var3, get_deadline()), expected_json); + verify_stream_matches_variant(abis, type, bytes, expected_json); auto bytes2 = abis.variant_to_binary(type, var2, yield_fn()); BOOST_REQUIRE_EQUAL(fc::to_hex(bytes2), hex); auto b2 = abis.variant_to_binary(type, var3, max_serialization_time); @@ -129,6 +147,8 @@ fc::variant verify_type_round_trip_conversion( const abi_serializer& abis, const std::string r3 = fc::json::to_string(var3, get_deadline()); BOOST_TEST( r2 == r3 ); + verify_stream_matches_variant( abis, type, bytes, r2 ); + auto bytes2 = abis.variant_to_binary(type, var2, yield_fn()); auto b3 = abis.variant_to_binary(type, var3, max_serialization_time); BOOST_TEST( bytes2 == b3 ); @@ -3397,17 +3417,19 @@ BOOST_AUTO_TEST_CASE(abi_to_variant__add_action__good_return_value) abi_serializer abis(abi_def(abidef), yield_fn()); { + fc::variant trace_var; + abi_serializer::to_variant(at, trace_var, get_resolver(abidef), yield_fn()); mutable_variant_object mvo; - sysio::chain::impl::abi_traverse_context ctx(yield_fn(), fc::microseconds{}); - sysio::chain::impl::abi_to_variant::add(mvo, "action_traces", at, get_resolver(abidef), ctx); + mvo("action_traces", std::move(trace_var)); std::string res = fc::json::to_string(mvo, get_deadline()); BOOST_CHECK_EQUAL(res, expected_json); } { + fc::variant trace_var; + abi_serializer::to_variant(at, trace_var, get_resolver(abidef), max_serialization_time); mutable_variant_object mvo; - sysio::chain::impl::abi_traverse_context ctx(abi_serializer::create_depth_yield_function(), max_serialization_time); - sysio::chain::impl::abi_to_variant::add(mvo, "action_traces", at, get_resolver(abidef), ctx); + mvo("action_traces", std::move(trace_var)); std::string res = fc::json::to_string(mvo, get_deadline()); BOOST_CHECK_EQUAL(res, expected_json); @@ -3432,17 +3454,19 @@ BOOST_AUTO_TEST_CASE(abi_to_variant__add_action__bad_return_value) abi_serializer abis(abi_def(abidef), yield_fn()); { + fc::variant trace_var; + abi_serializer::to_variant(at, trace_var, get_resolver(abidef), yield_fn()); mutable_variant_object mvo; - sysio::chain::impl::abi_traverse_context ctx(yield_fn(), fc::microseconds{}); - sysio::chain::impl::abi_to_variant::add(mvo, "action_traces", at, get_resolver(abidef), ctx); + mvo("action_traces", std::move(trace_var)); std::string res = fc::json::to_string(mvo, get_deadline()); BOOST_CHECK_EQUAL(res, expected_json); } { + fc::variant trace_var; + abi_serializer::to_variant(at, trace_var, get_resolver(abidef), max_serialization_time); mutable_variant_object mvo; - sysio::chain::impl::abi_traverse_context ctx(abi_serializer::create_depth_yield_function(), max_serialization_time); - sysio::chain::impl::abi_to_variant::add(mvo, "action_traces", at, get_resolver(abidef), ctx); + mvo("action_traces", std::move(trace_var)); std::string res = fc::json::to_string(mvo, get_deadline()); BOOST_CHECK_EQUAL(res, expected_json); @@ -3477,17 +3501,19 @@ BOOST_AUTO_TEST_CASE(abi_to_variant__add_action__no_return_value) abi_serializer abis(abi_def(abidef), yield_fn()); { + fc::variant trace_var; + abi_serializer::to_variant(at, trace_var, get_resolver(abidef), yield_fn()); mutable_variant_object mvo; - sysio::chain::impl::abi_traverse_context ctx(yield_fn(), fc::microseconds{}); - sysio::chain::impl::abi_to_variant::add(mvo, "action_traces", at, get_resolver(abidef), ctx); + mvo("action_traces", std::move(trace_var)); std::string res = fc::json::to_string(mvo, get_deadline()); BOOST_CHECK_EQUAL(res, expected_json); } { + fc::variant trace_var; + abi_serializer::to_variant(at, trace_var, get_resolver(abidef), max_serialization_time); mutable_variant_object mvo; - sysio::chain::impl::abi_traverse_context ctx(abi_serializer::create_depth_yield_function(), max_serialization_time); - sysio::chain::impl::abi_to_variant::add(mvo, "action_traces", at, get_resolver(abidef), ctx); + mvo("action_traces", std::move(trace_var)); std::string res = fc::json::to_string(mvo, get_deadline()); BOOST_CHECK_EQUAL(res, expected_json); @@ -4177,29 +4203,4 @@ BOOST_AUTO_TEST_CASE(abi_serializer_long_table_name) { try { } FC_LOG_AND_RETHROW() } -// Parity gate for `binary_to_json_stream`: the streaming path must produce -// byte-identical JSON output to `fc::json::to_string(binary_to_variant(...))`. -// Reuses `my_abi` + `my_other_json` from `general` so coverage stays in lock-step -// with the established round-trip suite. -BOOST_AUTO_TEST_CASE(binary_to_json_stream_parity) -{ try { - auto abi = sysio_contract_abi(fc::json::from_string(my_abi).as()); - abi_serializer abis(abi_def(abi), yield_fn()); - - auto fixture_var = fc::json::from_string(my_other_json); - auto packed = abis.variant_to_binary("A", fixture_var, yield_fn()); - - auto via_variant = abis.binary_to_variant("A", packed, yield_fn()); - std::string variant_json = fc::json::to_string(via_variant, get_deadline()); - - std::string streamed_json; - { - fc::json_writer w(streamed_json); - abis.binary_to_json_stream("A", packed, w, yield_fn()); - BOOST_CHECK( w.balanced() ); - } - - BOOST_CHECK_EQUAL( variant_json, streamed_json ); -} FC_LOG_AND_RETHROW() } - BOOST_AUTO_TEST_SUITE_END() From 0e45e988f9c012407f41a7fbd3097eb24a4e1a71 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Tue, 28 Apr 2026 07:10:54 -0500 Subject: [PATCH 30/71] chain: make next_function a copyable wrapper over std::move_only_function next_function is now a small class holding shared_ptr>. This lets async API callers store mutable lambdas with consume-on-call captures (e.g. [cb = std::move(cb)] mutable {...}) inside the callback, which std::function silently breaks. The user-visible API matches the old alias: default-constructible, null-checkable via operator bool, throws std::bad_function_call when invoked empty. A pure std::move_only_function alias wasn't viable. The callback travels through boost::signals2 (used by appbase::method_decl for incoming::methods::transaction_async), through boost::multi_index value storage in unapplied_transaction_queue, trx_retry_db's tracked_transaction, and snapshot_scheduler's pending_snapshot_index, and through many [=] or by-value lambda captures plus CATCH_AND_CALL fallbacks held alongside async captures in the same scope. All of those require a copyable outer type. The shared_ptr indirection keeps the wrapper copyable while leaving the inner move_only_function move-only so it accepts non-copyable callables. Tradeoff: every construction now allocates (one make_shared) where small std::function objects previously fit the SBO and stayed inline. In exchange, copies are an atomic refcount bump rather than a full re-allocation when the callable was heap-stored, so multi-copy paths (push_recurse, snapshot_scheduler fanout, the read_only_trx executor chain) get cheaper. Invocation gains one indirection. Semantic shift worth flagging: copies of the wrapper share state, where copies of std::function were independent. No code in tree was found relying on the old independence, but it is the sort of difference a future reader may want to verify against new call sites. --- libraries/chain/include/sysio/chain/types.hpp | 38 +++++++++++++++++-- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/libraries/chain/include/sysio/chain/types.hpp b/libraries/chain/include/sysio/chain/types.hpp index 61071c0240..1276b663e3 100644 --- a/libraries/chain/include/sysio/chain/types.hpp +++ b/libraries/chain/include/sysio/chain/types.hpp @@ -22,6 +22,7 @@ #include #include #include +#include #define OBJECT_CTOR1(NAME) \ public: \ @@ -506,13 +507,21 @@ namespace sysio::chain { template struct overloaded : Ts... { using Ts::operator()...; }; template overloaded(Ts...) -> overloaded; - // next_function is a function passed to an API (like send_transaction) and which is called at the end of - // the API processing on the main thread. The type T is a description of the API result that can be + // next_function is a function passed to an API (like send_transaction) and which is called at the end + // of the API processing on the main thread. The type T is a description of the API result that can be // serialized as output. // The function accepts a variant which can contain an exception_ptr (if an exception occured while // processing the API) or the result T. // The third option is a function which can be executed in a multithreaded context (likely on the // http_plugin thread pool) and which completes the API processing and returns the result T. + // + // The user-provided callable is held inside a std::move_only_function so consume-on-call captures + // (e.g. [cb = std::move(cb)] mutable {...}) work correctly. The outer wrapper is copyable via a + // shared_ptr indirection - required because the callback travels through paths that copy + // (boost::signals2 slot dispatch, boost::multi_index value storage, lambdas captured by [=], + // CATCH_AND_CALL fallbacks held alongside an async capture in the same scope). All copies share + // the same underlying callable; if the underlying lambda consumes its captures, calling through + // any copy after the first invocation reaches an exhausted state. // ------------------------------------------------------------------------------------------------------- template using t_or_exception = std::variant; @@ -521,7 +530,30 @@ namespace sysio::chain { using next_function_variant = std::variant()>>; template - using next_function = std::function&)>; + class next_function { + public: + using element_type = std::move_only_function&)>; + + next_function() = default; + next_function(std::nullptr_t) noexcept {} + + template + requires (!std::is_same_v, next_function>) + next_function(F&& f) + : _f(std::make_shared(std::forward(f))) {} + + void operator()(const next_function_variant& v) const { + if (!_f || !*_f) [[unlikely]] { + throw std::bad_function_call(); + } + (*_f)(v); + } + + explicit operator bool() const noexcept { return _f && static_cast(*_f); } + + private: + std::shared_ptr _f; + }; // to configure whether a process should be done asynchronously or not enum class async_t { no, yes }; From b2bf4fa37233e2587678a5e3c9a25f563f077937 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Tue, 28 Apr 2026 16:28:19 -0500 Subject: [PATCH 31/71] chain_api_plugin: migrate remaining variant-cb endpoints to streaming Every endpoint registered by chain_api_plugin is now on the streaming-cb path: - get_info, get_accounts_by_authorizers, get_raw_block, get_block_header, get_transaction_status: flipped from the sync variant-cb macros to CHAIN_RO_CALL_STREAM. Each emits its result struct via fc::to_json_stream through the reflector, no fc::variant tree built. - compute_transaction, push_transaction, push_transactions, send_transaction, send_transaction2: flipped to CHAIN_R*_CALL_ASYNC_STREAM. Possible now that next_function is a shared_ptr wrapper -- the streaming cb (move-only) survives the lambda capture into the next_function callback. - send_read_only_transaction: flipped to CHAIN_RO_CALL_ASYNC_STREAM. Threading is unchanged for every endpoint: (add_async_api, sync) pairs flip to (add_async_api_stream, sync stream) with the handler running on the http thread; (add_api(read_only), async) pairs flip to (add_api_stream(read_only), async stream) with the handler still posted to the read_only exec_queue and the api method still posting its inner work to read_write / read_exclusive as before. The local CALL_WITH_400 macro definition is gone from chain_api_plugin.cpp; the file now exposes only the streaming-cb wrappers (CHAIN_RO_CALL_STREAM / CHAIN_RO_CALL_STREAM_POST / CHAIN_RO_CALL_STREAM_POST_DIRECT / CHAIN_RO_CALL_ASYNC_STREAM / CHAIN_RW_CALL_ASYNC_STREAM). Supporting fc-level additions: - fc::to_json_stream(const std::shared_ptr&, w) -- dereference and emit; null emits JSON null. Mirrors the existing fc::to_variant(shared_ptr) used by every signed_block_ptr / packed_transaction_ptr emit site. - fc::to_json_stream(const sysio::chain::chain_id_type&, w) -- forwards to the existing sha256 stream emitter (string-form), matching to_variant. - fc/reflect/json_stream.hpp now includes fc/reflect/variant.hpp so to_json_stream_via_variant's `fc::to_variant(v, tmp)` sees the reflector-based generic template at instantiation. Without this, types like transaction_header would silently bind to to_variant(const fc::unsigned_int&) via unsigned_int's any-T converting ctor. --- libraries/chain/chain_id_type.cpp | 7 ++ .../include/sysio/chain/chain_id_type.hpp | 4 ++ .../libfc/include/fc/reflect/json_stream.hpp | 1 + libraries/libfc/include/fc/variant.hpp | 9 +++ .../chain_api_plugin/src/chain_api_plugin.cpp | 67 ++++++------------- 5 files changed, 43 insertions(+), 45 deletions(-) diff --git a/libraries/chain/chain_id_type.cpp b/libraries/chain/chain_id_type.cpp index 0ae1ce6708..90651597a4 100644 --- a/libraries/chain/chain_id_type.cpp +++ b/libraries/chain/chain_id_type.cpp @@ -1,6 +1,9 @@ #include #include +#include +#include + namespace sysio { namespace chain { void chain_id_type::reflector_init()const { @@ -19,4 +22,8 @@ namespace fc { from_variant( v, static_cast(cid) ); } + void to_json_stream(const sysio::chain::chain_id_type& cid, fc::json_writer& w) { + to_json_stream( static_cast(cid), w ); + } + } // fc diff --git a/libraries/chain/include/sysio/chain/chain_id_type.hpp b/libraries/chain/include/sysio/chain/chain_id_type.hpp index 062311001f..8f7e995d21 100644 --- a/libraries/chain/include/sysio/chain/chain_id_type.hpp +++ b/libraries/chain/include/sysio/chain/chain_id_type.hpp @@ -59,6 +59,10 @@ namespace chain { namespace fc { class variant; + class json_writer; void to_variant(const sysio::chain::chain_id_type& cid, fc::variant& v); void from_variant(const fc::variant& v, sysio::chain::chain_id_type& cid); + /// JSON shape: lowercase-hex string -- matches the to_variant emit (which writes + /// the underlying sha256 as a string). + void to_json_stream(const sysio::chain::chain_id_type& cid, fc::json_writer& w); } // fc diff --git a/libraries/libfc/include/fc/reflect/json_stream.hpp b/libraries/libfc/include/fc/reflect/json_stream.hpp index 870ddabe7c..300afab68d 100644 --- a/libraries/libfc/include/fc/reflect/json_stream.hpp +++ b/libraries/libfc/include/fc/reflect/json_stream.hpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include diff --git a/libraries/libfc/include/fc/variant.hpp b/libraries/libfc/include/fc/variant.hpp index b4d020f0c8..dddc5c54d4 100644 --- a/libraries/libfc/include/fc/variant.hpp +++ b/libraries/libfc/include/fc/variant.hpp @@ -833,6 +833,15 @@ namespace fc else vo = nullptr; } + /// JSON shape: dereference and emit the pointee, matching to_variant; null + /// pointers emit JSON `null`. + template + void to_json_stream( const std::shared_ptr& var, json_writer& w ) + { + if( var ) to_json_stream( *var, w ); + else w.value_null(); + } + template void from_variant( const fc::variant& var, std::shared_ptr& vo ) { diff --git a/plugins/chain_api_plugin/src/chain_api_plugin.cpp b/plugins/chain_api_plugin/src/chain_api_plugin.cpp index fb022b6ec9..da0f56b278 100644 --- a/plugins/chain_api_plugin/src/chain_api_plugin.cpp +++ b/plugins/chain_api_plugin/src/chain_api_plugin.cpp @@ -89,35 +89,15 @@ parse_params(body);\ - fc::variant result( api_handle.call_name( std::move(params), deadline ) ); \ - cb(http_response_code, std::move(result)); \ - } catch (...) { \ - http_plugin::handle_exception(#api_name, #call_name, body, cb); \ - } \ - }} - -#define CHAIN_RO_CALL(call_name, http_response_code, params_type) CALL_WITH_400(chain, chain_ro, ro_api, chain_apis::read_only, call_name, http_response_code, params_type) -#define CHAIN_RW_CALL(call_name, http_response_code, params_type) CALL_WITH_400(chain, chain_rw, rw_api, chain_apis::read_write, call_name, http_response_code, params_type) -#define CHAIN_RO_CALL_POST(call_name, call_result, http_response_code, params_type) CALL_WITH_400_POST(chain, chain_ro, ro_api, chain_apis::read_only, call_name, call_result, http_response_code, params_type) -#define CHAIN_RO_CALL_ASYNC(call_name, call_result, http_response_code, params_type) CALL_ASYNC_WITH_400(chain, chain_ro, ro_api, chain_apis::read_only, call_name, call_result, http_response_code, params_type) -#define CHAIN_RW_CALL_ASYNC(call_name, call_result, http_response_code, params_type) CALL_ASYNC_WITH_400(chain, chain_rw, rw_api, chain_apis::read_write, call_name, call_result, http_response_code, params_type) - -#define CHAIN_RO_CALL_WITH_400(call_name, http_response_code, params_type) CALL_WITH_400(chain, chain_ro, ro_api, chain_apis::read_only, call_name, http_response_code, params_type) - -// Streaming-cb counterparts. Migration in progress; endpoints flip from the variant -// macros above to these as their byte-identical equivalence is verified. After the -// last endpoint flips, the variant macros (and CALL_WITH_400 / CALL_WITH_400_POST / -// CALL_ASYNC_WITH_400) get removed in a cleanup commit. +// Streaming-cb endpoint registrations. Every chain_api_plugin endpoint is on the +// streaming path; the variant-cb counterparts (CALL_WITH_400 / CALL_ASYNC_WITH_400 / +// CHAIN_RO_CALL / CHAIN_RW_CALL / CHAIN_RO_CALL_ASYNC / CHAIN_RW_CALL_ASYNC / +// CHAIN_RO_CALL_WITH_400) are no longer used from this plugin. #define CHAIN_RO_CALL_STREAM(call_name, call_result, http_response_code, params_type) CALL_WITH_400_STREAM(chain, chain_ro, ro_api, chain_apis::read_only, call_name, call_result, http_response_code, params_type) #define CHAIN_RO_CALL_STREAM_POST(call_name, call_result, http_response_code, params_type) CALL_WITH_400_STREAM_POST(chain, chain_ro, ro_api, chain_apis::read_only, call_name, call_result, http_response_code, params_type) #define CHAIN_RO_CALL_STREAM_POST_DIRECT(call_name, http_response_code, params_type) CALL_WITH_400_STREAM_POST_DIRECT(chain, chain_ro, ro_api, chain_apis::read_only, call_name, http_response_code, params_type) +#define CHAIN_RO_CALL_ASYNC_STREAM(call_name, call_result, http_response_code, params_type) CALL_ASYNC_WITH_400_STREAM(chain, chain_ro, ro_api, chain_apis::read_only, call_name, call_result, http_response_code, params_type) +#define CHAIN_RW_CALL_ASYNC_STREAM(call_name, call_result, http_response_code, params_type) CALL_ASYNC_WITH_400_STREAM(chain, chain_rw, rw_api, chain_apis::read_write, call_name, call_result, http_response_code, params_type) void chain_api_plugin::plugin_startup() { dlog( "starting chain_api_plugin" ); @@ -132,22 +112,19 @@ void chain_api_plugin::plugin_startup() { ro_api.set_shorten_abi_errors( !http_plugin::verbose_errors() ); // Run get_info on http thread only - _http_plugin.add_async_api({ - CALL_WITH_400(chain, node, ro_api, chain_apis::read_only, get_info, 200, http_params_types::no_params) + _http_plugin.add_async_api_stream({ + CHAIN_RO_CALL_STREAM(get_info, chain_apis::get_info_db::get_info_results, 200, http_params_types::no_params) }); - _http_plugin.add_api({ + _http_plugin.add_api_stream({ // transaction related APIs will be posted to read_write queue after keys are recovered, they are safe to run in parallel until they post to the read_write queue - CHAIN_RO_CALL_ASYNC(compute_transaction, chain_apis::read_only::compute_transaction_results, 200, http_params_types::params_required), - CHAIN_RW_CALL_ASYNC(push_transaction, chain_apis::read_write::push_transaction_results, 202, http_params_types::params_required), - CHAIN_RW_CALL_ASYNC(push_transactions, chain_apis::read_write::push_transactions_results, 202, http_params_types::params_required), - CHAIN_RW_CALL_ASYNC(send_transaction, chain_apis::read_write::send_transaction_results, 202, http_params_types::params_required), - CHAIN_RW_CALL_ASYNC(send_transaction2, chain_apis::read_write::send_transaction_results, 202, http_params_types::params_required) + CHAIN_RO_CALL_ASYNC_STREAM(compute_transaction, chain_apis::read_only::compute_transaction_results, 200, http_params_types::params_required), + CHAIN_RW_CALL_ASYNC_STREAM(push_transaction, chain_apis::read_write::push_transaction_results, 202, http_params_types::params_required), + CHAIN_RW_CALL_ASYNC_STREAM(push_transactions, chain_apis::read_write::push_transactions_results, 202, http_params_types::params_required), + CHAIN_RW_CALL_ASYNC_STREAM(send_transaction, chain_apis::read_write::send_transaction_results, 202, http_params_types::params_required), + CHAIN_RW_CALL_ASYNC_STREAM(send_transaction2, chain_apis::read_write::send_transaction_results, 202, http_params_types::params_required) }, appbase::exec_queue::read_only); - // Streaming-cb endpoints. Same exec_queue and registration semantics as the - // variant-cb add_api block above; the difference is only how the response is - // delivered to the http thread pool (closure vs variant tree). _http_plugin.add_api_stream({ CHAIN_RO_CALL_STREAM(get_activated_protocol_features, chain_apis::read_only::get_activated_protocol_features_results, 200, http_params_types::possible_no_params), CHAIN_RO_CALL_STREAM_POST_DIRECT(get_block, 200, http_params_types::params_required), // _POST because get_block_stream() returns a lambda to be executed on the http thread pool @@ -172,21 +149,21 @@ void chain_api_plugin::plugin_startup() { }, appbase::exec_queue::read_only); if (chain.account_queries_enabled()) { - _http_plugin.add_async_api({ - CHAIN_RO_CALL_WITH_400(get_accounts_by_authorizers, 200, http_params_types::params_required), + _http_plugin.add_async_api_stream({ + CHAIN_RO_CALL_STREAM(get_accounts_by_authorizers, chain_apis::read_only::get_accounts_by_authorizers_result, 200, http_params_types::params_required), }); } - _http_plugin.add_async_api({ + _http_plugin.add_async_api_stream({ // chain_plugin send_read_only_transaction will post to read_exclusive queue - CHAIN_RO_CALL_ASYNC(send_read_only_transaction, chain_apis::read_only::send_read_only_transaction_results, 200, http_params_types::params_required), - CHAIN_RO_CALL_WITH_400(get_raw_block, 200, http_params_types::params_required), - CHAIN_RO_CALL_WITH_400(get_block_header, 200, http_params_types::params_required) + CHAIN_RO_CALL_ASYNC_STREAM(send_read_only_transaction, chain_apis::read_only::send_read_only_transaction_results, 200, http_params_types::params_required), + CHAIN_RO_CALL_STREAM(get_raw_block, chain::signed_block_ptr, 200, http_params_types::params_required), + CHAIN_RO_CALL_STREAM(get_block_header, chain_apis::read_only::get_block_header_result, 200, http_params_types::params_required) }); if (chain.transaction_finality_status_enabled()) { - _http_plugin.add_api({ - CHAIN_RO_CALL_WITH_400(get_transaction_status, 200, http_params_types::params_required), + _http_plugin.add_api_stream({ + CHAIN_RO_CALL_STREAM(get_transaction_status, chain_apis::read_only::get_transaction_status_results, 200, http_params_types::params_required), }, appbase::exec_queue::read_only); } From f2d238d57e3b3133c9ac2abf118337ccaa8315d0 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Tue, 28 Apr 2026 16:44:00 -0500 Subject: [PATCH 32/71] producer_api_plugin: migrate variant-cb endpoints to streaming All 21 producer_api endpoints flip to the streaming-cb path: - 8 read_only endpoints (paused, get_runtime_options, get_greylist, get_whitelist_blacklist, get_scheduled_protocol_feature_activations, get_supported_protocol_features, get_unapplied_transactions, get_snapshot_requests). - 12 read_write endpoints (pause, pause_at_block, resume, update_runtime_options, add_greylist_accounts, remove_greylist_accounts, set_whitelist_blacklist, schedule_snapshot, unschedule_snapshot, get_integrity_hash, schedule_protocol_feature_activations). - 1 async snapshot endpoint (create_snapshot). The local CALL_WITH_400 / CALL_ASYNC macros are replaced by their streaming counterparts (CALL_WITH_400_STREAM, CALL_ASYNC_STREAM); add_api flips to add_api_stream. Threading is unchanged: the read_only endpoints stay on read_only exec_queue, the read_write set stays on read_write, and create_snapshot still posts via next_function (now copyable wrapper around move_only_function). INVOKE_R_V_ASYNC now passes std::move(next) to the api method since the streaming callback's local next lambda captures the move-only stream cb and so cannot be copied. Supporting libfc additions to plug a few overloads the streaming reflector path needed: - fc::to_json_stream(boost::container::flat_map, w) and (flat_multimap, w) -- emit as array of [key, value] pairs, matching to_variant. Required by get_unapplied_transactions's account_billing field. - Helper detail::to_json_stream_from_map collapses the per-map shape into one body. --- .../include/fc/container/container_detail.hpp | 11 +++ libraries/libfc/include/fc/container/flat.hpp | 10 +++ .../src/producer_api_plugin.cpp | 67 ++++++++++--------- 3 files changed, 56 insertions(+), 32 deletions(-) diff --git a/libraries/libfc/include/fc/container/container_detail.hpp b/libraries/libfc/include/fc/container/container_detail.hpp index 4798851367..32bd5fa18b 100644 --- a/libraries/libfc/include/fc/container/container_detail.hpp +++ b/libraries/libfc/include/fc/container/container_detail.hpp @@ -130,6 +130,17 @@ namespace fc { vo = std::move( vars ); } + /// JSON shape mirrors to_variant_from_map: an array of [key, value] pairs. + template class Map, typename K, typename V, typename... U > + void to_json_stream_from_map( const Map< K, V, U... >& m, fc::json_writer& w ) { + FC_ASSERT( m.size() <= MAX_NUM_ARRAY_ELEMENTS ); + w.begin_array(); + for( const auto& item : m ) { + fc::to_json_stream( item, w ); + } + w.end_array(); + } + template class Map, typename K, typename V, typename... U> void from_variant_to_map( const variant& v, Map& m ) { const variants& vars = v.get_array(); diff --git a/libraries/libfc/include/fc/container/flat.hpp b/libraries/libfc/include/fc/container/flat.hpp index 09d3eb569d..d2fec70cc5 100644 --- a/libraries/libfc/include/fc/container/flat.hpp +++ b/libraries/libfc/include/fc/container/flat.hpp @@ -181,6 +181,11 @@ namespace fc { detail::to_variant_from_map( m, vo ); } + template + void to_json_stream( const flat_map< K, V, U... >& m, fc::json_writer& w ) { + detail::to_json_stream_from_map( m, w ); + } + template void from_variant( const variant& v, flat_map& m ) { detail::from_variant_to_flat_map( v, m ); @@ -191,6 +196,11 @@ namespace fc { detail::to_variant_from_map( m, vo ); } + template + void to_json_stream( const flat_multimap< K, V, U... >& m, fc::json_writer& w ) { + detail::to_json_stream_from_map( m, w ); + } + template void from_variant( const variant& v, flat_multimap& m ) { detail::from_variant_to_flat_map( v, m ); diff --git a/plugins/producer_api_plugin/src/producer_api_plugin.cpp b/plugins/producer_api_plugin/src/producer_api_plugin.cpp index 7887335ad5..a15cb5d45b 100644 --- a/plugins/producer_api_plugin/src/producer_api_plugin.cpp +++ b/plugins/producer_api_plugin/src/producer_api_plugin.cpp @@ -18,32 +18,35 @@ namespace sysio { using namespace sysio; -#define CALL_WITH_400(api_name, category, api_handle, call_name, INVOKE, http_response_code) \ +// Streaming-cb counterparts of the local CALL_WITH_400 / CALL_ASYNC. The result +// struct is captured by move into a json_writer-emitting closure and handed to +// the streaming cb; no fc::variant is built. +#define CALL_WITH_400_STREAM(api_name, category, api_handle, call_name, INVOKE, http_response_code) \ {std::string("/v1/" #api_name "/" #call_name), \ api_category::category, \ - [&producer](string&&, string&& body, url_response_callback&& cb) mutable { \ + [&producer](string&&, string&& body, url_response_stream_callback&& cb) mutable { \ try { \ INVOKE \ - cb(http_response_code, fc::variant(result)); \ + cb(http_response_code, [r = std::move(result)](fc::json_writer& w) mutable { fc::to_json_stream(r, w); }); \ } catch (...) { \ - http_plugin::handle_exception(#api_name, #call_name, body, cb); \ + http_plugin::handle_exception_stream(#api_name, #call_name, body, cb); \ } \ }} -#define CALL_ASYNC(api_name, category, api_handle, call_name, call_result, INVOKE, http_response_code) \ +#define CALL_ASYNC_STREAM(api_name, category, api_handle, call_name, call_result, INVOKE, http_response_code) \ {std::string("/v1/" #api_name "/" #call_name), \ api_category::category, \ - [&api_handle](string&&, string&& body, url_response_callback&& cb) mutable { \ + [&api_handle](string&&, string&& body, url_response_stream_callback&& cb) mutable { \ if (body.empty()) body = "{}"; \ - auto next = [cb=std::move(cb), body=std::move(body)](const chain::next_function_variant& result){ \ + auto next = [cb=std::move(cb), body=std::move(body)](const chain::next_function_variant& result) mutable { \ if (std::holds_alternative(result)) {\ try {\ throw *std::get(result);\ } catch (...) {\ - http_plugin::handle_exception(#api_name, #call_name, body, cb);\ + http_plugin::handle_exception_stream(#api_name, #call_name, body, cb);\ }\ } else if (std::holds_alternative(result)) { \ - cb(http_response_code, fc::variant(std::get(result)));\ + cb(http_response_code, [r = std::get(result)](fc::json_writer& w) mutable { fc::to_json_stream(r, w); });\ } else { \ assert(0); \ } \ @@ -73,7 +76,7 @@ using namespace sysio; auto result = api_handle.call_name(); #define INVOKE_R_V_ASYNC(api_handle, call_name)\ - api_handle.call_name(next); + api_handle.call_name(std::move(next)); #define INVOKE_V_R(api_handle, call_name, in_param) \ auto params = parse_params(body);\ @@ -91,51 +94,51 @@ void producer_api_plugin::plugin_startup() { // lifetime of plugin is lifetime of application auto& producer = app().get_plugin(); - app().get_plugin().add_api({ - CALL_WITH_400(producer, producer_ro, producer, paused, + app().get_plugin().add_api_stream({ + CALL_WITH_400_STREAM(producer, producer_ro, producer, paused, INVOKE_R_V(producer, paused), 201), - CALL_WITH_400(producer, producer_ro, producer, get_runtime_options, + CALL_WITH_400_STREAM(producer, producer_ro, producer, get_runtime_options, INVOKE_R_V(producer, get_runtime_options), 201), - CALL_WITH_400(producer, producer_ro, producer, get_greylist, + CALL_WITH_400_STREAM(producer, producer_ro, producer, get_greylist, INVOKE_R_V(producer, get_greylist), 201), - CALL_WITH_400(producer, producer_ro, producer, get_whitelist_blacklist, + CALL_WITH_400_STREAM(producer, producer_ro, producer, get_whitelist_blacklist, INVOKE_R_V(producer, get_whitelist_blacklist), 201), - CALL_WITH_400(producer, producer_ro, producer, get_scheduled_protocol_feature_activations, + CALL_WITH_400_STREAM(producer, producer_ro, producer, get_scheduled_protocol_feature_activations, INVOKE_R_V(producer, get_scheduled_protocol_feature_activations), 201), - CALL_WITH_400(producer, producer_ro, producer, get_supported_protocol_features, + CALL_WITH_400_STREAM(producer, producer_ro, producer, get_supported_protocol_features, INVOKE_R_R_II(producer, get_supported_protocol_features, producer_plugin::get_supported_protocol_features_params), 201), - CALL_WITH_400(producer, producer_ro, producer, get_unapplied_transactions, + CALL_WITH_400_STREAM(producer, producer_ro, producer, get_unapplied_transactions, INVOKE_R_R_D(producer, get_unapplied_transactions, producer_plugin::get_unapplied_transactions_params), 200), - CALL_WITH_400(producer, producer_ro, producer, get_snapshot_requests, + CALL_WITH_400_STREAM(producer, producer_ro, producer, get_snapshot_requests, INVOKE_R_V(producer, get_snapshot_requests), 201), }, appbase::exec_queue::read_only, appbase::priority::medium_high); // Not safe to run in parallel - app().get_plugin().add_api({ - CALL_WITH_400(producer, producer_rw, producer, pause, + app().get_plugin().add_api_stream({ + CALL_WITH_400_STREAM(producer, producer_rw, producer, pause, INVOKE_V_V(producer, pause), 201), - CALL_WITH_400(producer, producer_rw, producer, pause_at_block, + CALL_WITH_400_STREAM(producer, producer_rw, producer, pause_at_block, INVOKE_V_R(producer, pause_at_block, producer_plugin::pause_at_block_params), 201), - CALL_WITH_400(producer, producer_rw, producer, resume, + CALL_WITH_400_STREAM(producer, producer_rw, producer, resume, INVOKE_V_V(producer, resume), 201), - CALL_WITH_400(producer, producer_rw, producer, update_runtime_options, + CALL_WITH_400_STREAM(producer, producer_rw, producer, update_runtime_options, INVOKE_V_R(producer, update_runtime_options, producer_plugin::runtime_options), 201), - CALL_WITH_400(producer, producer_rw, producer, add_greylist_accounts, + CALL_WITH_400_STREAM(producer, producer_rw, producer, add_greylist_accounts, INVOKE_V_R(producer, add_greylist_accounts, producer_plugin::greylist_params), 201), - CALL_WITH_400(producer, producer_rw, producer, remove_greylist_accounts, + CALL_WITH_400_STREAM(producer, producer_rw, producer, remove_greylist_accounts, INVOKE_V_R(producer, remove_greylist_accounts, producer_plugin::greylist_params), 201), - CALL_WITH_400(producer, producer_rw, producer, set_whitelist_blacklist, + CALL_WITH_400_STREAM(producer, producer_rw, producer, set_whitelist_blacklist, INVOKE_V_R(producer, set_whitelist_blacklist, producer_plugin::whitelist_blacklist), 201), - CALL_ASYNC(producer, snapshot, producer, create_snapshot, chain::snapshot_scheduler::snapshot_information, + CALL_ASYNC_STREAM(producer, snapshot, producer, create_snapshot, chain::snapshot_scheduler::snapshot_information, INVOKE_R_V_ASYNC(producer, create_snapshot), 201), - CALL_WITH_400(producer, snapshot, producer, schedule_snapshot, + CALL_WITH_400_STREAM(producer, snapshot, producer, schedule_snapshot, INVOKE_R_R_II(producer, schedule_snapshot, chain::snapshot_scheduler::snapshot_request_params), 201), - CALL_WITH_400(producer, snapshot, producer, unschedule_snapshot, + CALL_WITH_400_STREAM(producer, snapshot, producer, unschedule_snapshot, INVOKE_R_R(producer, unschedule_snapshot, chain::snapshot_scheduler::snapshot_request_id_information), 201), - CALL_WITH_400(producer, producer_rw, producer, get_integrity_hash, + CALL_WITH_400_STREAM(producer, producer_rw, producer, get_integrity_hash, INVOKE_R_V(producer, get_integrity_hash), 201), - CALL_WITH_400(producer, producer_rw, producer, schedule_protocol_feature_activations, + CALL_WITH_400_STREAM(producer, producer_rw, producer, schedule_protocol_feature_activations, INVOKE_V_R(producer, schedule_protocol_feature_activations, producer_plugin::scheduled_protocol_feature_activations), 201), }, appbase::exec_queue::read_write, appbase::priority::medium_high); } From dd19ffcc11916f6cd1e1b1758f093eb889e1d8e1 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Tue, 28 Apr 2026 16:47:28 -0500 Subject: [PATCH 33/71] net_api_plugin: migrate variant-cb endpoints to streaming All 5 net_api endpoints (connect, disconnect, status, connections, bp_gossip_peers) flip to the streaming-cb path through CALL_WITH_400_STREAM and add_async_api_stream. INVOKE_R_R now produces a typed result (was fc::variant result(...)) so the streaming macro emits via fc::to_json_stream rather than wrapping in a variant first. Threading is unchanged: handler runs on the http thread directly (add_async_api_stream). --- plugins/net_api_plugin/src/net_api_plugin.cpp | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/plugins/net_api_plugin/src/net_api_plugin.cpp b/plugins/net_api_plugin/src/net_api_plugin.cpp index 4599ccf5f3..73fc5799d8 100644 --- a/plugins/net_api_plugin/src/net_api_plugin.cpp +++ b/plugins/net_api_plugin/src/net_api_plugin.cpp @@ -17,21 +17,21 @@ namespace sysio { using namespace sysio; -#define CALL_WITH_400(api_name, category, api_handle, call_name, INVOKE, http_response_code) \ +#define CALL_WITH_400_STREAM(api_name, category, api_handle, call_name, INVOKE, http_response_code) \ {std::string("/v1/" #api_name "/" #call_name), \ api_category::category, \ - [&api_handle](string&&, string&& body, url_response_callback&& cb) mutable { \ + [&api_handle](string&&, string&& body, url_response_stream_callback&& cb) mutable { \ try { \ INVOKE \ - cb(http_response_code, fc::variant(result)); \ + cb(http_response_code, [r = std::move(result)](fc::json_writer& w) mutable { fc::to_json_stream(r, w); }); \ } catch (...) { \ - http_plugin::handle_exception(#api_name, #call_name, body, cb); \ + http_plugin::handle_exception_stream(#api_name, #call_name, body, cb); \ } \ }} #define INVOKE_R_R(api_handle, call_name, in_param) \ auto params = parse_params(body);\ - fc::variant result( api_handle.call_name( std::move(params) ) ); + auto result = api_handle.call_name( std::move(params) ); #define INVOKE_R_V(api_handle, call_name) \ body = parse_params(body); \ @@ -52,16 +52,16 @@ void net_api_plugin::plugin_startup() { dlog("starting net_api_plugin"); // lifetime of plugin is lifetime of application auto& net_mgr = app().get_plugin(); - app().get_plugin().add_async_api({ - CALL_WITH_400(net, net_rw, net_mgr, connect, + app().get_plugin().add_async_api_stream({ + CALL_WITH_400_STREAM(net, net_rw, net_mgr, connect, INVOKE_R_R(net_mgr, connect, std::string), 201), - CALL_WITH_400(net, net_rw, net_mgr, disconnect, + CALL_WITH_400_STREAM(net, net_rw, net_mgr, disconnect, INVOKE_R_R(net_mgr, disconnect, std::string), 201), - CALL_WITH_400(net, net_ro, net_mgr, status, + CALL_WITH_400_STREAM(net, net_ro, net_mgr, status, INVOKE_R_R(net_mgr, status, std::string), 201), - CALL_WITH_400(net, net_ro, net_mgr, connections, + CALL_WITH_400_STREAM(net, net_ro, net_mgr, connections, INVOKE_R_V(net_mgr, connections), 201), - CALL_WITH_400(net, net_ro, net_mgr, bp_gossip_peers, + CALL_WITH_400_STREAM(net, net_ro, net_mgr, bp_gossip_peers, INVOKE_R_V(net_mgr, bp_gossip_peers), 201), } ); } From 65c3ec77263fe6beea3a7e6b03d3483efed00208 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Tue, 28 Apr 2026 16:48:16 -0500 Subject: [PATCH 34/71] db_size_api_plugin: migrate variant-cb endpoint to streaming The single /v1/db_size/get endpoint flips to CALL_WITH_400_STREAM via add_api_stream(read_only). Threading is unchanged: handler still runs on the read_only exec_queue. --- .../db_size_api_plugin/src/db_size_api_plugin.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/db_size_api_plugin/src/db_size_api_plugin.cpp b/plugins/db_size_api_plugin/src/db_size_api_plugin.cpp index 65e24ab1ec..58380209b4 100644 --- a/plugins/db_size_api_plugin/src/db_size_api_plugin.cpp +++ b/plugins/db_size_api_plugin/src/db_size_api_plugin.cpp @@ -7,16 +7,16 @@ namespace sysio { using namespace sysio; -#define CALL_WITH_400(api_name, api_handle, call_name, INVOKE, http_response_code) \ +#define CALL_WITH_400_STREAM(api_name, api_handle, call_name, INVOKE, http_response_code) \ {std::string("/v1/" #api_name "/" #call_name), \ api_category::db_size, \ - [api_handle](string&&, string&& body, url_response_callback&& cb) mutable { \ + [api_handle](string&&, string&& body, url_response_stream_callback&& cb) mutable { \ try { \ body = parse_params(body); \ INVOKE \ - cb(http_response_code, fc::variant(result)); \ + cb(http_response_code, [r = std::move(result)](fc::json_writer& w) mutable { fc::to_json_stream(r, w); }); \ } catch (...) { \ - http_plugin::handle_exception(#api_name, #call_name, body, cb); \ + http_plugin::handle_exception_stream(#api_name, #call_name, body, cb); \ } \ }} @@ -25,8 +25,8 @@ using namespace sysio; void db_size_api_plugin::plugin_startup() { - app().get_plugin().add_api({ - CALL_WITH_400(db_size, this, get, INVOKE_R_V(this, get), 200), + app().get_plugin().add_api_stream({ + CALL_WITH_400_STREAM(db_size, this, get, INVOKE_R_V(this, get), 200), }, appbase::exec_queue::read_only); } From 0f9fb120b8d023e4d5ce84b7858848a21be05e96 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Tue, 28 Apr 2026 16:58:57 -0500 Subject: [PATCH 35/71] http_plugin: drop unused CALL_WITH_400_POST variant-cb macro After the chain_api / producer_api / net_api / db_size_api migrations, no caller remains for CALL_WITH_400_POST. Removing it. CALL_ASYNC_WITH_400 stays -- prometheus_plugin uses it for the /v1/prometheus/metrics endpoint, which serves text/plain (the prometheus exposition format) and so cannot use the JSON-only streaming macros. CALL_WITH_400 is plugin-local in wallet_api_plugin and stays there as a deliberate guard against streaming private keys (no fc::to_json_stream(crypto::private_key) overload exists). Comment in chain_api_plugin.cpp updated; the now-stale list of variant-cb wrapper names dropped. --- .../chain_api_plugin/src/chain_api_plugin.cpp | 4 +- .../include/sysio/http_plugin/macros.hpp | 39 +------------------ 2 files changed, 2 insertions(+), 41 deletions(-) diff --git a/plugins/chain_api_plugin/src/chain_api_plugin.cpp b/plugins/chain_api_plugin/src/chain_api_plugin.cpp index da0f56b278..df5dab7f27 100644 --- a/plugins/chain_api_plugin/src/chain_api_plugin.cpp +++ b/plugins/chain_api_plugin/src/chain_api_plugin.cpp @@ -90,9 +90,7 @@ parse_params(body); \ - using http_fwd_t = std::function()>; \ - /* called on main application thread */ \ - http_fwd_t http_fwd(api_handle.call_name(std::move(params), deadline)); \ - _http_plugin.post_http_thread_pool([resp_code=http_resp_code, cb=std::move(cb), \ - body=std::move(body), \ - http_fwd = std::move(http_fwd)]() { \ - try { \ - chain::t_or_exception result = http_fwd(); \ - if (std::holds_alternative(result)) { \ - try { \ - throw *std::get(result); \ - } catch (...) { \ - http_plugin::handle_exception(#api_name, #call_name, body, cb); \ - } \ - } else { \ - cb(resp_code, fc::variant(std::get(std::move(result)))); \ - } \ - } catch (...) { \ - http_plugin::handle_exception(#api_name, #call_name, body, cb); \ - } \ - }); \ - } catch (...) { \ - http_plugin::handle_exception(#api_name, #call_name, body, cb); \ - } \ - }} - - -// Streaming-cb counterpart of CALL_WITH_400. Sync endpoint: call runs on the +// Streaming-cb counterpart of (locally-defined) CALL_WITH_400. Sync endpoint: call runs on the // calling queue (typically read_only), result is captured into a json_writer- // emitting closure that the cb forwards to the http thread pool for emission. // No fc::variant tree on the response path. From 8db0f57310a792ce98b9a94d8ce95bdb0e995402 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Tue, 28 Apr 2026 21:04:44 -0500 Subject: [PATCH 36/71] http_plugin: replace streaming-cb macros with bind_stream template Introduces `sysio::bind_stream` (new header `http_plugin/bind_stream.hpp`) as a typed replacement for the four streaming-cb macros previously in `http_plugin/macros.hpp`: - CALL_WITH_400_STREAM -> bind_stream<..., dispatch::sync> - CALL_WITH_400_STREAM_POST -> bind_stream<..., dispatch::post> - CALL_WITH_400_STREAM_POST_DIRECT -> bind_stream<..., dispatch::post_direct> - CALL_ASYNC_WITH_400_STREAM -> bind_stream<..., dispatch::async> The `dispatch` enum is required at every call site, making the threading shape (sync immediate / sync void / Phase-2 closure / Phase-2 emit-fn / async cb) explicit at registration. `static_assert`s pin Kind to the method signature so a refactor that changes the signature produces a build error rather than a silent behavior change. Migrates chain_api_plugin, producer_api_plugin, net_api_plugin, and db_size_api_plugin to the new shape and removes the four streaming-cb macros. `CALL_ASYNC_WITH_400` (variant cb) is retained because prometheus_plugin registers a non-JSON (text/plain) endpoint through it. `wallet_api_plugin` stays on the variant-cb path by design, to keep private-key responses out of the streaming JSON code path. A new helper `dispatch::sync_void` substitutes for producer_api_plugin's fabricated `producer_api_plugin_response{"ok"}` ack (now centralized as `http_detail::ok_response`). --- .../chain_api_plugin/src/chain_api_plugin.cpp | 116 +++-- .../sysio/chain_plugin/chain_plugin.hpp | 2 +- .../src/db_size_api_plugin.cpp | 29 +- .../include/sysio/http_plugin/bind_stream.hpp | 420 ++++++++++++++++++ .../include/sysio/http_plugin/macros.hpp | 159 +------ plugins/net_api_plugin/src/net_api_plugin.cpp | 74 +-- .../src/producer_api_plugin.cpp | 191 +++----- 7 files changed, 584 insertions(+), 407 deletions(-) create mode 100644 plugins/http_plugin/include/sysio/http_plugin/bind_stream.hpp diff --git a/plugins/chain_api_plugin/src/chain_api_plugin.cpp b/plugins/chain_api_plugin/src/chain_api_plugin.cpp index df5dab7f27..4716ba7037 100644 --- a/plugins/chain_api_plugin/src/chain_api_plugin.cpp +++ b/plugins/chain_api_plugin/src/chain_api_plugin.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include #include @@ -89,14 +89,6 @@ parse_params().chain())); @@ -109,59 +101,101 @@ void chain_api_plugin::plugin_startup() { ro_api.set_shorten_abi_errors( !http_plugin::verbose_errors() ); + using ro = chain_apis::read_only; + using rw = chain_apis::read_write; + using cat = api_category; + using pt = http_params_types; + // Run get_info on http thread only _http_plugin.add_async_api_stream({ - CHAIN_RO_CALL_STREAM(get_info, chain_apis::get_info_db::get_info_results, 200, http_params_types::no_params) + bind_stream<&ro::get_info, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_info", cat::chain_ro, pt::no_params, 200), }); _http_plugin.add_api_stream({ - // transaction related APIs will be posted to read_write queue after keys are recovered, they are safe to run in parallel until they post to the read_write queue - CHAIN_RO_CALL_ASYNC_STREAM(compute_transaction, chain_apis::read_only::compute_transaction_results, 200, http_params_types::params_required), - CHAIN_RW_CALL_ASYNC_STREAM(push_transaction, chain_apis::read_write::push_transaction_results, 202, http_params_types::params_required), - CHAIN_RW_CALL_ASYNC_STREAM(push_transactions, chain_apis::read_write::push_transactions_results, 202, http_params_types::params_required), - CHAIN_RW_CALL_ASYNC_STREAM(send_transaction, chain_apis::read_write::send_transaction_results, 202, http_params_types::params_required), - CHAIN_RW_CALL_ASYNC_STREAM(send_transaction2, chain_apis::read_write::send_transaction_results, 202, http_params_types::params_required) + // transaction related APIs will be posted to read_write queue after keys are recovered, they are safe to run + // in parallel until they post to the read_write queue + bind_stream<&ro::compute_transaction, dispatch::async>( + _http_plugin, ro_api, "/v1/chain/compute_transaction", cat::chain_ro, pt::params_required, 200), + bind_stream<&rw::push_transaction, dispatch::async>( + _http_plugin, rw_api, "/v1/chain/push_transaction", cat::chain_rw, pt::params_required, 202), + bind_stream<&rw::push_transactions, dispatch::async>( + _http_plugin, rw_api, "/v1/chain/push_transactions", cat::chain_rw, pt::params_required, 202), + bind_stream<&rw::send_transaction, dispatch::async>( + _http_plugin, rw_api, "/v1/chain/send_transaction", cat::chain_rw, pt::params_required, 202), + bind_stream<&rw::send_transaction2, dispatch::async>( + _http_plugin, rw_api, "/v1/chain/send_transaction2", cat::chain_rw, pt::params_required, 202), }, appbase::exec_queue::read_only); _http_plugin.add_api_stream({ - CHAIN_RO_CALL_STREAM(get_activated_protocol_features, chain_apis::read_only::get_activated_protocol_features_results, 200, http_params_types::possible_no_params), - CHAIN_RO_CALL_STREAM_POST_DIRECT(get_block, 200, http_params_types::params_required), // _POST because get_block_stream() returns a lambda to be executed on the http thread pool - CHAIN_RO_CALL_STREAM(get_block_info, fc::variant, 200, http_params_types::params_required), - CHAIN_RO_CALL_STREAM(get_block_header_state, fc::variant, 200, http_params_types::params_required), - CHAIN_RO_CALL_STREAM(get_code, chain_apis::read_only::get_code_results, 200, http_params_types::params_required), - CHAIN_RO_CALL_STREAM(get_code_hash, chain_apis::read_only::get_code_hash_results, 200, http_params_types::params_required), - CHAIN_RO_CALL_STREAM(get_consensus_parameters, chain_apis::read_only::get_consensus_parameters_results, 200, http_params_types::no_params), - CHAIN_RO_CALL_STREAM(get_abi, chain_apis::read_only::get_abi_results, 200, http_params_types::params_required), - CHAIN_RO_CALL_STREAM(get_raw_code_and_abi, chain_apis::read_only::get_raw_code_and_abi_results, 200, http_params_types::params_required), - CHAIN_RO_CALL_STREAM(get_raw_abi, chain_apis::read_only::get_raw_abi_results, 200, http_params_types::params_required), - CHAIN_RO_CALL_STREAM(get_finalizer_info, chain_apis::read_only::get_finalizer_info_result, 200, http_params_types::no_params), - CHAIN_RO_CALL_STREAM(get_table_by_scope, chain_apis::read_only::get_table_by_scope_result, 200, http_params_types::params_required), - CHAIN_RO_CALL_STREAM(get_currency_balance, std::vector, 200, http_params_types::params_required), - CHAIN_RO_CALL_STREAM(get_currency_stats, fc::variant, 200, http_params_types::params_required), - CHAIN_RO_CALL_STREAM(get_producers, chain_apis::read_only::get_producers_result, 200, http_params_types::params_required), - CHAIN_RO_CALL_STREAM(get_producer_schedule, chain_apis::read_only::get_producer_schedule_result, 200, http_params_types::no_params), - CHAIN_RO_CALL_STREAM(get_required_keys, chain_apis::read_only::get_required_keys_result, 200, http_params_types::params_required), - CHAIN_RO_CALL_STREAM(get_transaction_id, chain_apis::read_only::get_transaction_id_result, 200, http_params_types::params_required), - CHAIN_RO_CALL_STREAM_POST(get_account, chain_apis::read_only::get_account_results, 200, http_params_types::params_required), - CHAIN_RO_CALL_STREAM_POST_DIRECT(get_table_rows, 200, http_params_types::params_required), + bind_stream<&ro::get_activated_protocol_features, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_activated_protocol_features", cat::chain_ro, pt::possible_no_params, 200), + // get_block_stream returns Phase-2 closure (function()>) + bind_stream<&ro::get_block_stream, dispatch::post_direct>( + _http_plugin, ro_api, "/v1/chain/get_block", cat::chain_ro, pt::params_required, 200), + bind_stream<&ro::get_block_info, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_block_info", cat::chain_ro, pt::params_required, 200), + bind_stream<&ro::get_block_header_state, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_block_header_state", cat::chain_ro, pt::params_required, 200), + bind_stream<&ro::get_code, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_code", cat::chain_ro, pt::params_required, 200), + bind_stream<&ro::get_code_hash, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_code_hash", cat::chain_ro, pt::params_required, 200), + bind_stream<&ro::get_consensus_parameters, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_consensus_parameters", cat::chain_ro, pt::no_params, 200), + bind_stream<&ro::get_abi, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_abi", cat::chain_ro, pt::params_required, 200), + bind_stream<&ro::get_raw_code_and_abi, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_raw_code_and_abi", cat::chain_ro, pt::params_required, 200), + bind_stream<&ro::get_raw_abi, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_raw_abi", cat::chain_ro, pt::params_required, 200), + bind_stream<&ro::get_finalizer_info, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_finalizer_info", cat::chain_ro, pt::no_params, 200), + bind_stream<&ro::get_table_by_scope, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_table_by_scope", cat::chain_ro, pt::params_required, 200), + bind_stream<&ro::get_currency_balance, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_currency_balance", cat::chain_ro, pt::params_required, 200), + bind_stream<&ro::get_currency_stats, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_currency_stats", cat::chain_ro, pt::params_required, 200), + bind_stream<&ro::get_producers, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_producers", cat::chain_ro, pt::params_required, 200), + bind_stream<&ro::get_producer_schedule, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_producer_schedule", cat::chain_ro, pt::no_params, 200), + bind_stream<&ro::get_required_keys, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_required_keys", cat::chain_ro, pt::params_required, 200), + bind_stream<&ro::get_transaction_id, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_transaction_id", cat::chain_ro, pt::params_required, 200), + // get_account returns Phase-2 closure (function()>) + bind_stream<&ro::get_account, dispatch::post>( + _http_plugin, ro_api, "/v1/chain/get_account", cat::chain_ro, pt::params_required, 200), + // get_table_rows_stream returns Phase-2 closure + bind_stream<&ro::get_table_rows_stream, dispatch::post_direct>( + _http_plugin, ro_api, "/v1/chain/get_table_rows", cat::chain_ro, pt::params_required, 200), }, appbase::exec_queue::read_only); if (chain.account_queries_enabled()) { _http_plugin.add_async_api_stream({ - CHAIN_RO_CALL_STREAM(get_accounts_by_authorizers, chain_apis::read_only::get_accounts_by_authorizers_result, 200, http_params_types::params_required), + bind_stream<&ro::get_accounts_by_authorizers, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_accounts_by_authorizers", + cat::chain_ro, pt::params_required, 200), }); } _http_plugin.add_async_api_stream({ // chain_plugin send_read_only_transaction will post to read_exclusive queue - CHAIN_RO_CALL_ASYNC_STREAM(send_read_only_transaction, chain_apis::read_only::send_read_only_transaction_results, 200, http_params_types::params_required), - CHAIN_RO_CALL_STREAM(get_raw_block, chain::signed_block_ptr, 200, http_params_types::params_required), - CHAIN_RO_CALL_STREAM(get_block_header, chain_apis::read_only::get_block_header_result, 200, http_params_types::params_required) + bind_stream<&ro::send_read_only_transaction, dispatch::async>( + _http_plugin, ro_api, "/v1/chain/send_read_only_transaction", cat::chain_ro, pt::params_required, 200), + bind_stream<&ro::get_raw_block, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_raw_block", cat::chain_ro, pt::params_required, 200), + bind_stream<&ro::get_block_header, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_block_header", cat::chain_ro, pt::params_required, 200), }); if (chain.transaction_finality_status_enabled()) { _http_plugin.add_api_stream({ - CHAIN_RO_CALL_STREAM(get_transaction_status, chain_apis::read_only::get_transaction_status_results, 200, http_params_types::params_required), + bind_stream<&ro::get_transaction_status, dispatch::sync>( + _http_plugin, ro_api, "/v1/chain/get_transaction_status", + cat::chain_ro, pt::params_required, 200), }, appbase::exec_queue::read_only); } diff --git a/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp b/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp index 5b857c3a2a..a560183168 100644 --- a/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp +++ b/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp @@ -389,7 +389,7 @@ class read_only : public api_base { /// - Phase 1 (main app thread, read_only queue): this method's body. `get_raw_block` is itself thread-safe; /// `get_serializers_cache` is what *requires* the main thread because it reads each referenced account's `abi` /// field out of chainbase. Returns the outer http_fwd closure. - /// - Hop 1: `CALL_WITH_400_STREAM_POST_DIRECT` posts the outer closure to the http thread pool. + /// - Hop 1: `bind_stream<&get_block_stream, dispatch::post_direct>` posts the outer closure to the http thread pool. /// - Phase 2 (http thread pool): invokes the outer closure -- captured block + resolver snapshots, no chainbase /// access -- and produces the inner `json_writer`-emitting closure, handed to `cb`. /// - Hop 2: `make_http_stream_response_handler` re-dispatches onto the same http thread pool (inherited from the diff --git a/plugins/db_size_api_plugin/src/db_size_api_plugin.cpp b/plugins/db_size_api_plugin/src/db_size_api_plugin.cpp index 58380209b4..df886010b8 100644 --- a/plugins/db_size_api_plugin/src/db_size_api_plugin.cpp +++ b/plugins/db_size_api_plugin/src/db_size_api_plugin.cpp @@ -1,32 +1,18 @@ #include #include #include -#include +#include namespace sysio { using namespace sysio; -#define CALL_WITH_400_STREAM(api_name, api_handle, call_name, INVOKE, http_response_code) \ -{std::string("/v1/" #api_name "/" #call_name), \ - api_category::db_size, \ - [api_handle](string&&, string&& body, url_response_stream_callback&& cb) mutable { \ - try { \ - body = parse_params(body); \ - INVOKE \ - cb(http_response_code, [r = std::move(result)](fc::json_writer& w) mutable { fc::to_json_stream(r, w); }); \ - } catch (...) { \ - http_plugin::handle_exception_stream(#api_name, #call_name, body, cb); \ - } \ - }} - -#define INVOKE_R_V(api_handle, call_name) \ - auto result = api_handle->call_name(); - - void db_size_api_plugin::plugin_startup() { - app().get_plugin().add_api_stream({ - CALL_WITH_400_STREAM(db_size, this, get, INVOKE_R_V(this, get), 200), + auto& _http_plugin = app().get_plugin(); + _http_plugin.add_api_stream({ + bind_stream<&db_size_api_plugin::get, dispatch::sync>( + _http_plugin, this, "/v1/db_size/get", + api_category::db_size, http_params_types::no_params, 200), }, appbase::exec_queue::read_only); } @@ -46,7 +32,4 @@ db_size_stats db_size_api_plugin::get() { return ret; } -#undef INVOKE_R_V -#undef CALL - } diff --git a/plugins/http_plugin/include/sysio/http_plugin/bind_stream.hpp b/plugins/http_plugin/include/sysio/http_plugin/bind_stream.hpp new file mode 100644 index 0000000000..a414c78639 --- /dev/null +++ b/plugins/http_plugin/include/sysio/http_plugin/bind_stream.hpp @@ -0,0 +1,420 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace sysio { + +/** + * @brief Compile-time tag selecting the response control flow used by `bind_stream`. + * + * - `sync` : `R api.method(args...)` -- result is captured directly into the + * json_writer-emitting closure handed to the streaming cb. + * - `sync_void` : `void api.method(args...)` -- emits the convention `{"result":"ok"}` + * response, matching the producer_api / net_api void-method pattern. + * - `post` : `function()> api.method(args...)` -- Phase 1 + * runs on the calling queue and returns a closure (`http_fwd`) that + * will produce the typed result; Phase 2 runs `http_fwd` on the http + * thread pool and emits via `to_json_stream(result)`. + * - `post_direct` : `function()> api.method(args...)` -- like + * `post`, but Phase 2 hands the emit closure straight to the cb. + * Used when the api builds and emits its JSON without an + * intermediate reflected struct (eg per-row binary decode). + * - `async` : `void api.method(args..., next_function)` -- the api invokes + * the callback later with either a typed result, an exception, or a + * Phase-2 `http_fwd` closure (mirroring the variant-cb semantics). + */ +enum class dispatch { sync, sync_void, post, post_direct, async }; + +namespace http_detail { + + /// Standard `{"result":"ok"}` reply emitted by `dispatch::sync_void`. Reflected + /// here so each api plugin doesn't have to define its own ack struct. + struct ok_response { + std::string result = "ok"; + }; + + using emit_fn_t = std::function; + + /// @brief Pulls apart a member-function pointer into its pieces. + /// + /// Specialized for both non-const and const-qualified member functions so + /// `member_fn` works regardless of cv-qualification. + template struct member_fn; + + template + struct member_fn { + using class_t = Cls; + using ret_t = R; + template using arg_t = std::tuple_element_t>; + static constexpr size_t arity = sizeof...(A); + }; + + template + struct member_fn { + using class_t = Cls; + using ret_t = R; + template using arg_t = std::tuple_element_t>; + static constexpr size_t arity = sizeof...(A); + }; + + /// @brief Detects `function()>` (the Phase-2 closure shape). + template struct is_typed_fwd : std::false_type {}; + template + struct is_typed_fwd()>> : std::true_type { + using payload = T; + }; + + /// @brief Detects `chain::plugin_interface::next_function` (the async cb shape). + template struct is_next_fn : std::false_type {}; + template + struct is_next_fn> : std::true_type { + using payload = T; + }; + + /// @brief True iff T is `fc::time_point` after stripping cv/ref. + template + inline constexpr bool is_deadline_v = + std::is_same_v, fc::time_point>; + + /// Lazily evaluates the method's first argument bare-type when `HasArgs` is true. + /// Required because `std::conditional_t` eagerly instantiates both branches, so a + /// direct `mf::arg_t<0>` reference fails to compile for arity-0 methods. + template + struct first_arg_or_void { using type = std::monostate; }; + template + struct first_arg_or_void { + using type = std::remove_cvref_t>; + }; + + /// Compute the per-request deadline. Chain api handles expose `start()` which both + /// returns the deadline and registers the in-flight call; other handles fall back to + /// `http_plugin::get_max_response_time()`. + template + inline fc::time_point compute_deadline(Handle& handle, http_plugin& http) { + if constexpr (requires { handle.start(); }) { + return handle.start(); + } else if constexpr (requires { handle->start(); }) { + return handle->start(); + } else { + const fc::microseconds m = http.get_max_response_time(); + return m == fc::microseconds::maximum() ? fc::time_point::maximum() + : fc::time_point::now() + m; + } + } + + /// Runtime dispatch wrapper around the compile-time `parse_params` template. + /// Three instantiations per T -- compilers inline through the switch. + template + inline T parse_params_rt(const std::string& body, http_params_types pt) { + switch (pt) { + case http_params_types::no_params: + return parse_params(body); + case http_params_types::params_required: + return parse_params(body); + case http_params_types::possible_no_params: + return parse_params(body); + } + __builtin_unreachable(); + } + + /// Parse a registration path of the form "/v1//" into the + /// (api_name, call_name) labels used by `handle_exception_stream`. + inline std::pair split_path(const std::string& path) { + auto last_slash = path.find_last_of('/'); + auto prev_slash = path.find_last_of('/', last_slash > 0 ? last_slash - 1 : 0); + std::string api_name = path.substr(prev_slash + 1, last_slash - prev_slash - 1); + std::string call_name = path.substr(last_slash + 1); + return { std::move(api_name), std::move(call_name) }; + } + + /// Invoke `MethodPtr` against `handle` with the right argument shape: + /// arity 0 -> () + /// arity 1 -> (params) + /// arity 2 + deadline-> (params, deadline) + /// arity 1 async -> (next_function) + /// arity 2 async -> (params, next_function) + /// Deadline-vs-not is selected by inspecting the last param type at compile time. + template + inline auto invoke_sync(Handle& handle, Params&& params, fc::time_point deadline) { + using mf = member_fn; + if constexpr (mf::arity == 0) { + return std::invoke(MethodPtr, handle); + } else if constexpr (mf::arity == 1) { + return std::invoke(MethodPtr, handle, std::forward(params)); + } else { + // arity == 2: (params, deadline). + static_assert(is_deadline_v>, + "second argument must be fc::time_point (deadline)"); + return std::invoke(MethodPtr, handle, std::forward(params), deadline); + } + } + + /// Async variant: last argument is `next_function`, called by api method later. + template + inline void invoke_async(Handle& handle, Params&& params, Next&& next) { + using mf = member_fn; + if constexpr (mf::arity == 1) { + // (next_function) -- no params arg, body must be empty + std::invoke(MethodPtr, handle, std::forward(next)); + } else { + // (params, next_function) + std::invoke(MethodPtr, handle, std::forward(params), std::forward(next)); + } + } + +} // namespace http_detail + + +/** + * @brief Build an `api_entry_stream` registration for an api method, dispatching the + * response per `Kind`. + * + * @tparam MethodPtr pointer-to-member of the api class implementing the endpoint + * @tparam Kind which response shape (`dispatch::sync` etc) the method uses; + * compile-time `static_assert`s pin Kind to the method signature + * so a refactor that changes the signature produces a build error, + * not a silent behavior change. + * @tparam Handle the api handle type (read_only, read_write, producer_plugin&, + * plugin*, ...). Captured by-value into the lambda; for handles + * that should be referenced across requests pass an explicit + * reference type or pointer. + * + * @param http the http_plugin instance, used for the http thread pool, max + * response time, and exception logging. + * @param handle the api handle, captured into the request lambda. + * @param path e.g. "/v1/chain/get_info"; used for url registration AND for + * deriving the (api_name, call_name) labels passed to + * `handle_exception_stream` on errors. + * @param cat api category for the registration. + * @param pt parameter parsing mode for the request body. + * @param resp_code HTTP status code on success. + * + * @return an `api_entry_stream` ready to be batched into + * `http_plugin::add_api_stream` / `add_async_api_stream`. + */ +template +api_entry_stream bind_stream(http_plugin& http, Handle handle, + std::string path, api_category cat, + http_params_types pt, uint16_t resp_code) { + using namespace http_detail; + using mf = member_fn; + using ret_t = typename mf::ret_t; + + // ----- Compile-time signature validation against `Kind` ------------------ + if constexpr (Kind == dispatch::sync) { + static_assert(!std::is_void_v, + "dispatch::sync expects T return; method returns void (use dispatch::sync_void)."); + static_assert(!is_typed_fwd::value, + "dispatch::sync method returns function()>; use dispatch::post or dispatch::post_direct."); + } else if constexpr (Kind == dispatch::sync_void) { + static_assert(std::is_void_v, + "dispatch::sync_void expects void return; method returns a value."); + } else if constexpr (Kind == dispatch::post) { + static_assert(is_typed_fwd::value, + "dispatch::post expects function()> return."); + static_assert(!std::is_same_v::payload, emit_fn_t>, + "dispatch::post payload is emit_fn_t; use dispatch::post_direct instead."); + } else if constexpr (Kind == dispatch::post_direct) { + static_assert(std::is_same_v()>>, + "dispatch::post_direct expects function()> return."); + } else { // dispatch::async + static_assert(std::is_void_v, + "dispatch::async expects void return."); + static_assert(mf::arity >= 1, "dispatch::async requires at least one arg (next_function)."); + static_assert( + is_next_fn>>::value, + "dispatch::async requires the last argument to be next_function."); + } + + // ----- Parameter type deduction ------------------------------------------ + // `params_t` is the *first* method arg's bare type (when present). + // For arity-0 sync methods we don't parse a body. + constexpr bool has_params = []() { + if constexpr (Kind == dispatch::async) { + // async signature: (next_function) or (params, next_function) + return mf::arity >= 2; + } else { + return mf::arity >= 1; + } + }(); + + using params_t = typename http_detail::first_arg_or_void::type; + + auto labels = split_path(path); + std::string api_name = std::move(labels.first); + std::string call_name = std::move(labels.second); + + return api_entry_stream { + std::move(path), cat, + [handle = std::move(handle), &http, pt, resp_code, + api_name = std::move(api_name), call_name = std::move(call_name)] + (std::string&&, std::string&& body, url_response_stream_callback&& cb) mutable { + // `http` is referenced by some Kinds (post/post_direct/async/sync via start()); + // taking its address unconditionally keeps the capture flagged as used so the + // compiler does not emit -Wunused-lambda-capture for sync_void instantiations. + (void)&http; + + // For sync paths that need a deadline, compute it before the try{} so that + // chain_api's start() side effect runs even if param parsing throws. + fc::time_point deadline; + if constexpr (Kind != dispatch::sync_void) { + deadline = compute_deadline(handle, http); + } + + try { + // ---- Parse params (if the method takes any) ---- + [[maybe_unused]] params_t params; + if constexpr (has_params) { + params = parse_params_rt(body, pt); + } else { + // No params expected -- still validate the body per `pt`. + (void)parse_params_rt(body, pt); + } + + // ---- Dispatch on Kind -------------------------------------- + if constexpr (Kind == dispatch::sync) { + auto result = invoke_sync(handle, std::move(params), deadline); + cb(resp_code, [r = std::move(result)](fc::json_writer& w) mutable { + fc::to_json_stream(r, w); + }); + + } else if constexpr (Kind == dispatch::sync_void) { + if constexpr (mf::arity == 0) { + std::invoke(MethodPtr, handle); + } else { + std::invoke(MethodPtr, handle, std::move(params)); + } + cb(resp_code, [](fc::json_writer& w) { + ok_response r; + fc::to_json_stream(r, w); + }); + + } else if constexpr (Kind == dispatch::post) { + using payload_t = typename is_typed_fwd::payload; + using http_fwd_t = std::function()>; + http_fwd_t http_fwd = invoke_sync(handle, std::move(params), deadline); + http.post_http_thread_pool( + [resp_code, cb = std::move(cb), body = std::move(body), + api_name, call_name, + http_fwd = std::move(http_fwd)]() mutable { + try { + chain::t_or_exception result = http_fwd(); + if (std::holds_alternative(result)) { + try { throw *std::get(result); } + catch (...) { + http_plugin::handle_exception_stream( + api_name.c_str(), call_name.c_str(), body, cb); + } + } else { + cb(resp_code, + [r = std::get(std::move(result))] + (fc::json_writer& w) mutable { fc::to_json_stream(r, w); }); + } + } catch (...) { + http_plugin::handle_exception_stream( + api_name.c_str(), call_name.c_str(), body, cb); + } + }); + + } else if constexpr (Kind == dispatch::post_direct) { + using http_fwd_t = std::function()>; + http_fwd_t http_fwd = invoke_sync(handle, std::move(params), deadline); + http.post_http_thread_pool( + [resp_code, cb = std::move(cb), body = std::move(body), + api_name, call_name, + http_fwd = std::move(http_fwd)]() mutable { + try { + chain::t_or_exception result = http_fwd(); + if (std::holds_alternative(result)) { + try { throw *std::get(result); } + catch (...) { + http_plugin::handle_exception_stream( + api_name.c_str(), call_name.c_str(), body, cb); + } + } else { + cb(resp_code, std::get(std::move(result))); + } + } catch (...) { + http_plugin::handle_exception_stream( + api_name.c_str(), call_name.c_str(), body, cb); + } + }); + + } else { // dispatch::async + using payload_t = typename is_next_fn< + std::remove_cvref_t>>::payload; + using http_fwd_t = std::function()>; + + auto next = [&http, cb = std::move(cb), body = std::move(body), + api_name, call_name, resp_code] + (const chain::plugin_interface::next_function_variant& result) mutable { + if (std::holds_alternative(result)) { + try { throw *std::get(result); } + catch (...) { + http_plugin::handle_exception_stream( + api_name.c_str(), call_name.c_str(), body, cb); + } + } else if (std::holds_alternative(result)) { + cb(resp_code, + [r = std::get(result)] + (fc::json_writer& w) mutable { fc::to_json_stream(r, w); }); + } else { + http.post_http_thread_pool( + [resp_code, cb = std::move(cb), body = std::move(body), + api_name, call_name, + http_fwd = std::get(std::move(result))]() mutable { + try { + chain::t_or_exception r = http_fwd(); + if (std::holds_alternative(r)) { + try { throw *std::get(r); } + catch (...) { + http_plugin::handle_exception_stream( + api_name.c_str(), call_name.c_str(), body, cb); + } + } else { + cb(resp_code, + [v = std::get(std::move(r))] + (fc::json_writer& w) mutable { + fc::to_json_stream(v, w); + }); + } + } catch (...) { + http_plugin::handle_exception_stream( + api_name.c_str(), call_name.c_str(), body, cb); + } + }); + } + }; + + // Empty body -> default-constructed params for async (matches + // CALL_ASYNC_STREAM "if (body.empty()) body = "{}";" preflight). + if constexpr (has_params) { + invoke_async(handle, std::move(params), std::move(next)); + } else { + invoke_async(handle, std::monostate{}, std::move(next)); + } + } + } catch (...) { + http_plugin::handle_exception_stream( + api_name.c_str(), call_name.c_str(), body, cb); + } + } + }; +} + +} // namespace sysio + +FC_REFLECT(sysio::http_detail::ok_response, (result)); diff --git a/plugins/http_plugin/include/sysio/http_plugin/macros.hpp b/plugins/http_plugin/include/sysio/http_plugin/macros.hpp index b74aba894c..980463ce7a 100644 --- a/plugins/http_plugin/include/sysio/http_plugin/macros.hpp +++ b/plugins/http_plugin/include/sysio/http_plugin/macros.hpp @@ -1,7 +1,8 @@ #pragma once -#include -#include +// Variant-cb async-call macro retained for endpoints that emit a non-JSON content +// type (e.g. prometheus_plugin returning text/plain via fc::variant string payload). +// Streaming JSON endpoints use sysio::bind_stream from . #define CALL_ASYNC_WITH_400(api_name, category, api_handle, api_namespace, call_name, call_result, http_resp_code, params_type) \ { std::string("/v1/" #api_name "/" #call_name), \ @@ -46,157 +47,3 @@ } \ } \ } - - -// Streaming-cb counterpart of (locally-defined) CALL_WITH_400. Sync endpoint: call runs on the -// calling queue (typically read_only), result is captured into a json_writer- -// emitting closure that the cb forwards to the http thread pool for emission. -// No fc::variant tree on the response path. -// ------------------------------------------------------------------------------------------------------ -#define CALL_WITH_400_STREAM(api_name, category, api_handle, api_namespace, call_name, call_result, http_resp_code, params_type) \ -{std::string("/v1/" #api_name "/" #call_name), \ - api_category::category, \ - [api_handle](string&&, string&& body, url_response_stream_callback&& cb) mutable { \ - auto deadline = api_handle.start(); \ - try { \ - auto params = parse_params(body); \ - call_result result = api_handle.call_name(std::move(params), deadline); \ - cb(http_resp_code, [r = std::move(result)](fc::json_writer& w) mutable { \ - fc::to_json_stream(r, w); \ - }); \ - } catch (...) { \ - http_plugin::handle_exception_stream(#api_name, #call_name, body, cb); \ - } \ - }} - - -// Direct-streaming counterpart of CALL_WITH_400_STREAM_POST. Phase 1 returns a -// closure that, when invoked on the http thread pool (Phase 2), produces the -// final json_writer-emitting closure directly -- no typed `call_result` struct -// on the response path. Used when the api builds and emits its JSON response -// without an intermediate variant or reflected struct (eg per-row binary decode). -// Method name on api_handle is `_stream` so the variant-cb method -// continues to compile alongside. -// ------------------------------------------------------------------------------------------------------ -#define CALL_WITH_400_STREAM_POST_DIRECT(api_name, category, api_handle, api_namespace, call_name, http_resp_code, params_type) \ -{std::string("/v1/" #api_name "/" #call_name), \ - api_category::category, \ - [api_handle, &_http_plugin](string&&, string&& body, url_response_stream_callback&& cb) { \ - auto deadline = api_handle.start(); \ - try { \ - auto params = parse_params(body); \ - using emit_fn_t = std::function; \ - using http_fwd_t = std::function()>; \ - http_fwd_t http_fwd(api_handle.call_name ## _stream(std::move(params), deadline)); \ - _http_plugin.post_http_thread_pool([resp_code=http_resp_code, cb=std::move(cb), \ - body=std::move(body), \ - http_fwd = std::move(http_fwd)]() mutable { \ - try { \ - chain::t_or_exception result = http_fwd(); \ - if (std::holds_alternative(result)) { \ - try { \ - throw *std::get(result); \ - } catch (...) { \ - http_plugin::handle_exception_stream(#api_name, #call_name, body, cb); \ - } \ - } else { \ - cb(resp_code, std::get(std::move(result))); \ - } \ - } catch (...) { \ - http_plugin::handle_exception_stream(#api_name, #call_name, body, cb); \ - } \ - }); \ - } catch (...) { \ - http_plugin::handle_exception_stream(#api_name, #call_name, body, cb); \ - } \ - }} - - -// Streaming-cb counterpart of CALL_WITH_400_POST. Same Phase 1 (api thread) / -// Phase 2 (http thread pool) split as the variant version, but Phase 2 hands -// the typed call_result to cb as a json_writer-emitting closure - no fc::variant -// tree on the response path. -// ------------------------------------------------------------------------------------------------------ -#define CALL_WITH_400_STREAM_POST(api_name, category, api_handle, api_namespace, call_name, call_result, http_resp_code, params_type) \ -{std::string("/v1/" #api_name "/" #call_name), \ - api_category::category, \ - [api_handle, &_http_plugin](string&&, string&& body, url_response_stream_callback&& cb) { \ - auto deadline = api_handle.start(); \ - try { \ - auto params = parse_params(body); \ - using http_fwd_t = std::function()>; \ - /* called on main application thread */ \ - http_fwd_t http_fwd(api_handle.call_name(std::move(params), deadline)); \ - _http_plugin.post_http_thread_pool([resp_code=http_resp_code, cb=std::move(cb), \ - body=std::move(body), \ - http_fwd = std::move(http_fwd)]() mutable { \ - try { \ - chain::t_or_exception result = http_fwd(); \ - if (std::holds_alternative(result)) { \ - try { \ - throw *std::get(result); \ - } catch (...) { \ - http_plugin::handle_exception_stream(#api_name, #call_name, body, cb); \ - } \ - } else { \ - cb(resp_code, [r = std::get(std::move(result))] \ - (fc::json_writer& w) mutable { fc::to_json_stream(r, w); }); \ - } \ - } catch (...) { \ - http_plugin::handle_exception_stream(#api_name, #call_name, body, cb); \ - } \ - }); \ - } catch (...) { \ - http_plugin::handle_exception_stream(#api_name, #call_name, body, cb); \ - } \ - }} - - -// Streaming-cb counterpart of CALL_ASYNC_WITH_400. Same control flow; the api -// lambda hands the result struct to the cb as a json_writer-emitting closure -// instead of as a fc::variant. No per-field allocation on the response path. -// ------------------------------------------------------------------------------------------------------ -#define CALL_ASYNC_WITH_400_STREAM(api_name, category, api_handle, api_namespace, call_name, call_result, http_resp_code, params_type) \ -{ std::string("/v1/" #api_name "/" #call_name), \ - api_category::category, \ - [api_handle, &_http_plugin](string&&, string&& body, url_response_stream_callback&& cb) mutable { \ - api_handle.start(); \ - try { \ - auto params = parse_params(body); \ - using http_fwd_t = std::function()>; \ - api_handle.call_name( std::move(params), /* called on main application thread */ \ - [&_http_plugin, cb=std::move(cb), body=std::move(body)] \ - (const chain::next_function_variant& result) mutable { \ - if (std::holds_alternative(result)) { \ - try { \ - throw *std::get(result); \ - } catch (...) { \ - http_plugin::handle_exception_stream(#api_name, #call_name, body, cb); \ - } \ - } else if (std::holds_alternative(result)) { \ - cb(http_resp_code, [r = std::get(std::move(result))] \ - (fc::json_writer& w) mutable { fc::to_json_stream(r, w); }); \ - } else { \ - assert(std::holds_alternative(result)); \ - _http_plugin.post_http_thread_pool([resp_code=http_resp_code, cb=std::move(cb), \ - body=std::move(body), \ - http_fwd = std::get(std::move(result))]() mutable { \ - chain::t_or_exception result = http_fwd(); \ - if (std::holds_alternative(result)) { \ - try { \ - throw *std::get(result); \ - } catch (...) { \ - http_plugin::handle_exception_stream(#api_name, #call_name, body, cb); \ - } \ - } else { \ - cb(resp_code, [r = std::get(std::move(result))] \ - (fc::json_writer& w) mutable { fc::to_json_stream(r, w); }); \ - } \ - }); \ - } \ - }); \ - } catch (...) { \ - http_plugin::handle_exception_stream(#api_name, #call_name, body, cb); \ - } \ - } \ -} diff --git a/plugins/net_api_plugin/src/net_api_plugin.cpp b/plugins/net_api_plugin/src/net_api_plugin.cpp index 73fc5799d8..b7963063ee 100644 --- a/plugins/net_api_plugin/src/net_api_plugin.cpp +++ b/plugins/net_api_plugin/src/net_api_plugin.cpp @@ -1,69 +1,38 @@ #include #include #include +#include #include #include #include -namespace sysio { namespace detail { - struct net_api_plugin_empty {}; -}} - -FC_REFLECT(sysio::detail::net_api_plugin_empty, ); - namespace sysio { using namespace sysio; -#define CALL_WITH_400_STREAM(api_name, category, api_handle, call_name, INVOKE, http_response_code) \ -{std::string("/v1/" #api_name "/" #call_name), \ - api_category::category, \ - [&api_handle](string&&, string&& body, url_response_stream_callback&& cb) mutable { \ - try { \ - INVOKE \ - cb(http_response_code, [r = std::move(result)](fc::json_writer& w) mutable { fc::to_json_stream(r, w); }); \ - } catch (...) { \ - http_plugin::handle_exception_stream(#api_name, #call_name, body, cb); \ - } \ - }} - -#define INVOKE_R_R(api_handle, call_name, in_param) \ - auto params = parse_params(body);\ - auto result = api_handle.call_name( std::move(params) ); - -#define INVOKE_R_V(api_handle, call_name) \ - body = parse_params(body); \ - auto result = api_handle.call_name(); - -#define INVOKE_V_R(api_handle, call_name, in_param) \ - auto params = parse_params(body);\ - api_handle.call_name( std::move(params) ); \ - sysio::detail::net_api_plugin_empty result; - -#define INVOKE_V_V(api_handle, call_name) \ - body = parse_params(body); \ - api_handle.call_name(); \ - sysio::detail::net_api_plugin_empty result; - - void net_api_plugin::plugin_startup() { dlog("starting net_api_plugin"); // lifetime of plugin is lifetime of application auto& net_mgr = app().get_plugin(); - app().get_plugin().add_async_api_stream({ - CALL_WITH_400_STREAM(net, net_rw, net_mgr, connect, - INVOKE_R_R(net_mgr, connect, std::string), 201), - CALL_WITH_400_STREAM(net, net_rw, net_mgr, disconnect, - INVOKE_R_R(net_mgr, disconnect, std::string), 201), - CALL_WITH_400_STREAM(net, net_ro, net_mgr, status, - INVOKE_R_R(net_mgr, status, std::string), 201), - CALL_WITH_400_STREAM(net, net_ro, net_mgr, connections, - INVOKE_R_V(net_mgr, connections), 201), - CALL_WITH_400_STREAM(net, net_ro, net_mgr, bp_gossip_peers, - INVOKE_R_V(net_mgr, bp_gossip_peers), 201), - } ); + auto& _http_plugin = app().get_plugin(); + + using cat = api_category; + using pt = http_params_types; + + _http_plugin.add_async_api_stream({ + bind_stream<&net_plugin::connect, dispatch::sync>( + _http_plugin, std::ref(net_mgr), "/v1/net/connect", cat::net_rw, pt::params_required, 201), + bind_stream<&net_plugin::disconnect, dispatch::sync>( + _http_plugin, std::ref(net_mgr), "/v1/net/disconnect", cat::net_rw, pt::params_required, 201), + bind_stream<&net_plugin::status, dispatch::sync>( + _http_plugin, std::ref(net_mgr), "/v1/net/status", cat::net_ro, pt::params_required, 201), + bind_stream<&net_plugin::connections, dispatch::sync>( + _http_plugin, std::ref(net_mgr), "/v1/net/connections", cat::net_ro, pt::no_params, 201), + bind_stream<&net_plugin::bp_gossip_peers, dispatch::sync>( + _http_plugin, std::ref(net_mgr), "/v1/net/bp_gossip_peers", cat::net_ro, pt::no_params, 201), + }); } void net_api_plugin::plugin_initialize(const variables_map& options) { @@ -82,11 +51,4 @@ void net_api_plugin::plugin_initialize(const variables_map& options) { } FC_LOG_AND_RETHROW() } - -#undef INVOKE_R_R -#undef INVOKE_R_V -#undef INVOKE_V_R -#undef INVOKE_V_V -#undef CALL - } diff --git a/plugins/producer_api_plugin/src/producer_api_plugin.cpp b/plugins/producer_api_plugin/src/producer_api_plugin.cpp index a15cb5d45b..098d5bc7ee 100644 --- a/plugins/producer_api_plugin/src/producer_api_plugin.cpp +++ b/plugins/producer_api_plugin/src/producer_api_plugin.cpp @@ -1,145 +1,83 @@ #include #include +#include #include #include #include -namespace sysio { namespace detail { - struct producer_api_plugin_response { - std::string result; - }; -}} - -FC_REFLECT(sysio::detail::producer_api_plugin_response, (result)); - namespace sysio { using namespace sysio; -// Streaming-cb counterparts of the local CALL_WITH_400 / CALL_ASYNC. The result -// struct is captured by move into a json_writer-emitting closure and handed to -// the streaming cb; no fc::variant is built. -#define CALL_WITH_400_STREAM(api_name, category, api_handle, call_name, INVOKE, http_response_code) \ -{std::string("/v1/" #api_name "/" #call_name), \ - api_category::category, \ - [&producer](string&&, string&& body, url_response_stream_callback&& cb) mutable { \ - try { \ - INVOKE \ - cb(http_response_code, [r = std::move(result)](fc::json_writer& w) mutable { fc::to_json_stream(r, w); }); \ - } catch (...) { \ - http_plugin::handle_exception_stream(#api_name, #call_name, body, cb); \ - } \ - }} - -#define CALL_ASYNC_STREAM(api_name, category, api_handle, call_name, call_result, INVOKE, http_response_code) \ -{std::string("/v1/" #api_name "/" #call_name), \ - api_category::category, \ - [&api_handle](string&&, string&& body, url_response_stream_callback&& cb) mutable { \ - if (body.empty()) body = "{}"; \ - auto next = [cb=std::move(cb), body=std::move(body)](const chain::next_function_variant& result) mutable { \ - if (std::holds_alternative(result)) {\ - try {\ - throw *std::get(result);\ - } catch (...) {\ - http_plugin::handle_exception_stream(#api_name, #call_name, body, cb);\ - }\ - } else if (std::holds_alternative(result)) { \ - cb(http_response_code, [r = std::get(result)](fc::json_writer& w) mutable { fc::to_json_stream(r, w); });\ - } else { \ - assert(0); \ - } \ - };\ - INVOKE\ - }\ -} - -#define INVOKE_R_R(api_handle, call_name, in_param) \ - auto params = parse_params(body);\ - auto result = api_handle.call_name(std::move(params)); - -#define INVOKE_R_R_II(api_handle, call_name, in_param) \ - auto params = parse_params(body);\ - auto result = api_handle.call_name(std::move(params)); - -#define INVOKE_R_R_D(api_handle, call_name, in_param) \ - auto& http = app().get_plugin(); \ - const fc::microseconds http_max_response_time = http.get_max_response_time(); \ - auto deadline = http_max_response_time == fc::microseconds::maximum() ? fc::time_point::maximum() \ - : fc::time_point::now() + http_max_response_time; \ - auto params = parse_params(body);\ - auto result = api_handle.call_name(std::move(params), deadline); - -#define INVOKE_R_V(api_handle, call_name) \ - body = parse_params(body); \ - auto result = api_handle.call_name(); - -#define INVOKE_R_V_ASYNC(api_handle, call_name)\ - api_handle.call_name(std::move(next)); - -#define INVOKE_V_R(api_handle, call_name, in_param) \ - auto params = parse_params(body);\ - api_handle.call_name(std::move(params)); \ - sysio::detail::producer_api_plugin_response result{"ok"}; - -#define INVOKE_V_V(api_handle, call_name) \ - body = parse_params(body); \ - api_handle.call_name(); \ - sysio::detail::producer_api_plugin_response result{"ok"}; - - void producer_api_plugin::plugin_startup() { dlog("starting producer_api_plugin"); // lifetime of plugin is lifetime of application auto& producer = app().get_plugin(); - - app().get_plugin().add_api_stream({ - CALL_WITH_400_STREAM(producer, producer_ro, producer, paused, - INVOKE_R_V(producer, paused), 201), - CALL_WITH_400_STREAM(producer, producer_ro, producer, get_runtime_options, - INVOKE_R_V(producer, get_runtime_options), 201), - CALL_WITH_400_STREAM(producer, producer_ro, producer, get_greylist, - INVOKE_R_V(producer, get_greylist), 201), - CALL_WITH_400_STREAM(producer, producer_ro, producer, get_whitelist_blacklist, - INVOKE_R_V(producer, get_whitelist_blacklist), 201), - CALL_WITH_400_STREAM(producer, producer_ro, producer, get_scheduled_protocol_feature_activations, - INVOKE_R_V(producer, get_scheduled_protocol_feature_activations), 201), - CALL_WITH_400_STREAM(producer, producer_ro, producer, get_supported_protocol_features, - INVOKE_R_R_II(producer, get_supported_protocol_features, - producer_plugin::get_supported_protocol_features_params), 201), - CALL_WITH_400_STREAM(producer, producer_ro, producer, get_unapplied_transactions, - INVOKE_R_R_D(producer, get_unapplied_transactions, producer_plugin::get_unapplied_transactions_params), 200), - CALL_WITH_400_STREAM(producer, producer_ro, producer, get_snapshot_requests, - INVOKE_R_V(producer, get_snapshot_requests), 201), + auto& _http_plugin = app().get_plugin(); + + using pp = producer_plugin; + using cat = api_category; + using pt = http_params_types; + + _http_plugin.add_api_stream({ + bind_stream<&pp::paused, dispatch::sync>( + _http_plugin, std::ref(producer), "/v1/producer/paused", cat::producer_ro, pt::no_params, 201), + bind_stream<&pp::get_runtime_options, dispatch::sync>( + _http_plugin, std::ref(producer), "/v1/producer/get_runtime_options", cat::producer_ro, pt::no_params, 201), + bind_stream<&pp::get_greylist, dispatch::sync>( + _http_plugin, std::ref(producer), "/v1/producer/get_greylist", cat::producer_ro, pt::no_params, 201), + bind_stream<&pp::get_whitelist_blacklist, dispatch::sync>( + _http_plugin, std::ref(producer), "/v1/producer/get_whitelist_blacklist", cat::producer_ro, pt::no_params, 201), + bind_stream<&pp::get_scheduled_protocol_feature_activations, dispatch::sync>( + _http_plugin, std::ref(producer), "/v1/producer/get_scheduled_protocol_feature_activations", + cat::producer_ro, pt::no_params, 201), + bind_stream<&pp::get_supported_protocol_features, dispatch::sync>( + _http_plugin, std::ref(producer), "/v1/producer/get_supported_protocol_features", + cat::producer_ro, pt::possible_no_params, 201), + bind_stream<&pp::get_unapplied_transactions, dispatch::sync>( + _http_plugin, std::ref(producer), "/v1/producer/get_unapplied_transactions", + cat::producer_ro, pt::possible_no_params, 200), + bind_stream<&pp::get_snapshot_requests, dispatch::sync>( + _http_plugin, std::ref(producer), "/v1/producer/get_snapshot_requests", + cat::producer_ro, pt::no_params, 201), }, appbase::exec_queue::read_only, appbase::priority::medium_high); // Not safe to run in parallel - app().get_plugin().add_api_stream({ - CALL_WITH_400_STREAM(producer, producer_rw, producer, pause, - INVOKE_V_V(producer, pause), 201), - CALL_WITH_400_STREAM(producer, producer_rw, producer, pause_at_block, - INVOKE_V_R(producer, pause_at_block, producer_plugin::pause_at_block_params), 201), - CALL_WITH_400_STREAM(producer, producer_rw, producer, resume, - INVOKE_V_V(producer, resume), 201), - CALL_WITH_400_STREAM(producer, producer_rw, producer, update_runtime_options, - INVOKE_V_R(producer, update_runtime_options, producer_plugin::runtime_options), 201), - CALL_WITH_400_STREAM(producer, producer_rw, producer, add_greylist_accounts, - INVOKE_V_R(producer, add_greylist_accounts, producer_plugin::greylist_params), 201), - CALL_WITH_400_STREAM(producer, producer_rw, producer, remove_greylist_accounts, - INVOKE_V_R(producer, remove_greylist_accounts, producer_plugin::greylist_params), 201), - CALL_WITH_400_STREAM(producer, producer_rw, producer, set_whitelist_blacklist, - INVOKE_V_R(producer, set_whitelist_blacklist, producer_plugin::whitelist_blacklist), 201), - CALL_ASYNC_STREAM(producer, snapshot, producer, create_snapshot, chain::snapshot_scheduler::snapshot_information, - INVOKE_R_V_ASYNC(producer, create_snapshot), 201), - CALL_WITH_400_STREAM(producer, snapshot, producer, schedule_snapshot, - INVOKE_R_R_II(producer, schedule_snapshot, chain::snapshot_scheduler::snapshot_request_params), 201), - CALL_WITH_400_STREAM(producer, snapshot, producer, unschedule_snapshot, - INVOKE_R_R(producer, unschedule_snapshot, chain::snapshot_scheduler::snapshot_request_id_information), 201), - CALL_WITH_400_STREAM(producer, producer_rw, producer, get_integrity_hash, - INVOKE_R_V(producer, get_integrity_hash), 201), - CALL_WITH_400_STREAM(producer, producer_rw, producer, schedule_protocol_feature_activations, - INVOKE_V_R(producer, schedule_protocol_feature_activations, producer_plugin::scheduled_protocol_feature_activations), 201), + _http_plugin.add_api_stream({ + bind_stream<&pp::pause, dispatch::sync_void>( + _http_plugin, std::ref(producer), "/v1/producer/pause", cat::producer_rw, pt::no_params, 201), + bind_stream<&pp::pause_at_block, dispatch::sync_void>( + _http_plugin, std::ref(producer), "/v1/producer/pause_at_block", cat::producer_rw, pt::params_required, 201), + bind_stream<&pp::resume, dispatch::sync_void>( + _http_plugin, std::ref(producer), "/v1/producer/resume", cat::producer_rw, pt::no_params, 201), + bind_stream<&pp::update_runtime_options, dispatch::sync_void>( + _http_plugin, std::ref(producer), "/v1/producer/update_runtime_options", + cat::producer_rw, pt::params_required, 201), + bind_stream<&pp::add_greylist_accounts, dispatch::sync_void>( + _http_plugin, std::ref(producer), "/v1/producer/add_greylist_accounts", + cat::producer_rw, pt::params_required, 201), + bind_stream<&pp::remove_greylist_accounts, dispatch::sync_void>( + _http_plugin, std::ref(producer), "/v1/producer/remove_greylist_accounts", + cat::producer_rw, pt::params_required, 201), + bind_stream<&pp::set_whitelist_blacklist, dispatch::sync_void>( + _http_plugin, std::ref(producer), "/v1/producer/set_whitelist_blacklist", + cat::producer_rw, pt::params_required, 201), + bind_stream<&pp::create_snapshot, dispatch::async>( + _http_plugin, std::ref(producer), "/v1/producer/create_snapshot", cat::snapshot, pt::no_params, 201), + bind_stream<&pp::schedule_snapshot, dispatch::sync>( + _http_plugin, std::ref(producer), "/v1/producer/schedule_snapshot", + cat::snapshot, pt::possible_no_params, 201), + bind_stream<&pp::unschedule_snapshot, dispatch::sync>( + _http_plugin, std::ref(producer), "/v1/producer/unschedule_snapshot", + cat::snapshot, pt::params_required, 201), + bind_stream<&pp::get_integrity_hash, dispatch::sync>( + _http_plugin, std::ref(producer), "/v1/producer/get_integrity_hash", + cat::producer_rw, pt::no_params, 201), + bind_stream<&pp::schedule_protocol_feature_activations, dispatch::sync_void>( + _http_plugin, std::ref(producer), "/v1/producer/schedule_protocol_feature_activations", + cat::producer_rw, pt::params_required, 201), }, appbase::exec_queue::read_write, appbase::priority::medium_high); } @@ -171,11 +109,4 @@ void producer_api_plugin::plugin_initialize(const variables_map& options) { } FC_LOG_AND_RETHROW() } - -#undef INVOKE_R_R -#undef INVOKE_R_V -#undef INVOKE_V_R -#undef INVOKE_V_V -#undef CALL - } From afaa4cc2f158974bfb83f7d4af69cd9661b38b94 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Tue, 28 Apr 2026 22:33:28 -0500 Subject: [PATCH 37/71] libfc: add to_variant overload + fix map_object_emission expectation Adds an explicit `to_variant` overload constrained on `T == bool` to `fc/variant.hpp`. Without it, once `fc/reflect/variant.hpp` is in scope (via `fc/reflect/json_stream.hpp` since b2bf4fa372), the reflected `to_variant` template becomes an exact match for `bool` and is selected by overload resolution, then fails compile on `fc::reflector::visit`. This broke `test_static_variant.cpp`'s round-trip case for `std::variant`. The overload is constrained via `requires std::is_same_v` rather than declared as `to_variant(bool, ...)` so that types implicitly convertible to bool (raw pointers, classes with `operator bool()`) do NOT silently route here -- they continue to fall through to the reflected path and produce a loud build error if no reflector exists. Also corrects the expectation in `test_json_stream/map_object_emission`: the test was authored expecting JSON-object form (`{"a":1,"b":2}`) but both `to_variant_from_map` and `to_json_stream_from_map` emit an array of [key, value] pairs (`[["a",1],["b",2]]`) -- this preserves round-trip for non-string key types and matches the long-standing fc::variant behaviour. The test never ran successfully because the same compile chain blocked test_fc's link. --- libraries/libfc/include/fc/variant.hpp | 14 ++++++++++++++ libraries/libfc/test/io/test_json_stream.cpp | 6 ++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/libraries/libfc/include/fc/variant.hpp b/libraries/libfc/include/fc/variant.hpp index dddc5c54d4..cb025d48c4 100644 --- a/libraries/libfc/include/fc/variant.hpp +++ b/libraries/libfc/include/fc/variant.hpp @@ -74,6 +74,20 @@ namespace fc template void to_variant( const std::variant& s, fc::variant& v ); template void from_variant( const fc::variant& v, std::variant& s ); + /// Explicit bool overload so std::variant elements of type bool route here, not into the + /// reflected `to_variant` template (which expects T to have an `FC_REFLECT`). Without + /// this overload the call resolves to the template once `fc/reflect/variant.hpp` is in + /// scope, then fails on `fc::reflector::visit`. + /// + /// Constrained on `T == bool` exactly so implicitly-convertible-to-bool types (raw + /// pointers, types with `operator bool()`, etc.) do NOT silently route here -- those + /// fall through to the reflected path and produce a loud build error if no reflector + /// exists. See `from_variant(variant&, bool&)` for the inverse (already type-safe by + /// virtue of taking `bool&`, which non-bools can't bind to). + template + requires std::is_same_v + void to_variant( const T& var, fc::variant& vo ) { vo = var; } + void to_variant( const uint8_t& var, fc::variant& vo ); void from_variant( const fc::variant& var, uint8_t& vo ); void to_variant( const int8_t& var, fc::variant& vo ); diff --git a/libraries/libfc/test/io/test_json_stream.cpp b/libraries/libfc/test/io/test_json_stream.cpp index 8f046bb591..4d29b31d48 100644 --- a/libraries/libfc/test/io/test_json_stream.cpp +++ b/libraries/libfc/test/io/test_json_stream.cpp @@ -133,8 +133,10 @@ BOOST_AUTO_TEST_CASE(map_object_emission) { with_map_t m; m.counts["a"] = 1; m.counts["b"] = 2; - // std::map iterates in sorted order so output is deterministic. - BOOST_CHECK_EQUAL(fc::to_json_string(m), "{\"counts\":{\"a\":1,\"b\":2}}"); + // std::map iterates in sorted order so output is deterministic. Maps serialize as an + // array of [key, value] pairs (matching the long-standing to_variant_from_map shape), + // not a JSON object -- this preserves round-trip support for non-string key types. + BOOST_CHECK_EQUAL(fc::to_json_string(m), "{\"counts\":[[\"a\",1],[\"b\",2]]}"); } BOOST_AUTO_TEST_CASE(raw_value_embeds_fragment) { From 174f80f390530982e7a6e368a137c84262ff36fe Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Tue, 28 Apr 2026 22:33:28 -0500 Subject: [PATCH 38/71] libfc-test: locate source root when this checkout is a git worktree `get_source_root_path()` walks parent directories looking for `.git` and treats it as the source root marker. In a regular clone, `.git` is a directory. In a git worktree, `.git` is a regular file containing a `gitdir: ` pointer to the linked repository. The previous check (`bfs::is_directory`) returned false for the worktree case, the loop walked past the source root, and the test fixture lookup threw `Unable to find source root directory`. Accept either shape as the marker. Affected tests on dev workstations using worktrees (e.g. rlp_encoder via test_ethereum_client_and_rlp.cpp); CI is unaffected since CI uses regular clones. --- libraries/libfc-test/src/build_info.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libraries/libfc-test/src/build_info.cpp b/libraries/libfc-test/src/build_info.cpp index 2b813b3539..6dece8c1ff 100644 --- a/libraries/libfc-test/src/build_info.cpp +++ b/libraries/libfc-test/src/build_info.cpp @@ -67,7 +67,10 @@ bfs::path get_source_root_path() { auto current_path = get_build_root_path(); while (current_path != current_path.root_path()) { - if (bfs::is_directory(current_path / ".git")) { + // .git is a directory in a normal clone, but a regular file (containing + // `gitdir: ...`) when this checkout is a git worktree -- accept either. + const auto git_marker = current_path / ".git"; + if (bfs::is_directory(git_marker) || bfs::is_regular_file(git_marker)) { source_root_path = current_path; return current_path; } From 57fff0f3d7937448b2696029457eb40ca1b253f7 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Wed, 29 Apr 2026 05:56:42 -0500 Subject: [PATCH 39/71] chain_plugin: defer abi parsing in get_block_stream off the read-only queue Splits get_block_stream's Phase 1 into a chainbase-only capture step (still on the read-only queue) and an abi-parse step that runs on the http thread pool. Phase 1 now does only the chainbase reads it actually requires: get_raw_block (thread-safe per controller, but invoked here while we hold shared chainbase access), and a new capture_abis() helper that copies each referenced account's raw `abi` byte string out of chainbase. ABI parsing (abi_serializer construction per unique account in the block) moves to Phase 2 in the http thread pool via build_resolver_from_captured_abis. Net effect on a request: - read-only queue holds the request only long enough for one block fetch plus the chainbase reads for unique accounts referenced in the block. - per-account abi_serializer construction (often the dominant CPU work when blocks reference many contracts with sizable ABIs) no longer blocks the read-only queue from servicing the next request. byte-for-byte response parity with get_block (variant) is preserved; the get_block_streaming_vs_variant_byte_identical test continues to pass. --- .../sysio/chain_plugin/chain_plugin.hpp | 92 ++++++++++++++++++- plugins/chain_plugin/src/chain_plugin.cpp | 26 ++++-- 2 files changed, 107 insertions(+), 11 deletions(-) diff --git a/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp b/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp index a560183168..30f90ddbf4 100644 --- a/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp +++ b/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp @@ -81,6 +81,85 @@ namespace sysio { return abi_resolver(abi_serializer_cache_builder(make_resolver(db, max_time, throw_on_yield::no)).add_serializers(obj).get()); } + /// Phase-split abi capture for the streaming-cb response path. `captured_abis` + /// holds the raw `abi` byte-string for every account referenced in a block / + /// trace, copied out of chainbase on the calling (read-only-queue) thread. + /// Parsing the bytes into `abi_serializer` instances is the slow part and + /// happens later in `build_resolver_from_captured_abis`, which can run on any + /// thread (typically the http thread pool) since it does not touch chainbase. + /// + /// Empty entries (`raw_abis[name] == {}`) record "we looked, no abi" so that + /// we don't re-walk chainbase off the read-only queue if the same account + /// appears multiple times. + struct captured_abis { + std::unordered_map> raw_abis; + }; + + /// Walks the actions in `obj` (a `signed_block_ptr` or `transaction_trace_ptr`) + /// and copies each unique action.account's `abi` bytes out of chainbase. + /// Must be called on a thread with shared chainbase access (read-only queue). + template + inline captured_abis capture_abis(const controller& control, const T& obj) { + captured_abis cache; + auto add = [&](const account_name& name) { + if (!name.good()) return; + if (cache.raw_abis.find(name) != cache.raw_abis.end()) return; + const auto* accnt = control.find_account_metadata(name); + if (accnt && accnt->abi.size() > 0) { + cache.raw_abis.emplace(name, + chain::vector{accnt->abi.data(), + accnt->abi.data() + accnt->abi.size()}); + } else { + cache.raw_abis.emplace(name, chain::vector{}); + } + }; + auto visit_actions = [&](const auto& actions) { + for (const auto& a : actions) add(a.account); + }; + if constexpr (std::is_same_v) { + for (const auto& receipt : obj->transactions) { + const auto& t = receipt.trx.get_transaction(); + visit_actions(t.actions); + visit_actions(t.context_free_actions); + } + } else { + // transaction_trace_ptr + for (const auto& trace : obj->action_traces) add(trace.act.account); + } + return cache; + } + + /// Parses each captured `abi` byte-string into an `abi_serializer` and returns + /// the resolver. CPU-bound; safe to run on any thread (no chainbase access). + /// On a per-account basis, an exception during `to_abi` / `abi_serializer` is + /// swallowed and the account's slot is left empty -- callers fall back to the + /// hex representation for actions with un-decodable ABI, matching the prior + /// `make_resolver(throw_on_yield::no)` behaviour. + inline abi_resolver build_resolver_from_captured_abis(captured_abis cache, + const fc::microseconds& max_time) { + chain::abi_serializer_cache_t serializers; + serializers.reserve(cache.raw_abis.size()); + for (auto& [name, raw] : cache.raw_abis) { + if (raw.empty()) { + serializers.emplace(name, std::nullopt); + continue; + } + try { + chain::abi_def abi; + if (chain::abi_serializer::to_abi(raw, abi)) { + serializers.emplace(name, + chain::abi_serializer(std::move(abi), + chain::abi_serializer::create_yield_function(max_time))); + } else { + serializers.emplace(name, std::nullopt); + } + } catch (...) { + serializers.emplace(name, std::nullopt); + } + } + return abi_resolver(std::move(serializers)); + } + namespace chain_apis { struct empty{}; @@ -386,12 +465,15 @@ class read_only : public api_base { std::function()> get_block(const get_block_params& params, const fc::time_point& deadline) const; /// Direct-streaming counterpart to `get_block`. Thread sequence: - /// - Phase 1 (main app thread, read_only queue): this method's body. `get_raw_block` is itself thread-safe; - /// `get_serializers_cache` is what *requires* the main thread because it reads each referenced account's `abi` - /// field out of chainbase. Returns the outer http_fwd closure. + /// - Phase 1 (main app thread, read_only queue): this method's body. Does *only* chainbase + /// reads -- `get_raw_block` (thread-safe per controller) and `capture_abis` (copies each + /// referenced account's raw `abi` bytes out of chainbase). ABI parsing is deferred so + /// the read-only queue isn't blocked on per-account `abi_serializer` construction. + /// Returns the outer http_fwd closure. /// - Hop 1: `bind_stream<&get_block_stream, dispatch::post_direct>` posts the outer closure to the http thread pool. - /// - Phase 2 (http thread pool): invokes the outer closure -- captured block + resolver snapshots, no chainbase - /// access -- and produces the inner `json_writer`-emitting closure, handed to `cb`. + /// - Phase 2 (http thread pool): invokes the outer closure, parses the captured abi bytes + /// into an `abi_resolver` via `build_resolver_from_captured_abis` (CPU-only, no chainbase + /// access), and produces the inner `json_writer`-emitting closure, handed to `cb`. /// - Hop 2: `make_http_stream_response_handler` re-dispatches onto the same http thread pool (inherited from the /// variant-cb `cb` shape, where Phase 2 does the row-decode work). /// - Phase 3 (http thread pool): walks the captured block via `abi_serializer::to_json_stream`, diff --git a/plugins/chain_plugin/src/chain_plugin.cpp b/plugins/chain_plugin/src/chain_plugin.cpp index e8a083baee..fdb1c67037 100644 --- a/plugins/chain_plugin/src/chain_plugin.cpp +++ b/plugins/chain_plugin/src/chain_plugin.cpp @@ -2790,17 +2790,31 @@ std::function()> read_only::get_block(const g read_only::get_block_stream_return_t read_only::get_block_stream(const get_block_params& params, const fc::time_point& deadline) const { + // Phase 1 (read-only queue, main thread): fetch the block and capture each + // referenced account's raw abi bytes out of chainbase. Both operations require + // shared chainbase access; everything else (abi parsing, block walk, JSON emit) + // is CPU work that runs later on the http thread pool. chain::signed_block_ptr block = get_raw_block(params, deadline); + captured_abis abi_bytes = capture_abis(db, block); using return_type = chain::t_or_exception; return [this, - resolver = get_serializers_cache(db, block, abi_serializer_max_time), - block = std::move(block)]() mutable -> return_type { + block = std::move(block), + abi_bytes = std::move(abi_bytes), + max_time = abi_serializer_max_time]() mutable -> return_type { try { - return get_block_stream_emit_fn{[this, resolver = std::move(resolver), block = std::move(block)] - (fc::json_writer& w) mutable { - convert_block_stream(block, resolver, w); - }}; + // Phase 2 (http thread pool): parse the captured abi bytes into an + // abi_resolver. This is the CPU-heavy step (one abi_serializer build + // per unique account in the block); doing it here instead of in Phase 1 + // keeps the read-only queue free for the next request. + abi_resolver resolver = build_resolver_from_captured_abis(std::move(abi_bytes), max_time); + return get_block_stream_emit_fn{ + [this, resolver = std::move(resolver), block = std::move(block)] + (fc::json_writer& w) mutable { + // Phase 3 (http thread pool): walk the block, decode each action's + // data via the resolver, and emit the JSON response directly into w. + convert_block_stream(block, resolver, w); + }}; } CATCH_AND_RETURN(return_type); }; } From a2487db6da70f785337b281b44bc87d74a84c7ae Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Wed, 29 Apr 2026 06:56:47 -0500 Subject: [PATCH 40/71] chain_plugin / http_plugin: fix misleading "Hop 2" comments -- asio dispatch already inlines The threading comment on `read_only::get_block_stream` and the cb body in `make_http_stream_response_handler` described `boost::asio::dispatch` to the http thread pool as a real second hop. It isn't: boost/asio/impl/thread_pool.hpp `do_execute` short-circuits to an inline call ("Invoke immediately if the blocking.possibly property is enabled and we are already inside the thread pool") when the calling thread is a pool worker. Phase 2 of `dispatch::post_direct` is invoked from a `post_http_thread_pool` task, so it always already is. No code change to behaviour -- this is comment-only, recording the optimisation that asio performs so future readers don't see "Hop 2" and think there's a redundant post to eliminate. --- .../include/sysio/chain_plugin/chain_plugin.hpp | 7 +++++-- .../http_plugin/include/sysio/http_plugin/common.hpp | 10 ++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp b/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp index 30f90ddbf4..c7dd71c969 100644 --- a/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp +++ b/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp @@ -474,8 +474,11 @@ class read_only : public api_base { /// - Phase 2 (http thread pool): invokes the outer closure, parses the captured abi bytes /// into an `abi_resolver` via `build_resolver_from_captured_abis` (CPU-only, no chainbase /// access), and produces the inner `json_writer`-emitting closure, handed to `cb`. - /// - Hop 2: `make_http_stream_response_handler` re-dispatches onto the same http thread pool (inherited from the - /// variant-cb `cb` shape, where Phase 2 does the row-decode work). + /// - cb (`make_http_stream_response_handler`) calls `boost::asio::dispatch` to the same + /// http thread pool. Asio's `thread_pool::executor::do_execute` short-circuits this + /// to an inline call when the calling thread is already a pool worker (which it is + /// here, since we're inside the Phase 2 task posted via `post_http_thread_pool`). + /// Net effect: no extra task post -- Phase 3 runs in the same task as Phase 2. /// - Phase 3 (http thread pool): walks the captured block via `abi_serializer::to_json_stream`, /// decoding each action's `data` straight into the writer with `binary_to_json_stream`. No `fc::variant` tree /// is built per action. diff --git a/plugins/http_plugin/include/sysio/http_plugin/common.hpp b/plugins/http_plugin/include/sysio/http_plugin/common.hpp index 678b230b6c..772c9d685f 100644 --- a/plugins/http_plugin/include/sysio/http_plugin/common.hpp +++ b/plugins/http_plugin/include/sysio/http_plugin/common.hpp @@ -242,6 +242,16 @@ inline auto make_http_stream_response_handler(http_plugin_state& plugin_state, d [&plugin_state, session_ptr{std::move(session_ptr)}] (int code, stream_emitter emitter) mutable { + // Note on threading: boost::asio::dispatch on a thread_pool executor runs + // the function inline when called from a thread already inside the pool + // (impl/thread_pool.hpp `do_execute`: "Invoke immediately if the + // blocking.possibly property is enabled and we are already inside the + // thread pool"). So when this cb is fired from a `post_http_thread_pool` + // task -- e.g. the Phase 2 closure of a `dispatch::post_direct` registration + // -- the dispatch below is a function-call short-circuit, not a real post. + // For sync / sync_void / async-immediate paths (cb invoked on read-only + // or read-write queue), it correctly bounces JSON serialization + socket + // I/O onto the http thread pool. boost::asio::dispatch(plugin_state.thread_pool.get_executor(), [&plugin_state, session_ptr{std::move(session_ptr)}, code, emitter{std::move(emitter)}]() mutable { try { From e65ce392d4ae1d1c7e911153703c3544b230d2f9 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Wed, 29 Apr 2026 08:40:27 -0500 Subject: [PATCH 41/71] chain_plugin: move get_raw_block off the read-only queue via dispatch::async Migrates `get_block` from `dispatch::post_direct` (read-only queue runs both the block fetch and the abi capture) to `dispatch::async` plus a small choreography in the api method. New flow: - Phase 0 on the http thread pool: `get_raw_block` (thread-safe per controller; uses fork_db / blocks_log internal locks rather than chainbase, so it can proceed even while a block is being applied). - Hop 1: post onto exec_queue::read_only at medium_low priority. - Phase 1 on the read-only queue: `capture_abis(block)` copies each unique referenced account's raw abi bytes; fires next() with an http_fwd Phase-2 closure. - Hop 2: bind_stream::async posts http_fwd back to the http pool. - Phase 2 on the http pool: parse the captured bytes -> abi_resolver, construct emit_fn. - Phase 3 on the http pool (asio-inlined): emit_fn walks the block and writes JSON via convert_block_stream. Adds a `to_json_stream(const std::function&, ...)` overload in fc/io/json_stream.hpp -- the streaming machinery's typed-T path goes through `to_json_stream(payload, w)`, and for our payload (`emit_fn = function`) that just invokes the closure. This lets the new method use the existing `dispatch::async` path with no new dispatch kind. Removes the now-dead `read_only::get_block_stream` (the `dispatch::post_direct` path it replaces) and the `read_only::get_block` variant path (already dead since the streaming migration); `convert_block` and `convert_block_stream` stay -- both are still used by `chain_plugin_tests` / `test_chain_plugin`. Also tightens earlier comments that referred to "shared chainbase access" / "shared chainbase lock"; wire-sysio doesn't have a chainbase-level shared lock -- read serialization is the appbase exec_queue mechanism. --- libraries/libfc/include/fc/io/json_stream.hpp | 11 +++ .../chain_api_plugin/src/chain_api_plugin.cpp | 7 +- .../sysio/chain_plugin/chain_plugin.hpp | 41 ++++----- plugins/chain_plugin/src/chain_plugin.cpp | 83 +++++++++---------- 4 files changed, 74 insertions(+), 68 deletions(-) diff --git a/libraries/libfc/include/fc/io/json_stream.hpp b/libraries/libfc/include/fc/io/json_stream.hpp index cb704950ee..96be78edc7 100644 --- a/libraries/libfc/include/fc/io/json_stream.hpp +++ b/libraries/libfc/include/fc/io/json_stream.hpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -225,4 +226,14 @@ class json_writer { bool awaiting_value_ = false; }; +/// to_json_stream overload for an emit-closure: a `std::function` +/// whose JSON output is whatever the closure writes when invoked. Used by callers +/// that want to deliver a streaming response through machinery (eg `bind_stream`'s +/// async/typed-T paths) that wraps the payload in `to_json_stream(payload, w)` -- +/// the closure itself is the emitter, so we just invoke it. +inline void to_json_stream(const std::function& fn, json_writer& w) { + if (fn) fn(w); + else w.value_null(); +} + } // namespace fc diff --git a/plugins/chain_api_plugin/src/chain_api_plugin.cpp b/plugins/chain_api_plugin/src/chain_api_plugin.cpp index 4716ba7037..0c4380f720 100644 --- a/plugins/chain_api_plugin/src/chain_api_plugin.cpp +++ b/plugins/chain_api_plugin/src/chain_api_plugin.cpp @@ -130,9 +130,6 @@ void chain_api_plugin::plugin_startup() { _http_plugin.add_api_stream({ bind_stream<&ro::get_activated_protocol_features, dispatch::sync>( _http_plugin, ro_api, "/v1/chain/get_activated_protocol_features", cat::chain_ro, pt::possible_no_params, 200), - // get_block_stream returns Phase-2 closure (function()>) - bind_stream<&ro::get_block_stream, dispatch::post_direct>( - _http_plugin, ro_api, "/v1/chain/get_block", cat::chain_ro, pt::params_required, 200), bind_stream<&ro::get_block_info, dispatch::sync>( _http_plugin, ro_api, "/v1/chain/get_block_info", cat::chain_ro, pt::params_required, 200), bind_stream<&ro::get_block_header_state, dispatch::sync>( @@ -185,6 +182,10 @@ void chain_api_plugin::plugin_startup() { // chain_plugin send_read_only_transaction will post to read_exclusive queue bind_stream<&ro::send_read_only_transaction, dispatch::async>( _http_plugin, ro_api, "/v1/chain/send_read_only_transaction", cat::chain_ro, pt::params_required, 200), + // get_block runs Phase 0 (block fetch) on the http pool then bounces to the read-only + // queue internally for the abi capture; see read_only::get_block_stream_async. + bind_stream<&ro::get_block_stream_async, dispatch::async>( + _http_plugin, ro_api, "/v1/chain/get_block", cat::chain_ro, pt::params_required, 200), bind_stream<&ro::get_raw_block, dispatch::sync>( _http_plugin, ro_api, "/v1/chain/get_raw_block", cat::chain_ro, pt::params_required, 200), bind_stream<&ro::get_block_header, dispatch::sync>( diff --git a/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp b/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp index c7dd71c969..61fe1639bc 100644 --- a/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp +++ b/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp @@ -97,7 +97,8 @@ namespace sysio { /// Walks the actions in `obj` (a `signed_block_ptr` or `transaction_trace_ptr`) /// and copies each unique action.account's `abi` bytes out of chainbase. - /// Must be called on a thread with shared chainbase access (read-only queue). + /// Must be called from an `exec_queue::read_only` (or `read_write`) task so + /// that chainbase is not being mutated concurrently. template inline captured_abis capture_abis(const controller& control, const T& obj) { captured_abis cache; @@ -462,29 +463,23 @@ class read_only : public api_base { chain::signed_block_ptr get_raw_block(const get_raw_block_params& params, const fc::time_point& deadline) const; using get_block_params = get_raw_block_params; - std::function()> get_block(const get_block_params& params, const fc::time_point& deadline) const; - - /// Direct-streaming counterpart to `get_block`. Thread sequence: - /// - Phase 1 (main app thread, read_only queue): this method's body. Does *only* chainbase - /// reads -- `get_raw_block` (thread-safe per controller) and `capture_abis` (copies each - /// referenced account's raw `abi` bytes out of chainbase). ABI parsing is deferred so - /// the read-only queue isn't blocked on per-account `abi_serializer` construction. - /// Returns the outer http_fwd closure. - /// - Hop 1: `bind_stream<&get_block_stream, dispatch::post_direct>` posts the outer closure to the http thread pool. - /// - Phase 2 (http thread pool): invokes the outer closure, parses the captured abi bytes - /// into an `abi_resolver` via `build_resolver_from_captured_abis` (CPU-only, no chainbase - /// access), and produces the inner `json_writer`-emitting closure, handed to `cb`. - /// - cb (`make_http_stream_response_handler`) calls `boost::asio::dispatch` to the same - /// http thread pool. Asio's `thread_pool::executor::do_execute` short-circuits this - /// to an inline call when the calling thread is already a pool worker (which it is - /// here, since we're inside the Phase 2 task posted via `post_http_thread_pool`). - /// Net effect: no extra task post -- Phase 3 runs in the same task as Phase 2. - /// - Phase 3 (http thread pool): walks the captured block via `abi_serializer::to_json_stream`, - /// decoding each action's `data` straight into the writer with `binary_to_json_stream`. No `fc::variant` tree - /// is built per action. + using get_block_stream_emit_fn = std::function; - using get_block_stream_return_t = std::function()>; - get_block_stream_return_t get_block_stream(const get_block_params& params, const fc::time_point& deadline) const; + + /// Async streaming `/v1/chain/get_block` handler. Runs `get_raw_block` on the http thread pool instead of the + /// read-only queue. Threading sequence: + /// - Phase 0 (http thread pool): registration uses `add_async_api_stream`, so the api method body runs on the + /// http worker that handled the request. Does only `get_raw_block`, which is thread-safe per controller. + /// - Hop 1: the method posts onto `exec_queue::read_only` for the chainbase abi reads. + /// - Phase 1 (read-only queue): `capture_abis(block)` copies each unique account's raw `abi` bytes, then fires + /// the next callback with an http_fwd Phase-2 closure. + /// - Hop 2: bind_stream::async posts the http_fwd back to the http thread pool. + /// - Phase 2 (http thread pool): `build_resolver_from_captured_abis` parses the bytes (CPU-only) and constructs + /// the emit_fn. + /// - Phase 3 (http thread pool, asio-inlined): bind_stream wraps the emit_fn in a `to_json_stream(emit_fn, w)` + /// adapter and hands it to `cb`; emit_fn walks the block + emits JSON. + void get_block_stream_async(get_block_params params, + chain::next_function next) const; // call from app() thread abi_resolver get_block_serializers( const chain::signed_block_ptr& block, const fc::microseconds& max_time ) const; diff --git a/plugins/chain_plugin/src/chain_plugin.cpp b/plugins/chain_plugin/src/chain_plugin.cpp index fdb1c67037..5c74c1a914 100644 --- a/plugins/chain_plugin/src/chain_plugin.cpp +++ b/plugins/chain_plugin/src/chain_plugin.cpp @@ -2775,48 +2775,47 @@ chain::signed_block_ptr read_only::get_raw_block(const read_only::get_raw_block_ return block; } -std::function()> read_only::get_block(const get_raw_block_params& params, const fc::time_point& deadline) const { - chain::signed_block_ptr block = get_raw_block(params, deadline); - - using return_type = t_or_exception; - return [this, - resolver = get_serializers_cache(db, block, abi_serializer_max_time), - block = std::move(block)]() mutable -> return_type { - try { - return convert_block(block, resolver); - } CATCH_AND_RETURN(return_type); - }; -} - -read_only::get_block_stream_return_t -read_only::get_block_stream(const get_block_params& params, const fc::time_point& deadline) const { - // Phase 1 (read-only queue, main thread): fetch the block and capture each - // referenced account's raw abi bytes out of chainbase. Both operations require - // shared chainbase access; everything else (abi parsing, block walk, JSON emit) - // is CPU work that runs later on the http thread pool. - chain::signed_block_ptr block = get_raw_block(params, deadline); - captured_abis abi_bytes = capture_abis(db, block); - - using return_type = chain::t_or_exception; - return [this, - block = std::move(block), - abi_bytes = std::move(abi_bytes), - max_time = abi_serializer_max_time]() mutable -> return_type { - try { - // Phase 2 (http thread pool): parse the captured abi bytes into an - // abi_resolver. This is the CPU-heavy step (one abi_serializer build - // per unique account in the block); doing it here instead of in Phase 1 - // keeps the read-only queue free for the next request. - abi_resolver resolver = build_resolver_from_captured_abis(std::move(abi_bytes), max_time); - return get_block_stream_emit_fn{ - [this, resolver = std::move(resolver), block = std::move(block)] - (fc::json_writer& w) mutable { - // Phase 3 (http thread pool): walk the block, decode each action's - // data via the resolver, and emit the JSON response directly into w. - convert_block_stream(block, resolver, w); - }}; - } CATCH_AND_RETURN(return_type); - }; +void read_only::get_block_stream_async(get_block_params params, + chain::next_function next) const { + try { + // Phase 0 (http thread pool): block fetch. get_raw_block is thread-safe per + // controller. The deadline argument is unused by the current get_raw_block + // body; pass time_point::maximum so it's a no-op. + chain::signed_block_ptr block = get_raw_block(params, fc::time_point::maximum()); + + // Hop 1: post the chainbase abi capture onto the read-only queue. Phase 2 + // (parse + emit_fn build) returns to the http pool via the http_fwd alternative + // of next_function_variant. + app().executor().post(appbase::priority::medium_low, + appbase::exec_queue::read_only, + [this, block = std::move(block), + max_time = abi_serializer_max_time, + next = std::move(next)]() mutable { + try { + captured_abis abi_bytes = capture_abis(db, block); + + // Fire next with an http_fwd that builds Phase 2 + the emit_fn on + // the http pool. bind_stream::async sees the http_fwd alternative + // and posts it via post_http_thread_pool(). + next(std::function()>{ + [this, block = std::move(block), + abi_bytes = std::move(abi_bytes), + max_time]() mutable + -> chain::t_or_exception { + try { + auto resolver = build_resolver_from_captured_abis(std::move(abi_bytes), + max_time); + return get_block_stream_emit_fn{ + [this, block = std::move(block), + resolver = std::move(resolver)] + (fc::json_writer& w) mutable { + convert_block_stream(block, resolver, w); + }}; + } CATCH_AND_RETURN(chain::t_or_exception); + }}); + } CATCH_AND_CALL(next); + }); + } CATCH_AND_CALL(next); } read_only::get_block_header_result read_only::get_block_header(const read_only::get_block_header_params& params, const fc::time_point& deadline) const{ From be01f114176b2af15370e1c8648bc91d8982bef6 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Wed, 29 Apr 2026 09:25:40 -0500 Subject: [PATCH 42/71] chain_plugin: get_block_info async + typed result; defer ROA abi parse off read-only in get_account get_block_info - Migrates `/v1/chain/get_block_info` off the read-only queue to the http thread pool, and switches the response shape from `fc::variant` to a typed `get_block_info_results` struct emitted via to_json_stream. - `fetch_block_header_by_number` is thread-safe per controller, and the rest of the body (id calc + struct build) is CPU-only. Registration moves from `add_api_stream(read_only) + dispatch::sync` to `add_async_api_stream + dispatch::async`; the api method body now runs on the http worker that handled the request -- no read-only queue posting. - Skipping the variant tree means each field is written straight to the json_writer; field order in `FC_REFLECT(get_block_info_results, ...)` matches the previous `mutable_variant_object` insertion order so output is byte-identical. get_account - Mirrors the get_block_stream phase split: Phase 1 captures the ROA contract's raw `abi` bytes (chainbase read), Phase 2 on the http thread pool runs both `abi_serializer::to_abi` (deserialize) and the `abi_serializer` construction. The to_abi parse, which previously ran on the read-only queue, is now off it. - Phase 1 gates on `code_account != nullptr && code_account->abi.size() > 0` instead of "abi parses". If the ROA abi is corrupt, Phase 2's `to_abi` returns false and we skip the reslimit decode (leaving total_resources unset). The pre-refactor code skipped the kv lookups entirely in that case; the observable difference is that result.core_liquid_balance is now populated when the ROA contract has bytes but a corrupt abi. Essentially a fix -- core_liquid_balance comes from sysio.token (typed unpack), independent of the ROA abi -- but it is a behaviour change in that edge case. Both endpoints' streaming-vs-variant byte-parity tests continue to pass. --- .../chain_api_plugin/src/chain_api_plugin.cpp | 6 +- .../sysio/chain_plugin/chain_plugin.hpp | 26 ++++- plugins/chain_plugin/src/chain_plugin.cpp | 95 ++++++++++++------- 3 files changed, 90 insertions(+), 37 deletions(-) diff --git a/plugins/chain_api_plugin/src/chain_api_plugin.cpp b/plugins/chain_api_plugin/src/chain_api_plugin.cpp index 0c4380f720..7491b10616 100644 --- a/plugins/chain_api_plugin/src/chain_api_plugin.cpp +++ b/plugins/chain_api_plugin/src/chain_api_plugin.cpp @@ -130,8 +130,6 @@ void chain_api_plugin::plugin_startup() { _http_plugin.add_api_stream({ bind_stream<&ro::get_activated_protocol_features, dispatch::sync>( _http_plugin, ro_api, "/v1/chain/get_activated_protocol_features", cat::chain_ro, pt::possible_no_params, 200), - bind_stream<&ro::get_block_info, dispatch::sync>( - _http_plugin, ro_api, "/v1/chain/get_block_info", cat::chain_ro, pt::params_required, 200), bind_stream<&ro::get_block_header_state, dispatch::sync>( _http_plugin, ro_api, "/v1/chain/get_block_header_state", cat::chain_ro, pt::params_required, 200), bind_stream<&ro::get_code, dispatch::sync>( @@ -186,6 +184,10 @@ void chain_api_plugin::plugin_startup() { // queue internally for the abi capture; see read_only::get_block_stream_async. bind_stream<&ro::get_block_stream_async, dispatch::async>( _http_plugin, ro_api, "/v1/chain/get_block", cat::chain_ro, pt::params_required, 200), + // get_block_info is fully off the read-only queue: fetch_block_header_by_number is + // thread-safe and the rest is CPU; see read_only::get_block_info_async. + bind_stream<&ro::get_block_info_async, dispatch::async>( + _http_plugin, ro_api, "/v1/chain/get_block_info", cat::chain_ro, pt::params_required, 200), bind_stream<&ro::get_raw_block, dispatch::sync>( _http_plugin, ro_api, "/v1/chain/get_raw_block", cat::chain_ro, pt::params_required, 200), bind_stream<&ro::get_block_header, dispatch::sync>( diff --git a/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp b/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp index 61fe1639bc..a43c80e399 100644 --- a/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp +++ b/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp @@ -507,7 +507,27 @@ class read_only : public api_base { uint32_t block_num = 0; }; - fc::variant get_block_info(const get_block_info_params& params, const fc::time_point& deadline) const; + /// Typed result for `/v1/chain/get_block_info`. Field order matches the previous mutable_variant_object + /// shape so the streaming JSON output is byte-identical to the prior variant path. + struct get_block_info_results { + uint32_t block_num = 0; + uint16_t ref_block_num = 0; + chain::block_id_type id; + chain::block_timestamp_type timestamp; + chain::account_name producer; + chain::block_id_type previous; + chain::checksum256_type transaction_mroot; + chain::checksum256_type finality_mroot; + chain::qc_claim_t qc_claim; + std::vector producer_signatures; + uint32_t ref_block_prefix = 0; + }; + + /// `/v1/chain/get_block_info` -- async streaming so the body runs on the http thread pool, not the read-only queue. + /// `fetch_block_header_by_number` is thread-safe per controller and the rest is just CPU (id calc + struct build), + /// so the entire body is safe off the read-only queue. + void get_block_info_async(get_block_info_params params, + chain::next_function next) const; struct get_block_header_state_params { string block_num_or_id; @@ -859,6 +879,10 @@ FC_REFLECT(sysio::chain_apis::read_only::get_activated_protocol_features_params, FC_REFLECT(sysio::chain_apis::read_only::get_activated_protocol_features_results, (activated_protocol_features)(more) ) FC_REFLECT(sysio::chain_apis::read_only::get_raw_block_params, (block_num_or_id)) FC_REFLECT(sysio::chain_apis::read_only::get_block_info_params, (block_num)) +FC_REFLECT(sysio::chain_apis::read_only::get_block_info_results, + (block_num)(ref_block_num)(id)(timestamp)(producer)(previous) + (transaction_mroot)(finality_mroot)(qc_claim) + (producer_signatures)(ref_block_prefix)) FC_REFLECT(sysio::chain_apis::read_only::get_block_header_state_params, (block_num_or_id)) FC_REFLECT(sysio::chain_apis::read_only::get_block_header_params, (block_num_or_id)(include_extensions)) FC_REFLECT(sysio::chain_apis::read_only::get_block_header_result, (id)(signed_block_header)(block_extensions)) diff --git a/plugins/chain_plugin/src/chain_plugin.cpp b/plugins/chain_plugin/src/chain_plugin.cpp index 5c74c1a914..802f731571 100644 --- a/plugins/chain_plugin/src/chain_plugin.cpp +++ b/plugins/chain_plugin/src/chain_plugin.cpp @@ -2892,32 +2892,38 @@ void read_only::convert_block_stream( const chain::signed_block_ptr& block, abi_ sink.end_object(); } -fc::variant read_only::get_block_info(const read_only::get_block_info_params& params, const fc::time_point&) const { - - std::optional block; +void read_only::get_block_info_async(get_block_info_params params, + chain::next_function next) const { try { - block = db.fetch_block_header_by_number( params.block_num ); - } catch (...) { - // assert below will handle the invalid block num - } - - SYS_ASSERT( block, unknown_block_exception, "Could not find block: {}", params.block_num); - - const auto id = block->calculate_id(); - const uint32_t ref_block_prefix = id._hash[1]; - - return fc::mutable_variant_object () - ("block_num", block->block_num()) - ("ref_block_num", static_cast(block->block_num())) - ("id", id) - ("timestamp", block->timestamp) - ("producer", block->producer) - ("previous", block->previous) - ("transaction_mroot", block->transaction_mroot) - ("finality_mroot", block->finality_mroot) - ("qc_claim", block->qc_claim) - ("producer_signatures", block->producer_signatures) - ("ref_block_prefix", ref_block_prefix); + // Runs on the http thread pool (registered via add_async_api_stream). + // fetch_block_header_by_number is thread-safe per controller, so the entire + // body stays off the read-only queue. + std::optional block; + try { + block = db.fetch_block_header_by_number(params.block_num); + } catch (...) { + // assert below will handle the invalid block num + } + + SYS_ASSERT(block, unknown_block_exception, "Could not find block: {}", params.block_num); + + const auto id = block->calculate_id(); + const uint32_t ref_block_prefix = id._hash[1]; + + next(get_block_info_results{ + .block_num = block->block_num(), + .ref_block_num = static_cast(block->block_num()), + .id = id, + .timestamp = block->timestamp, + .producer = block->producer, + .previous = block->previous, + .transaction_mroot = block->transaction_mroot, + .finality_mroot = block->finality_mroot, + .qc_claim = block->qc_claim, + .producer_signatures = block->producer_signatures, + .ref_block_prefix = ref_block_prefix, + }); + } CATCH_AND_CALL(next); } fc::variant read_only::get_block_header_state(const get_block_header_state_params& params, const fc::time_point&) const { @@ -3353,7 +3359,11 @@ read_only::get_account_return_t read_only::get_account( const get_account_params }; http_params_t http_params; - if( abi_def abi; code_account != nullptr && abi_serializer::to_abi(code_account->abi, abi) ) { + // Phase 1 gate: ROA contract loaded with at least some abi bytes. We don't parse the + // abi here -- to_abi (deserialize) and abi_serializer construction both move to Phase 2 + // so the read-only queue isn't blocked on the ROA abi parse. The content lookups below + // (sysio.token balance + reslimit row) only need the raw bytes plus chainbase access. + if (code_account != nullptr && code_account->abi.size() > 0) { const auto token_code = "sysio.token"_n; @@ -3392,16 +3402,33 @@ read_only::get_account_return_t read_only::get_account( const get_account_params return {}; }; - http_params.total_resources = lookup_object("reslimit"_n, params.account_name); + http_params.total_resources = lookup_object("reslimit"_n, params.account_name); - return [http_params = std::move(http_params), result = std::move(result), abi=std::move(abi), shorten_abi_errors=shorten_abi_errors, - abi_serializer_max_time=abi_serializer_max_time]() mutable -> chain::t_or_exception { - auto yield = [&]() { return abi_serializer::create_yield_function(abi_serializer_max_time); }; - abi_serializer abis(std::move(abi), yield()); + // Capture raw ABI bytes for Phase 2; both to_abi (deserialize) and abi_serializer + // construction will run on the http thread pool. If parsing fails there, we skip + // the reslimit decode and return result with total_resources unset (preserving + // pre-refactor behaviour for the corrupt-abi case). + vector abi_bytes(code_account->abi.data(), + code_account->abi.data() + code_account->abi.size()); - if (http_params.total_resources) - result.total_resources = abis.binary_to_variant("reslimit", *http_params.total_resources, yield(), shorten_abi_errors); - return std::move(result); + return [http_params = std::move(http_params), result = std::move(result), + abi_bytes = std::move(abi_bytes), + shorten_abi_errors = shorten_abi_errors, + abi_serializer_max_time = abi_serializer_max_time]() mutable + -> chain::t_or_exception { + try { + abi_def abi; + if (abi_serializer::to_abi(abi_bytes, abi)) { + auto yield = [&]() { return abi_serializer::create_yield_function(abi_serializer_max_time); }; + abi_serializer abis(std::move(abi), yield()); + if (http_params.total_resources) { + result.total_resources = abis.binary_to_variant("reslimit", + *http_params.total_resources, + yield(), shorten_abi_errors); + } + } + return std::move(result); + } CATCH_AND_RETURN(chain::t_or_exception); }; } return [result = std::move(result)]() mutable -> chain::t_or_exception { From 48d536fa8cde07d61615a7c30ce3201115100873 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Wed, 29 Apr 2026 09:51:37 -0500 Subject: [PATCH 43/71] benchmark: add abi_serializer scenarios to the unified benchmark harness Adds an `abi_serializer` benchmarking feature to the existing /benchmark harness so the variant-vs-streaming JSON paths can be compared on the same build. Scenarios: - construct token / system abi_serializer (parse + setup cost) - decode transfer / newaccount / regproducer (variant path, steady state) - roundtrip transfer (pack + unpack) - resolve_type (typedef chain walk) - transfer / newaccount / regproducer (variant) vs (stream) json (per-action decode + JSON emit; isolates the variant-tree-vs-streaming gap) - block (4 transfers) (variant) vs (stream) json (uses abi_serializer::to_variant vs to_json_stream; mirrors chain_plugin::convert_block / convert_block_stream) - get_block-shape (variant + abi build) vs (stream + abi build) json (rebuilds the abi_serializer per iteration; matches the per-request cost of read_only::get_block_stream_async's Phase 2 + Phase 3) Run: ninja -C cmake-build-release benchmark ./cmake-build-release/benchmark/benchmark -f abi_serializer -r 10 Pulls token + system ABI fixture paths in as compile defines so the benchmark works regardless of build state (it reads the source-tree .abi files, not built artifacts). --- benchmark/CMakeLists.txt | 8 + benchmark/abi_serializer.cpp | 344 +++++++++++++++++++++++++++++++++++ benchmark/benchmark.cpp | 3 +- benchmark/benchmark.hpp | 1 + 4 files changed, 355 insertions(+), 1 deletion(-) create mode 100644 benchmark/abi_serializer.cpp diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index 9ec64101a1..d646b6626d 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -9,3 +9,11 @@ target_include_directories( benchmark PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/../unittests/include" ) + +# ABI fixture paths consumed by abi_serializer.cpp. These point at the system-contract +# sources, not the built artifacts, so the benchmark works regardless of build state. +target_compile_definitions( benchmark + PRIVATE + TOKEN_ABI_PATH="${CMAKE_SOURCE_DIR}/contracts/sysio.token/sysio.token.abi" + SYSTEM_ABI_PATH="${CMAKE_SOURCE_DIR}/contracts/sysio.system/sysio.system.abi" +) diff --git a/benchmark/abi_serializer.cpp b/benchmark/abi_serializer.cpp new file mode 100644 index 0000000000..675ffd80f8 --- /dev/null +++ b/benchmark/abi_serializer.cpp @@ -0,0 +1,344 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +// Microbenchmark for sysio::chain::abi_serializer. +// +// Measures construction cost and steady-state decode throughput so that +// successive optimization commits can report a verifiable before/after +// delta. NOT a correctness test -- abi_tests.cpp covers that. +// +// Run via the unified benchmark harness: +// +// ninja -C cmake-build-release -j8 benchmark +// ./cmake-build-release/benchmark/benchmark abi_serializer --num-runs 10 +// +// Release build (`-DCMAKE_BUILD_TYPE=Release`, `-O3`) is required for +// comparable numbers; debug / RelWithDebInfo timings are dominated by +// instrumentation. Each scenario does its own inner loop sized to keep +// per-call overhead amortised; the harness's `--num-runs` controls how +// many times the outer scenario is repeated for variance reporting. + +namespace sysio::benchmark { + +using namespace sysio::chain; +using bytes = sysio::benchmark::bytes; + +namespace { + +constexpr fc::microseconds max_serialization_time = fc::microseconds{1'000'000}; + +abi_serializer::yield_function_t yield_fn() { + return abi_serializer::create_yield_function(max_serialization_time); +} + +std::string slurp(const std::filesystem::path& p) { + std::ifstream f(p); + if (!f) { + std::cerr << "abi_serializer benchmark: failed to open " << p << "\n"; + std::exit(1); + } + std::stringstream ss; + ss << f.rdbuf(); + return ss.str(); +} + +} // namespace + +void abi_serializer_benchmarking() { + const std::filesystem::path token_path = TOKEN_ABI_PATH; + const std::filesystem::path system_path = SYSTEM_ABI_PATH; + + const auto token_abi = fc::json::from_string(slurp(token_path)).as(); + const auto system_abi = fc::json::from_string(slurp(system_path)).as(); + + // Prebuild serializers + packed payloads for the steady-state decode + // scenarios. Cost of these setup steps is NOT in the measured region. + abi_serializer token_abis{abi_def(token_abi), yield_fn()}; + abi_serializer system_abis{abi_def(system_abi), yield_fn()}; + + const auto transfer_var = fc::json::from_string( + R"({"from":"alice","to":"bob","quantity":"1.0000 SYS","memo":"benchmark payment"})"); + chain::bytes transfer_bytes; + try { + transfer_bytes = token_abis.variant_to_binary("transfer", transfer_var, yield_fn()); + } catch (const std::exception& e) { + std::cerr << "abi_serializer benchmark: transfer pack failed: " << e.what() << "\n"; + return; + } + + // newaccount exercises a nested struct path (authority sub-struct + array + // of key_weights). An authority with one key is enough to cover the + // nested+array code paths without bloating the fixture. + const auto newaccount_var = fc::json::from_string( + R"({"creator":"sysio","name":"alice","owner":)" + R"({"threshold":1,"keys":[{"key":"SYS1111111111111111111111111111111114T1Anm","weight":1}],)" + R"("accounts":[],"waits":[]},"active":)" + R"({"threshold":1,"keys":[{"key":"SYS1111111111111111111111111111111114T1Anm","weight":1}],)" + R"("accounts":[],"waits":[]}})"); + chain::bytes newaccount_bytes; + try { + newaccount_bytes = system_abis.variant_to_binary("newaccount", newaccount_var, yield_fn()); + } catch (const std::exception& e) { + std::cerr << "abi_serializer benchmark: newaccount pack failed: " << e.what() << "\n"; + return; + } + + // regproducer -- flat-struct baseline comparable to transfer but in a + // larger ABI context. Measures per-field dispatch amortised over the + // whole-action cost in the big-ABI case. + chain::bytes regproducer_bytes; + bool regproducer_ok = false; + try { + const auto regproducer_var = fc::json::from_string( + R"({"producer":"alice","producer_key":"SYS1111111111111111111111111111111114T1Anm",)" + R"("url":"https://example.com","location":0})"); + regproducer_bytes = system_abis.variant_to_binary("regproducer", regproducer_var, yield_fn()); + regproducer_ok = true; + } catch (const std::exception& e) { + std::cerr << "abi_serializer benchmark warning: regproducer prepack failed (" << e.what() + << "); skipping regproducer scenarios.\n"; + } + + // Construction scenarios. Each construction is slow enough that one outer + // run is meaningful; no inner loop needed. + benchmarking("abi_serializer construct token (small ~4KB)", [&] { + abi_serializer a{abi_def(token_abi), yield_fn()}; + (void)a; + }); + benchmarking("abi_serializer construct system (~37KB)", [&] { + abi_serializer a{abi_def(system_abi), yield_fn()}; + (void)a; + }); + + // Steady-state decode (variant path). Inner loop sized to keep per-op + // overhead well above timer resolution. + constexpr size_t decode_iters = 10000; + constexpr size_t big_decode_iters = 5000; + + benchmarking("decode transfer x10000 (variant)", [&] { + for (size_t i = 0; i < decode_iters; ++i) { + auto v = token_abis.binary_to_variant("transfer", transfer_bytes, yield_fn()); + (void)v; + } + }); + benchmarking("decode newaccount x5000 (variant, nested)", [&] { + for (size_t i = 0; i < big_decode_iters; ++i) { + auto v = system_abis.binary_to_variant("newaccount", newaccount_bytes, yield_fn()); + (void)v; + } + }); + if (regproducer_ok) { + benchmarking("decode regproducer x5000 (variant)", [&] { + for (size_t i = 0; i < big_decode_iters; ++i) { + auto v = system_abis.binary_to_variant("regproducer", regproducer_bytes, yield_fn()); + (void)v; + } + }); + } + + // Round-trip pack+unpack of transfer -- covers the pack half, not just unpack. + benchmarking("roundtrip transfer x5000 (pack+unpack)", [&] { + for (size_t i = 0; i < big_decode_iters; ++i) { + auto bin = token_abis.variant_to_binary("transfer", transfer_var, yield_fn()); + auto var = token_abis.binary_to_variant("transfer", bin, yield_fn()); + (void)var; + } + }); + + // resolve_type on a typedef ("account_name" -> "name") -- typedef chain walk. + benchmarking("resolve_type account_name x100000", [&] { + const std::string probe = "account_name"; + for (size_t i = 0; i < 100000; ++i) { + auto r = system_abis.resolve_type(probe); + (void)r; + } + }); + + // ------------------------------------------------------------------------- + // Streaming JSON comparisons. Each pair measures the same payload twice: + // - "(variant) json": binary_to_variant + fc::json::to_string + // - "(stream) json" : binary_to_json_stream directly into a json_writer + // + // The streaming path skips the per-field fc::variant tree and the post-pass + // string serialiser, so the gap should grow with field count. + // ------------------------------------------------------------------------- + + benchmarking("transfer x10000 (variant) json", [&] { + for (size_t i = 0; i < decode_iters; ++i) { + auto v = token_abis.binary_to_variant("transfer", transfer_bytes, yield_fn()); + auto s = fc::json::to_string(v, fc::time_point::maximum()); + (void)s; + } + }); + benchmarking("transfer x10000 (stream) json", [&] { + for (size_t i = 0; i < decode_iters; ++i) { + std::string out; + out.reserve(256); + fc::json_writer w(out); + token_abis.binary_to_json_stream("transfer", transfer_bytes, w, yield_fn()); + } + }); + + benchmarking("newaccount x5000 (variant) json", [&] { + for (size_t i = 0; i < big_decode_iters; ++i) { + auto v = system_abis.binary_to_variant("newaccount", newaccount_bytes, yield_fn()); + auto s = fc::json::to_string(v, fc::time_point::maximum()); + (void)s; + } + }); + benchmarking("newaccount x5000 (stream) json", [&] { + for (size_t i = 0; i < big_decode_iters; ++i) { + std::string out; + out.reserve(512); + fc::json_writer w(out); + system_abis.binary_to_json_stream("newaccount", newaccount_bytes, w, yield_fn()); + } + }); + + if (regproducer_ok) { + benchmarking("regproducer x5000 (variant) json", [&] { + for (size_t i = 0; i < big_decode_iters; ++i) { + auto v = system_abis.binary_to_variant("regproducer", regproducer_bytes, yield_fn()); + auto s = fc::json::to_string(v, fc::time_point::maximum()); + (void)s; + } + }); + benchmarking("regproducer x5000 (stream) json", [&] { + for (size_t i = 0; i < big_decode_iters; ++i) { + std::string out; + out.reserve(256); + fc::json_writer w(out); + system_abis.binary_to_json_stream("regproducer", regproducer_bytes, w, yield_fn()); + } + }); + } + + // ------------------------------------------------------------------------- + // get_block-shape comparison. Builds a synthetic signed_block holding a + // single transaction with N transfer actions, then times two paths: + // - variant: abi_serializer::to_variant + json::to_string + // (mirrors chain_plugin::convert_block) + // - stream: abi_serializer::to_json_stream straight into + // a json_writer (mirrors convert_block_stream) + // The actions count surfaces the per-action ABI decode cost, which is the + // variant path's main allocation hot spot. + // ------------------------------------------------------------------------- + + constexpr size_t actions_per_trx = 4; + constexpr size_t block_iters = 2000; + + transaction inner_trx; + inner_trx.expiration = fc::time_point_sec{ fc::time_point::now() + fc::seconds(60) }; + inner_trx.ref_block_num = 1; + inner_trx.ref_block_prefix = 1; + for (size_t i = 0; i < actions_per_trx; ++i) { + action act; + act.account = "sysio.token"_n; + act.name = "transfer"_n; + act.authorization = { { "alice"_n, "active"_n } }; + act.data = transfer_bytes; + inner_trx.actions.push_back(std::move(act)); + } + + signed_transaction signed_trx{ std::move(inner_trx), {}, {} }; + packed_transaction ptrx(std::move(signed_trx), packed_transaction::compression_type::none); + transaction_receipt rcpt(std::move(ptrx)); + + signed_block block; + block.producer = "alice"_n; + block.timestamp = block_timestamp_type{}; + block.transactions.push_back(std::move(rcpt)); + + // Resolver: sysio.token -> the prebuilt token_abis; other accounts -> nullopt + // (matches production chain_plugin behaviour for unknown accounts). + auto block_resolver = [&token_abis](const name& acct) -> std::optional { + if (acct == "sysio.token"_n) return token_abis; + return {}; + }; + + benchmarking("block (4 transfers) x2000 (variant) json", [&] { + for (size_t i = 0; i < block_iters; ++i) { + fc::variant pretty; + abi_serializer::to_variant(block, pretty, block_resolver, max_serialization_time); + auto s = fc::json::to_string(pretty, fc::time_point::maximum()); + (void)s; + } + }); + benchmarking("block (4 transfers) x2000 (stream) json", [&] { + for (size_t i = 0; i < block_iters; ++i) { + std::string out; + out.reserve(2048); + fc::json_writer w(out); + abi_serializer::to_json_stream(block, w, block_resolver, max_serialization_time); + } + }); + + // ------------------------------------------------------------------------- + // get_block-shape comparison INCLUDING per-request resolver build. The + // scenarios above hold the abi_serializer constant across iterations -- that + // measures only the walk + emit cost. In production each /v1/chain/get_block + // request rebuilds the resolver from the captured abi bytes (Phase 2 of + // read_only::get_block_stream_async), so the per-request cost includes + // abi_serializer::to_abi(bytes, abi_def) + abi_serializer ctor for every + // unique account referenced in the block. These scenarios capture that. + // + // Setup: pack the parsed token ABI to raw bytes once (matches what + // capture_abis would produce), then in each iteration parse + construct + + // walk + emit, simulating the full Phase 2 + Phase 3 work for one request. + // ------------------------------------------------------------------------- + + const chain::bytes token_abi_bytes = fc::raw::pack(token_abi); + + constexpr size_t block_full_iters = 500; + + benchmarking("get_block-shape x500 (variant + abi build) json", [&] { + for (size_t i = 0; i < block_full_iters; ++i) { + abi_def abi; + abi_serializer::to_abi(token_abi_bytes, abi); + abi_serializer fresh{std::move(abi), yield_fn()}; + auto resolver = [&fresh](const name& acct) -> std::optional { + if (acct == "sysio.token"_n) return fresh; + return {}; + }; + fc::variant pretty; + abi_serializer::to_variant(block, pretty, resolver, max_serialization_time); + auto s = fc::json::to_string(pretty, fc::time_point::maximum()); + (void)s; + } + }); + benchmarking("get_block-shape x500 (stream + abi build) json", [&] { + for (size_t i = 0; i < block_full_iters; ++i) { + abi_def abi; + abi_serializer::to_abi(token_abi_bytes, abi); + abi_serializer fresh{std::move(abi), yield_fn()}; + auto resolver = [&fresh](const name& acct) -> std::optional { + if (acct == "sysio.token"_n) return fresh; + return {}; + }; + std::string out; + out.reserve(2048); + fc::json_writer w(out); + abi_serializer::to_json_stream(block, w, resolver, max_serialization_time); + } + }); +} + +} // namespace sysio::benchmark diff --git a/benchmark/benchmark.cpp b/benchmark/benchmark.cpp index 3d06627e82..88551dcaeb 100644 --- a/benchmark/benchmark.cpp +++ b/benchmark/benchmark.cpp @@ -17,7 +17,8 @@ std::map> features { { "blake2", blake2_benchmarking }, { "bls", bls_benchmarking }, { "merkle", merkle_benchmarking }, - { "trace_api_json", trace_api_json_benchmarking } + { "trace_api_json", trace_api_json_benchmarking }, + { "abi_serializer", abi_serializer_benchmarking } }; // values to control cout format diff --git a/benchmark/benchmark.hpp b/benchmark/benchmark.hpp index 5aa2232009..7e4df8fc1b 100644 --- a/benchmark/benchmark.hpp +++ b/benchmark/benchmark.hpp @@ -25,6 +25,7 @@ void blake2_benchmarking(); void bls_benchmarking(); void merkle_benchmarking(); void trace_api_json_benchmarking(); +void abi_serializer_benchmarking(); void benchmarking(const std::string& name, const std::function& func, std::optional num_runs = {}); From 673f34dbc1ff2c828ae7611ff8f50945dee92d7f Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Wed, 29 Apr 2026 11:27:27 -0500 Subject: [PATCH 44/71] chain_plugin: replace fc::variant fields in get_block_header / get_finalizer_info results with typed values Changes the response-struct fields to use typed values directly instead of an `fc::variant` wrapper: - `get_block_header_result.signed_block_header`: `fc::variant` -> `chain::signed_block_header` - `get_finalizer_info_result.active_finalizer_policy` / `.pending_finalizer_policy`: `fc::variant` -> `std::optional` Both target types are already FC_REFLECT'd, so the streaming-reflector `to_json_stream` walks the same fields as the variant tree did. JSON output is byte-equivalent: empty `fc::variant` and empty `std::optional` both emit `null`; populated forms emit identical reflected-field shape. No public API or response-shape change; just removes a variant tree from the response build. --- .../include/sysio/chain_plugin/chain_plugin.hpp | 8 ++++---- plugins/chain_plugin/src/chain_plugin.cpp | 11 +++++------ 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp b/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp index a43c80e399..1eb2b0bef7 100644 --- a/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp +++ b/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp @@ -496,8 +496,8 @@ class read_only : public api_base { }; struct get_block_header_result { - chain::block_id_type id; - fc::variant signed_block_header; + chain::block_id_type id; + chain::signed_block_header signed_block_header; std::optional block_extensions; }; @@ -632,8 +632,8 @@ class read_only : public api_base { }; struct get_finalizer_info_result { - fc::variant active_finalizer_policy; // current active policy - fc::variant pending_finalizer_policy; // current pending policy. Empty if not existing + std::optional active_finalizer_policy; // current active policy + std::optional pending_finalizer_policy; // current pending policy. Empty if not existing // Last tracked vote information for each of the finalizers in // active_finalizer_policy and pending_finalizer_policy. diff --git a/plugins/chain_plugin/src/chain_plugin.cpp b/plugins/chain_plugin/src/chain_plugin.cpp index 802f731571..385c1ca2d6 100644 --- a/plugins/chain_plugin/src/chain_plugin.cpp +++ b/plugins/chain_plugin/src/chain_plugin.cpp @@ -2685,11 +2685,10 @@ read_only::get_finalizer_info_result read_only::get_finalizer_info( const read_o std::set finalizer_keys; // Populate a particular finalizer policy - auto add_policy_to_result = [&](const finalizer_policy_ptr& from_policy, fc::variant& to_policy) { + auto add_policy_to_result = [&](const finalizer_policy_ptr& from_policy, + std::optional& to_policy) { if (from_policy) { - // Use string format of public key for easy uses - to_variant(*from_policy, to_policy); - + to_policy = *from_policy; for (const auto& f: from_policy->finalizers) { finalizer_keys.insert(f.public_key); } @@ -2841,7 +2840,7 @@ read_only::get_block_header_result read_only::get_block_header(const read_only:: } SYS_RETHROW_EXCEPTIONS(chain::block_id_type_exception, "Invalid block ID: {}", params.block_num_or_id) } SYS_ASSERT( header, unknown_block_exception, "Could not find block header: {}", params.block_num_or_id); - return { header->calculate_id(), fc::variant{*header}, {}}; + return { header->calculate_id(), std::move(*header), {}}; } else { signed_block_ptr block; if( block_num ) { @@ -2852,7 +2851,7 @@ read_only::get_block_header_result read_only::get_block_header(const read_only:: } SYS_RETHROW_EXCEPTIONS(chain::block_id_type_exception, "Invalid block ID: {}", params.block_num_or_id) } SYS_ASSERT( block, unknown_block_exception, "Could not find block header: {}", params.block_num_or_id); - return { block->calculate_id(), fc::variant{static_cast(*block)}, block->block_extensions}; + return { block->calculate_id(), static_cast(*block), block->block_extensions }; } } From d5ec7e2107e976afa0f762feb910508a7f8da7c3 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Wed, 29 Apr 2026 14:01:06 -0500 Subject: [PATCH 45/71] libfc: streaming overloads for fc::blob, fc::exception, fc::log_message, fc::log_context Adds the `to_json_stream` overloads needed for the streaming reflector path to emit these types with byte-equivalent JSON to their `to_variant` counterparts: - `fc::blob`: was emitting `{"data": ""}` because `fc::blob` is FC_REFLECT'd as `(data)` and the reflector-based `to_json_stream` has no special-case. `to_variant(blob)` produces a base64 string; the streaming overload now matches. Latent bug -- callers like clio's `result["abi"].as_blob()` were failing on streamed `get_raw_abi`. - `fc::exception`: friend declaration + impl. Needed by the reflected walk of `transaction_trace.except`. - `fc::log_message` / `fc::log_context`: thin wrappers around their `to_variant`-then-walk path; these are inside `fc::exception`'s `stack` field. --- libraries/libfc/include/fc/exception/exception.hpp | 4 ++++ libraries/libfc/include/fc/log/log_message.hpp | 6 ++++++ libraries/libfc/include/fc/variant.hpp | 5 +++++ libraries/libfc/src/exception.cpp | 12 ++++++++++++ libraries/libfc/src/log/log_message.cpp | 13 +++++++++++++ libraries/libfc/src/variant.cpp | 5 +++++ 6 files changed, 45 insertions(+) diff --git a/libraries/libfc/include/fc/exception/exception.hpp b/libraries/libfc/include/fc/exception/exception.hpp index 6da3f6efd7..e2cb9b48cb 100644 --- a/libraries/libfc/include/fc/exception/exception.hpp +++ b/libraries/libfc/include/fc/exception/exception.hpp @@ -12,6 +12,7 @@ namespace fc { namespace detail { class exception_impl; } + class json_writer; enum exception_code { @@ -120,6 +121,7 @@ namespace fc friend void to_variant( const exception& e, variant& v ); friend void from_variant( const variant& e, exception& ll ); + friend void to_json_stream( const exception& e, json_writer& w ); exception& operator=( const exception& copy ); exception& operator=( exception&& copy ); @@ -129,6 +131,8 @@ namespace fc void to_variant( const exception& e, variant& v ); void from_variant( const variant& e, exception& ll ); + /// JSON shape mirrors `to_variant(exception)`: { code, name, message, stack: [log_messages...] }. + void to_json_stream( const exception& e, json_writer& w ); typedef std::shared_ptr exception_ptr; typedef std::optional oexception; diff --git a/libraries/libfc/include/fc/log/log_message.hpp b/libraries/libfc/include/fc/log/log_message.hpp index 6bc81712b9..a86f7eab11 100644 --- a/libraries/libfc/include/fc/log/log_message.hpp +++ b/libraries/libfc/include/fc/log/log_message.hpp @@ -137,6 +137,12 @@ namespace fc void to_variant( const log_message& l, variant& v ); void from_variant( const variant& l, log_message& c ); + /// JSON shape mirrors `to_variant(log_message)`; uses a per-message variant intermediate as the streaming form + /// (log_message has a private impl pointer rather than reflected fields, so we can't walk it directly). + class json_writer; + void to_json_stream( const log_message& m, json_writer& w ); + void to_json_stream( const log_context& l, json_writer& w ); + typedef std::vector log_messages; template diff --git a/libraries/libfc/include/fc/variant.hpp b/libraries/libfc/include/fc/variant.hpp index cb025d48c4..a0f98e6496 100644 --- a/libraries/libfc/include/fc/variant.hpp +++ b/libraries/libfc/include/fc/variant.hpp @@ -61,6 +61,11 @@ namespace fc void to_variant( const blob& var, fc::variant& vo ); void from_variant( const fc::variant& var, blob& vo ); + /// JSON shape: a base64-encoded string -- matches `to_variant(blob)` (which produces a string variant). Necessary + /// because `fc::blob` is FC_REFLECT'd as `(data)`, so without this explicit overload the reflector-based + /// `to_json_stream` would emit `{"data": ""}` (object form) instead. Callers like clio's + /// `result["abi"].as_blob()` expect the string form. + void to_json_stream( const blob& var, fc::json_writer& w ); template void to_variant( const boost::multi_index_container& s, fc::variant& v ); diff --git a/libraries/libfc/src/exception.cpp b/libraries/libfc/src/exception.cpp index 3f2f597987..960450bffa 100644 --- a/libraries/libfc/src/exception.cpp +++ b/libraries/libfc/src/exception.cpp @@ -2,6 +2,8 @@ #include #include #include +#include +#include #include #include @@ -105,6 +107,16 @@ namespace fc ( "stack", e.get_log() ); } + + void to_json_stream( const exception& e, json_writer& w ) + { + w.begin_object(); + w.set( "code", e.code() ); + w.set( "name", e.name() ); + w.set( "message", e.my->_what ); + w.set( "stack", e.get_log() ); + w.end_object(); + } void from_variant( const variant& v, exception& ll ) { const auto& obj = v.get_object(); diff --git a/libraries/libfc/src/log/log_message.cpp b/libraries/libfc/src/log/log_message.cpp index b7452e3704..d2d1bbecdd 100644 --- a/libraries/libfc/src/log/log_message.cpp +++ b/libraries/libfc/src/log/log_message.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace fc { @@ -106,6 +107,18 @@ namespace fc v = m.to_variant(); } + void to_json_stream( const log_message& m, json_writer& w ) + { + variant v; + to_variant(m, v); + to_json_stream(v, w); + } + + void to_json_stream( const log_context& l, json_writer& w ) + { + to_json_stream(l.to_variant(), w); + } + void to_variant( log_level e, variant& v ) { switch( e ) diff --git a/libraries/libfc/src/variant.cpp b/libraries/libfc/src/variant.cpp index 994c50d3c4..bdf689ae71 100644 --- a/libraries/libfc/src/variant.cpp +++ b/libraries/libfc/src/variant.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -1027,6 +1028,10 @@ void to_variant( const blob& b, variant& v ) { v = variant(base64_encode(b.data.data(), b.data.size())); } +void to_json_stream( const blob& b, json_writer& w ) { + w.value_string(base64_encode(b.data.data(), b.data.size())); +} + void from_variant( const variant& v, blob& b ) { b.data = base64_decode(v.as_string()); } From e661cd574539024a2ae9b3284e6f429ab0b231fc Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Wed, 29 Apr 2026 14:02:20 -0500 Subject: [PATCH 46/71] chain_plugin: trx_retry stores raw ABIs + trace; defer ABI parse and JSON to http pool Removes the eager `abi_serializer::to_variant` build from `trx_retry_db::on_applied_transaction` (chain thread / block-apply path). In its place: - `tracked_transaction` stores `transaction_trace_ptr` + `captured_abis` (raw bytes per referenced account, frozen at apply time). - A new `chain_apis::streamed_processed_trace` wraps `(trace, raw_abis, max_serialization_time)` with a custom `fc::to_json_stream` overload that builds an `abi_resolver` from the raw bytes and calls `abi_serializer::to_json_stream` -- both the parse and the JSON emission run on whichever thread invokes `to_json_stream`, which for HTTP responses is the http thread pool (via `bind_stream::async`). - `read_only::compute_transaction_results.processed`, `read_only::send_read_only_transaction_results.processed`, and a new separate `read_write::send_transaction_results.processed` are now `streamed_processed_trace` instead of `fc::variant`. `send_transaction _results` is broken out from its previous `= push_transaction_results` alias; `push_transaction_results` (legacy) keeps `fc::variant processed` because its body restructures the variant tree (inline_traces tree rebuild) which the streaming form would have to replicate -- worth a separate PR if pursued. - `send_transaction_gen` constructs `streamed_processed_trace` directly for the non-retry path; the trx_retry callback path adapts to the new `unique_ptr` payload. - `tracked_transaction::memory_size()` heuristic: 2x packed-trx size for the trace bulk + captured raw ABI bytes + the trace struct itself. Close enough for the existing `transaction-retry-max-storage-size-gb` budget. Net effect: per-trx ABI serialization (variant or streamed) moves off the chain thread entirely. When a tracked trx times out / rolls back, no serialization work is wasted. Existing `nodeop_retry_transaction_test.py` integration test exercises the full retry + send_transaction path. --- .../sysio/chain_plugin/chain_plugin.hpp | 55 +++++++++++++-- .../sysio/chain_plugin/trx_retry_db.hpp | 5 +- plugins/chain_plugin/src/chain_plugin.cpp | 41 +++++++---- plugins/chain_plugin/src/trx_retry_db.cpp | 70 +++++++++++-------- 4 files changed, 123 insertions(+), 48 deletions(-) diff --git a/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp b/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp index 1eb2b0bef7..2b73f0dc32 100644 --- a/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp +++ b/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp @@ -136,11 +136,11 @@ namespace sysio { /// swallowed and the account's slot is left empty -- callers fall back to the /// hex representation for actions with un-decodable ABI, matching the prior /// `make_resolver(throw_on_yield::no)` behaviour. - inline abi_resolver build_resolver_from_captured_abis(captured_abis cache, + inline abi_resolver build_resolver_from_captured_abis(const captured_abis& cache, const fc::microseconds& max_time) { chain::abi_serializer_cache_t serializers; serializers.reserve(cache.raw_abis.size()); - for (auto& [name, raw] : cache.raw_abis) { + for (const auto& [name, raw] : cache.raw_abis) { if (raw.empty()) { serializers.emplace(name, std::nullopt); continue; @@ -161,9 +161,36 @@ namespace sysio { return abi_resolver(std::move(serializers)); } + /// Heuristic byte-size estimate for memory budgeting. Counts the captured + /// raw-ABI bytes plus a small per-entry overhead -- close enough for the + /// trx_retry_db memory accounting without parsing each ABI eagerly. + inline size_t estimated_size(const captured_abis& cache) { + size_t total = 0; + for (const auto& [name, raw] : cache.raw_abis) { + total += sizeof(name) + raw.size(); + } + return total; + } + namespace chain_apis { struct empty{}; +/// Holds a transaction trace plus the ABI bytes captured at the moment the trace was produced. The trace's action data +/// is not eagerly serialized to JSON or to a variant tree -- instead, when the result is emitted, the captured ABI +/// bytes are parsed into an `abi_serializer` resolver on the http thread pool and the trace is emitted directly via +/// `abi_serializer::to_json_stream`. Compared to the prior `fc::variant processed` field this: +/// - moves the ABI parse + JSON emission off the main / chain thread; +/// - skips the intermediate variant tree allocation per response; +/// - preserves "ABIs at apply time, not fire time" semantics for `trx_retry_db` (the captured bytes are the +/// apply-time snapshot). +/// On individual actions whose contract has a corrupt ABI, the abi_serializer's streaming path falls back to hex for +/// that action's data -- same per-action fallback used by `signed_block`. +struct streamed_processed_trace { + chain::transaction_trace_ptr trace; + captured_abis raw_abis; + fc::microseconds max_serialization_time; +}; + struct linked_action { name account; std::optional action; @@ -670,8 +697,8 @@ class read_only : public api_base { get_producer_schedule_result get_producer_schedule( const get_producer_schedule_params& params, const fc::time_point& deadline )const; struct compute_transaction_results { - chain::transaction_id_type transaction_id; - fc::variant processed; // "processed" is expected JSON for trxs in clio + chain::transaction_id_type transaction_id; + chain_apis::streamed_processed_trace processed; // "processed" is expected JSON for trxs in clio }; struct compute_transaction_params { @@ -681,8 +708,8 @@ class read_only : public api_base { void compute_transaction(compute_transaction_params params, chain::plugin_interface::next_function next ); struct send_read_only_transaction_results { - chain::transaction_id_type transaction_id; - fc::variant processed; + chain::transaction_id_type transaction_id; + chain_apis::streamed_processed_trace processed; }; struct send_read_only_transaction_params { fc::variant transaction; @@ -780,7 +807,13 @@ class read_write : public api_base { void push_transactions(const push_transactions_params& params, chain::plugin_interface::next_function next); using send_transaction_params = push_transaction_params; - using send_transaction_results = push_transaction_results; + /// Distinct from `push_transaction_results` -- send_transaction's `processed` is emitted via streaming directly + /// from the trace + captured ABIs. push_transaction stays on `fc::variant` because it post-processes the variant + /// tree (inline_traces tree restructuring) before returning. + struct send_transaction_results { + chain::transaction_id_type transaction_id; + chain_apis::streamed_processed_trace processed; + }; void send_transaction(send_transaction_params params, chain::plugin_interface::next_function next); struct send_transaction2_params { @@ -888,6 +921,7 @@ FC_REFLECT(sysio::chain_apis::read_only::get_block_header_params, (block_num_or_ FC_REFLECT(sysio::chain_apis::read_only::get_block_header_result, (id)(signed_block_header)(block_extensions)) FC_REFLECT( sysio::chain_apis::read_write::push_transaction_results, (transaction_id)(processed) ) +FC_REFLECT( sysio::chain_apis::read_write::send_transaction_results, (transaction_id)(processed) ) FC_REFLECT( sysio::chain_apis::read_write::send_transaction2_params, (return_failure_trace)(retry_trx)(retry_trx_num_blocks)(transaction) ) // `all_rows` and `filter` deliberately excluded -- gated to in-process callers so HTTP cannot trigger a full-table walk @@ -938,3 +972,10 @@ FC_REFLECT( sysio::chain_apis::read_only::compute_transaction_results, (transact FC_REFLECT( sysio::chain_apis::read_only::send_read_only_transaction_params, (transaction)) FC_REFLECT( sysio::chain_apis::read_only::send_read_only_transaction_results, (transaction_id)(processed) ) FC_REFLECT( sysio::chain_apis::read_only::get_consensus_parameters_results, (chain_config)(wasm_config)) + +namespace fc { + /// JSON shape: emits the trace's reflected fields with action data decoded ABI-aware via the captured ABI bytes. + /// Builds a fresh `abi_resolver` from `t.raw_abis` and calls `abi_serializer::to_json_stream` -- + /// matching the prior `abi_serializer::to_variant` output but skipping the variant tree. + void to_json_stream(const sysio::chain_apis::streamed_processed_trace& t, fc::json_writer& w); +} // namespace fc diff --git a/plugins/chain_plugin/include/sysio/chain_plugin/trx_retry_db.hpp b/plugins/chain_plugin/include/sysio/chain_plugin/trx_retry_db.hpp index 1bd9b4b989..68350580e3 100644 --- a/plugins/chain_plugin/include/sysio/chain_plugin/trx_retry_db.hpp +++ b/plugins/chain_plugin/include/sysio/chain_plugin/trx_retry_db.hpp @@ -4,6 +4,8 @@ namespace sysio::chain_apis { +struct streamed_processed_trace; // defined in chain_plugin.hpp + /** * This class manages the ephemeral indices and data that provide the transaction retry feature. * It is designed to be run on an API node, as it only tracks incoming API transactions. @@ -47,7 +49,8 @@ class trx_retry_db { * @param next report result to user by calling next * @throws throw tx_resource_exhaustion if trx would exceeds max_mem_usage_size */ - void track_transaction( chain::packed_transaction_ptr ptrx, std::optional num_blocks, sysio::chain::next_function> next ); + void track_transaction( chain::packed_transaction_ptr ptrx, std::optional num_blocks, + sysio::chain::next_function> next ); /** * Attach to chain applied_transaction signal diff --git a/plugins/chain_plugin/src/chain_plugin.cpp b/plugins/chain_plugin/src/chain_plugin.cpp index 385c1ca2d6..7f1916faff 100644 --- a/plugins/chain_plugin/src/chain_plugin.cpp +++ b/plugins/chain_plugin/src/chain_plugin.cpp @@ -3112,12 +3112,12 @@ void api_base::send_transaction_gen(API &api, send_transaction_params_t params, if( retry && api.trx_retry.has_value() && !trx_trace_ptr->except) { // will be ack'ed via next later api.trx_retry->track_transaction( ptrx, retry_num_blocks, - [ptrx, next](const next_function_variant>& result ) { + [ptrx, next](const next_function_variant>& result ) { if( std::holds_alternative( result ) ) { next( std::get( result ) ); } else { - fc::variant& output = *std::get>( result ); - next( Result{ptrx->id(), std::move( output )} ); + auto& processed = *std::get>( result ); + next( Result{ptrx->id(), std::move( processed )} ); } } ); retried = true; @@ -3128,20 +3128,20 @@ void api_base::send_transaction_gen(API &api, send_transaction_params_t params, (void)retry_num_blocks; // ref variable to avoid compilation warning } if (!retried) { - // we are still on main thread here. The lambda passed to `next()` below will be executed on the http thread pool + // we are still on main thread here. The lambda passed to `next()` below will be executed on the + // http thread pool. Phase 1 (here, main thread): copy raw ABI bytes for accounts referenced + // in the trace. Phase 2 (http pool): build the streamed_processed_trace which defers ABI + // parsing + JSON emission to response-emit time. using return_type = t_or_exception; next([&api, trx_trace_ptr, - resolver = get_serializers_cache(api.db, trx_trace_ptr, api.abi_serializer_max_time)]() mutable { + raw_abis = sysio::capture_abis(api.db, trx_trace_ptr)]() mutable { try { - fc::variant output; - try { - abi_serializer::to_variant(*trx_trace_ptr, output, resolver, api.abi_serializer_max_time); - } catch( abi_exception& ) { - output = *trx_trace_ptr; - } const transaction_id_type& id = trx_trace_ptr->id; - return return_type(Result{id, std::move( output )}); + return return_type(Result{id, + chain_apis::streamed_processed_trace{ std::move(trx_trace_ptr), + std::move(raw_abis), + api.abi_serializer_max_time }}); } CATCH_AND_RETURN(return_type); }); } @@ -3571,4 +3571,21 @@ const controller::config& chain_plugin::chain_config() const { } } // namespace sysio +namespace fc { +void to_json_stream(const sysio::chain_apis::streamed_processed_trace& t, fc::json_writer& w) { + if (!t.trace) { + w.value_null(); + return; + } + try { + auto resolver = sysio::build_resolver_from_captured_abis(t.raw_abis, t.max_serialization_time); + sysio::chain::abi_serializer::to_json_stream(*t.trace, w, resolver, t.max_serialization_time); + } catch (sysio::chain::abi_exception&) { + // Fall back: emit via reflector with no ABI-aware action data decode. Matches the prior variant path's + // `output = *trx_trace_ptr;` fallback. + to_json_stream(*t.trace, w); + } +} +} // namespace fc + FC_REFLECT( sysio::chain_apis::detail::ram_market_exchange_state_t, (ignore1)(ignore2)(ignore3)(core_symbol)(ignore4) ) diff --git a/plugins/chain_plugin/src/trx_retry_db.cpp b/plugins/chain_plugin/src/trx_retry_db.cpp index 8c40ea41c7..b0f9a58ef5 100644 --- a/plugins/chain_plugin/src/trx_retry_db.cpp +++ b/plugins/chain_plugin/src/trx_retry_db.cpp @@ -27,12 +27,13 @@ namespace { constexpr uint16_t lib_totem = std::numeric_limits::max(); struct tracked_transaction { - const packed_transaction_ptr ptrx; - const uint16_t num_blocks = 0; // lib is lib_totem - uint32_t block_num = 0; - fc::variant trx_trace_v; - fc::time_point last_try; - next_function> next; + const packed_transaction_ptr ptrx; + const uint16_t num_blocks = 0; // lib is lib_totem + uint32_t block_num = 0; + chain::transaction_trace_ptr trace_ptr; + sysio::captured_abis raw_abis; + fc::time_point last_try; + next_function> next; const transaction_id_type& id()const { return ptrx->id(); } fc::time_point_sec expiry()const { return ptrx->expiration(); } @@ -52,7 +53,16 @@ struct tracked_transaction { return block_num != 0; } - size_t memory_size()const { return ptrx->get_estimated_size() + trx_trace_v.estimated_size() + sizeof(*this); } + /// Heuristic for the memory budget: 2x the packed-trx size accounts for the action trace bulk (each action trace + /// mirrors the packed action plus per-action receipt/console/etc), plus the captured raw-ABI bytes, plus the trace + /// struct overhead. + size_t memory_size()const { + const size_t ptrx_sz = ptrx->get_estimated_size(); + return ptrx_sz * 2 + + sysio::estimated_size(raw_abis) + + (trace_ptr ? sizeof(*trace_ptr) : 0) + + sizeof(*this); + } }; struct by_trx_id; @@ -105,7 +115,8 @@ struct trx_retry_db_impl { return _tracked_trxs.index().size(); } - void track_transaction( packed_transaction_ptr ptrx, std::optional num_blocks, next_function> next ) { + void track_transaction( packed_transaction_ptr ptrx, std::optional num_blocks, + next_function> next ) { SYS_ASSERT( _tracked_trxs.memory_size() < _max_mem_usage_size, tx_resource_exhaustion, "Transaction exceeded transaction-retry-max-storage-size-gb limit: {} bytes", _tracked_trxs.memory_size() ); auto i = _tracked_trxs.index().get().find( ptrx->id() ); @@ -113,6 +124,7 @@ struct trx_retry_db_impl { _tracked_trxs.insert( {std::move(ptrx), !num_blocks.has_value() ? lib_totem : *num_blocks, 0, + nullptr, {}, fc::time_point::now(), std::move(next)} ); @@ -133,19 +145,15 @@ struct trx_retry_db_impl { auto& idx = _tracked_trxs.index().get(); auto itr = idx.find(trace->id); if( itr != idx.end() ) { - _tracked_trxs.modify( itr, [&trace, &control=_controller, &abi_max_time=_abi_serializer_max_time]( tracked_transaction& tt ) { + _tracked_trxs.modify( itr, [&trace, &control=_controller]( tracked_transaction& tt ) { tt.block_num = trace->block_num; - try { - // send_transaction trace output format. - // Convert to variant with abi here and now because abi could change in very next transaction. - // Alternatively, we could store off all the abis needed and do the conversion later, but as this is designed - // to run on an API node, probably the best trade off to perform the abi serialization during block processing. - auto resolver = get_serializers_cache(control, trace, abi_max_time); - tt.trx_trace_v.clear(); - abi_serializer::to_variant(*trace, tt.trx_trace_v, resolver, abi_max_time); - } catch( chain::abi_exception& ) { - tt.trx_trace_v = *trace; - } + // Capture the trace and the raw ABI bytes for accounts referenced by it. ABI parsing + JSON emission are + // deferred to response-emit time on the http thread pool (see sysio::chain_apis::streamed_processed_trace + // and `fc::to_json_stream(streamed_processed_trace, json_writer&)`). Capturing the raw bytes here + // preserves "ABIs frozen at apply time" semantics -- subsequent on-chain ABI updates won't affect the + // captured copy. + tt.trace_ptr = trace; + tt.raw_abis = sysio::capture_abis(control, trace); } ); } } @@ -190,7 +198,8 @@ struct trx_retry_db_impl { tt.last_try += fc::seconds( 10 ); } tt.block_num = 0; - tt.trx_trace_v.clear(); + tt.trace_ptr.reset(); + tt.raw_abis = {}; } ); } } @@ -228,9 +237,11 @@ struct trx_retry_db_impl { } // ack for( auto& i: to_process ) { - _tracked_trxs.modify( i, [&]( tracked_transaction& tt ) { - tt.next( std::make_unique( std::move( tt.trx_trace_v ) ) ); - tt.trx_trace_v.clear(); + _tracked_trxs.modify( i, [this]( tracked_transaction& tt ) { + tt.next( std::make_unique( + sysio::chain_apis::streamed_processed_trace{ std::move(tt.trace_ptr), + std::move(tt.raw_abis), + _abi_serializer_max_time } ) ); } ); _tracked_trxs.erase( i ); } @@ -246,9 +257,11 @@ struct trx_retry_db_impl { } // ack for( auto& i: to_process ) { - _tracked_trxs.modify( i, [&]( tracked_transaction& tt ) { - tt.next( std::make_unique( std::move( tt.trx_trace_v ) ) ); - tt.trx_trace_v.clear(); + _tracked_trxs.modify( i, [this]( tracked_transaction& tt ) { + tt.next( std::make_unique( + sysio::chain_apis::streamed_processed_trace{ std::move(tt.trace_ptr), + std::move(tt.raw_abis), + _abi_serializer_max_time } ) ); } ); _tracked_trxs.erase( i ); } @@ -290,7 +303,8 @@ trx_retry_db::trx_retry_db( const chain::controller& controller, size_t max_mem_ trx_retry_db::~trx_retry_db() = default; -void trx_retry_db::track_transaction( chain::packed_transaction_ptr ptrx, std::optional num_blocks, next_function> next ) { +void trx_retry_db::track_transaction( chain::packed_transaction_ptr ptrx, std::optional num_blocks, + next_function> next ) { _impl->track_transaction( std::move( ptrx ), num_blocks, next ); } From 7e149ed70fcd9f5a3c916b477c567cca0d53c812 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Thu, 30 Apr 2026 11:50:22 -0500 Subject: [PATCH 47/71] trace_api: fix status enum + block_time JSON shape regressions Two silent backward-compat breaks in process_block_to_json: - transaction_receipt::status_enum is FC_REFLECT_ENUM-reflected; the variant path emits the member-name string ("executed"). Streaming was emitting the underlying integer via static_cast. Drop the cast and let the existing to_json_stream(enum_type<>) overload emit the member name. - block_timestamp_type uses FC_SERIALIZE_AS_STRING_TEMPLATE; variant emits the ISO date string ("2025-01-01T00:00:00.000"). Streaming was emitting the bare slot uint32 via .slot. Drop .slot and let the trait-generated to_json_stream emit the ISO string. Add streaming_vs_variant_block_response_parity test that round-trips the streaming JSON output through fc::json::from_string and asserts it matches the variant path key-by-key. Catches both regressions and locks the parity invariant for future endpoint migrations. --- .../trace_api_plugin/src/request_handler.cpp | 18 +++--- .../trace_api_plugin/test/test_responses.cpp | 55 +++++++++++++++++++ 2 files changed, 64 insertions(+), 9 deletions(-) diff --git a/plugins/trace_api_plugin/src/request_handler.cpp b/plugins/trace_api_plugin/src/request_handler.cpp index b8820cfd4f..b20e2ac06c 100644 --- a/plugins/trace_api_plugin/src/request_handler.cpp +++ b/plugins/trace_api_plugin/src/request_handler.cpp @@ -149,9 +149,11 @@ namespace { w.key("authorization"); write_authorizations(w, a.authorization); w.key("data"); w.value_hex(a.data.data(), a.data.size()); w.key("return_value"); w.value_hex(a.return_value.data(), a.return_value.size()); - // The abi decode path still produces fc::variant (structure is ABI-dependent, - // no streaming equivalent here). Splice its serialized form as raw JSON so - // the action body stays in the streaming pipeline. + // The abi decode path still produces fc::variant (structure is ABI-dependent). + // TODO: route through abi_serializer::binary_to_json_stream (already in use for + // table rows + streamed_processed_trace) so the action body stays allocation-free + // end-to-end; data_handler currently builds the variant tree and we re-emit it. + // For now, splice its serialized form as raw JSON. auto [params, return_data] = data_handler(a); if (!params.is_null()) { w.set_raw("params", fc::json::to_string(params, fc::json::yield_function_t())); @@ -172,14 +174,12 @@ namespace { w.begin_object(); w.set("id", t.id) .set("block_num", t.block_num) - // block_timestamp_type serializes as its slot (uint32) in to_variant; preserve - // that shape so clients see identical output. - .set("block_time", t.block_time.slot) + // FC_SERIALIZE_AS_STRING-reflected; emits the ISO date string. + .set("block_time", t.block_time) .set("producer_block_id", t.producer_block_id); w.key("actions"); write_actions(w, t.actions, data_handler); - // status is fc::enum_type; to_variant emits the numeric - // underlying value. Match that. - w.set("status", static_cast(t.status)) + // FC_REFLECT_ENUM-reflected; emits the member-name string via to_json_stream(enum_type). + w.set("status", t.status) .set("cpu_usage_us", t.cpu_usage_us) .set("net_usage_words", t.net_usage_words.value) .set("signatures", t.signatures); diff --git a/plugins/trace_api_plugin/test/test_responses.cpp b/plugins/trace_api_plugin/test/test_responses.cpp index 6232659470..649123d8ad 100644 --- a/plugins/trace_api_plugin/test/test_responses.cpp +++ b/plugins/trace_api_plugin/test/test_responses.cpp @@ -1,5 +1,6 @@ #include +#include #include #include @@ -64,6 +65,10 @@ struct response_test_fixture { return response_impl.get_block_trace( block_height ); } + std::string get_block_trace_json( uint32_t block_height ) { + return response_impl.get_block_trace_json( block_height ); + } + // fixture data and methods std::function mock_get_block; std::function>(const action_trace_v0&)> mock_data_handler = default_mock_data_handler; @@ -491,4 +496,54 @@ BOOST_AUTO_TEST_SUITE(trace_responses) BOOST_TEST(null_response.is_null()); } + // The streaming JSON response (process_block_to_json) must produce the same + // observable shape as the variant response (process_block) for every field. + // Round-trip the streaming bytes through fc::json::from_string and compare + // key-by-key with the variant path. Catches silent shape regressions like + // status: "executed" -> 0 or block_time: {"slot":N} -> N. + BOOST_FIXTURE_TEST_CASE(streaming_vs_variant_block_response_parity, response_test_fixture) + { + auto block_trace = block_trace_v0 { + "b000000000000000000000000000000000000000000000000000000000000001"_h, + 1, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + chain::block_timestamp_type(0), + "bp.one"_n, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + std::vector { + { + "0000000000000000000000000000000000000000000000000000000000000001"_h, + std::vector { + { + 0, + "receiver"_n, "contract"_n, "action"_n, + {{ "alice"_n, "active"_n }}, + { 0x00, 0x01, 0x02, 0x03 }, + { 0x04, 0x05, 0x06, 0x07 } + } + }, + fc::enum_type{chain::transaction_receipt_header::status_enum::executed}, + 10, + 5, + std::vector{ chain::signature_type() }, + { chain::time_point_sec(), 1, 0, 100, 50, 0 }, + 1, + chain::block_timestamp_type(0), + std::optional{} + } + } + }; + + mock_get_block = [&block_trace]( uint32_t height ) -> get_block_t { + return std::make_tuple(data_log_entry(block_trace), false); + }; + + fc::variant variant_response = get_block_trace( 1 ); + std::string streamed_response = get_block_trace_json( 1 ); + fc::variant streamed_as_variant = fc::json::from_string( streamed_response ); + + BOOST_TEST(to_kv(variant_response) == to_kv(streamed_as_variant), boost::test_tools::per_element()); + } + BOOST_AUTO_TEST_SUITE_END() From a401e7f83957307d54aad500600cc5538c360751 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Thu, 30 Apr 2026 11:52:25 -0500 Subject: [PATCH 48/71] chain_api_plugin: restore /v1/chain/get_info to api_category::node The streaming-cb migration silently switched /v1/chain/get_info from api_category::node to chain_ro. Operators with category-per-port whitelists (--http-server-address-node vs --http-server-address-chain-ro) would see get_info disappear from the node port or appear on a port that wasn't configured for it. get_info is the most-called endpoint by clients (block height polling, sync status); silently moving it across category boundaries is a deployment-visible change. Restore the original cat::node mapping. --- plugins/chain_api_plugin/src/chain_api_plugin.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/chain_api_plugin/src/chain_api_plugin.cpp b/plugins/chain_api_plugin/src/chain_api_plugin.cpp index 7491b10616..782c060430 100644 --- a/plugins/chain_api_plugin/src/chain_api_plugin.cpp +++ b/plugins/chain_api_plugin/src/chain_api_plugin.cpp @@ -109,7 +109,7 @@ void chain_api_plugin::plugin_startup() { // Run get_info on http thread only _http_plugin.add_async_api_stream({ bind_stream<&ro::get_info, dispatch::sync>( - _http_plugin, ro_api, "/v1/chain/get_info", cat::chain_ro, pt::no_params, 200), + _http_plugin, ro_api, "/v1/chain/get_info", cat::node, pt::no_params, 200), }); _http_plugin.add_api_stream({ From de584b3e27295b101561be62e8f1eb4141332397 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Thu, 30 Apr 2026 11:54:39 -0500 Subject: [PATCH 49/71] fc::bitset: restore MAX_NUM_ARRAY_ELEMENTS cap on to_string The pre-PR to_variant(bitset) had a guard: if (num_blocks > MAX_NUM_ARRAY_ELEMENTS) throw std::range_error("..."); Migration to FC_SERIALIZE_AS_STRING(fc::bitset) deleted that path without porting the cap. to_string() now allocates an unbounded result string (one char per bit) for any caller; bitset is used in finalizer voting bitmasks (bounded by node count today, but the guard is no longer enforced). Move the cap into to_string() itself so every JSON path (variant, streaming, datastream) gets it for free. Add a unit test exercising the throw at over_cap and the boundary success at the cap. --- libraries/libfc/include/fc/bitset.hpp | 6 ++++++ libraries/libfc/test/test_bitset.cpp | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/libraries/libfc/include/fc/bitset.hpp b/libraries/libfc/include/fc/bitset.hpp index c503c0c322..bd2c95b083 100644 --- a/libraries/libfc/include/fc/bitset.hpp +++ b/libraries/libfc/include/fc/bitset.hpp @@ -7,6 +7,7 @@ #include #include #include +#include namespace fc { @@ -153,6 +154,11 @@ struct bitset { } std::string to_string() const { + // Guard against allocating an unbounded result string for an oversize bitset. + // Mirrors the pre-FC_SERIALIZE_AS_STRING to_variant cap; enforced here so it + // applies to every JSON path (variant, streaming, datastream). + if (num_blocks() > MAX_NUM_ARRAY_ELEMENTS) + throw std::range_error("number of blocks of bitset cannot be greater than MAX_NUM_ARRAY_ELEMENTS"); std::string res; res.resize(size()); size_t idx = 0; diff --git a/libraries/libfc/test/test_bitset.cpp b/libraries/libfc/test/test_bitset.cpp index 9d86e556c0..6423b39e1c 100644 --- a/libraries/libfc/test/test_bitset.cpp +++ b/libraries/libfc/test/test_bitset.cpp @@ -1,8 +1,11 @@ #include #include +#include #include +#include + using namespace fc; BOOST_AUTO_TEST_SUITE(bitset_test_suite) @@ -167,4 +170,19 @@ BOOST_AUTO_TEST_CASE(bitset_test) test_bitset("01110011010100101111001101100100100111111111111111111001"); } +// to_string() drives every JSON path (FC_SERIALIZE_AS_STRING). Cap on the +// number of blocks must be enforced here so an oversize bitset can't allocate +// an unbounded result string. Threshold matches the pre-FC_SERIALIZE_AS_STRING +// to_variant guard (num_blocks > MAX_NUM_ARRAY_ELEMENTS). +BOOST_AUTO_TEST_CASE(bitset_to_string_caps_oversize) +{ + const size_t over_bits = (MAX_NUM_ARRAY_ELEMENTS + 1) * fc::bitset::bits_per_block; + fc::bitset bs(over_bits); + BOOST_CHECK_THROW(bs.to_string(), std::range_error); + + // Boundary: exactly MAX_NUM_ARRAY_ELEMENTS blocks must still succeed. + fc::bitset bs_at_cap(MAX_NUM_ARRAY_ELEMENTS * fc::bitset::bits_per_block); + BOOST_CHECK_NO_THROW(bs_at_cap.to_string()); +} + BOOST_AUTO_TEST_SUITE_END() From 0f429b404312fdf105578cdddd322452070ece81 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Thu, 30 Apr 2026 11:55:33 -0500 Subject: [PATCH 50/71] fc::crypto: =delete to_json_stream for private keys Streaming HTTP responses must never carry private key material. For fc::crypto::private_key the polymorphic wrapper, no to_json_stream overload exists today, but the FC_REFLECT(_storage) reflector path could synthesize one in the future if a std::variant overload is added. The =delete pre-empts any silent enablement and produces a hard compile error if any future code tries to stream a private key. For fc::crypto::bls::private_key the FC_SERIALIZE_AS_STRING macro auto-generates a streaming overload; add a non-template =delete that outranks the constrained template (overload resolution prefers the non-template). to_variant / from_variant remain intact so the wallet_api_plugin and signature_provider_manager_plugin keep working. --- libraries/libfc/include/fc/crypto/bls_private_key.hpp | 9 +++++++++ libraries/libfc/include/fc/crypto/private_key.hpp | 7 +++++++ 2 files changed, 16 insertions(+) diff --git a/libraries/libfc/include/fc/crypto/bls_private_key.hpp b/libraries/libfc/include/fc/crypto/bls_private_key.hpp index 78579d2226..9b15af7306 100644 --- a/libraries/libfc/include/fc/crypto/bls_private_key.hpp +++ b/libraries/libfc/include/fc/crypto/bls_private_key.hpp @@ -227,6 +227,15 @@ FC_REFLECT_TYPENAME(fc::crypto::bls::public_key) #include FC_SERIALIZE_AS_STRING(fc::crypto::bls::private_key) +// Override the FC_SERIALIZE_AS_STRING-generated to_json_stream for bls::private_key +// with a deleted non-template overload: streaming HTTP responses must never carry +// private key material. to_variant / from_variant remain intact so server-internal +// flows (signature_provider, wallet_api_plugin) keep working. +namespace fc { + class json_writer; + void to_json_stream(const crypto::bls::private_key& var, json_writer& w) = delete; +} + FC_REFLECT(fc::crypto::bls::public_key_shim, (shim_ptr)) FC_REFLECT(fc::crypto::bls::signature_shim, (shim_ptr)) diff --git a/libraries/libfc/include/fc/crypto/private_key.hpp b/libraries/libfc/include/fc/crypto/private_key.hpp index 9fbdd1c6b3..e7f116fa6e 100644 --- a/libraries/libfc/include/fc/crypto/private_key.hpp +++ b/libraries/libfc/include/fc/crypto/private_key.hpp @@ -105,6 +105,13 @@ namespace fc { void to_variant(const crypto::private_key& var, variant& vo, const fc::yield_function_t& yield = fc::yield_function_t()); void from_variant(const variant& var, crypto::private_key& vo); + + // Deleting the streaming-JSON overload is a load-bearing safety check, not a + // not-yet-implemented marker. HTTP responses must never expose private key + // material; the variant path remains available for wallet_api_plugin and other + // server-internal consumers that intentionally serialize private keys. + class json_writer; + void to_json_stream(const crypto::private_key& var, json_writer& w) = delete; } // namespace fc FC_REFLECT(fc::crypto::private_key, (_storage) ) From bb66b8d040a21537934f68816829e3486357d01c Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Thu, 30 Apr 2026 11:57:00 -0500 Subject: [PATCH 51/71] libfc: streaming JSON correctness + small cleanups JSON correctness: - value_double: switch from snprintf("%.17g", ...) to std::to_chars, which is locale-independent and noexcept. %g honors LC_NUMERIC, so any plugin that calls setlocale(LC_NUMERIC, "de_DE.UTF-8") would silently emit comma-radix doubles -- invalid JSON. - value_double: throw std::invalid_argument on NaN/inf BEFORE value_prefix() runs, so a non-finite value fails loudly without leaving the writer's frame state inconsistent. - key() / end_object(): assert against the dangling-key state machine bugs (key after key, end_object with awaiting_value_). Programming errors caught by CI's assert-on builds. - balanced(): tighten to !awaiting_value_ as well; tests / wrappers use it to confirm end-state integrity. - value_int64 / value_uint64: in the to_json_stream overloads, quote 64-bit integers with magnitude > 0xffffffff to preserve JS's 2^53 mantissa precision. Matches the variant emission shape; the raw value_int64 / value_uint64 writers stay number-only. - to_json_stream(std::array): add a hex-string overload so reflected struct fields of that shape match the variant path (otherwise the generic vector template emits a JSON array of byte-numbers). - ctor reserve(): only reserve when capacity slack is < 4096, so a caller that pre-reserves a larger buffer doesn't risk a spec-allowed shrink. - int64_decimal_buf: name the to_chars buffer size constant with a static_assert that uint64 max fits. - raw_fwd.hpp: drop stale fc::fixed_string forward decl (type was removed earlier). - variant.cpp: drop duplicate include. - serialize_as_string.hpp: expand FC_SERIALIZE_AS_STRING_TEMPLATE doc with worked correct/wrong examples (parens needed on both args). - variant.hpp: rewrite the to_json_stream(variant) doc -- the streaming walker now writes tokens directly without a fc::json::to_string intermediate string allocation. - to_json_stream(char) gets a comment documenting the int8 emit shape so future readers don't try to "fix" it to a single-char string. - Unreflected-T branch of the reflector dispatch gets a static_assert with a friendly diagnostic instead of a wall of template errors. - bls_signature.cpp / keccak256.cpp: trailing newline. Tests added in test_json_stream.cpp: - array_of_char_emits_hex_string: shape parity with variant - large_int_quoting_matches_variant: precision-guard quoting - value_double_locale_independent: locale-radix safety (skips when no comma-radix locale is installed) - value_double_rejects_non_finite: NaN/inf rejection + frame state - streaming_vs_variant_parity_libfc_leaf_types: unified parity check for exception, log_message, int128, bls public_key/signature, url, bitset, enum_type, static_variant, vector, blob. --- libraries/libfc/include/fc/io/json_stream.hpp | 70 ++++++-- libraries/libfc/include/fc/io/raw_fwd.hpp | 1 - .../libfc/include/fc/reflect/json_stream.hpp | 33 +++- .../libfc/include/fc/serialize_as_string.hpp | 17 +- libraries/libfc/include/fc/variant.hpp | 17 +- libraries/libfc/src/crypto/bls_signature.cpp | 2 +- libraries/libfc/src/crypto/keccak256.cpp | 2 +- libraries/libfc/src/variant.cpp | 1 - libraries/libfc/test/io/test_json_stream.cpp | 170 ++++++++++++++++++ 9 files changed, 284 insertions(+), 29 deletions(-) diff --git a/libraries/libfc/include/fc/io/json_stream.hpp b/libraries/libfc/include/fc/io/json_stream.hpp index 96be78edc7..fc114dea66 100644 --- a/libraries/libfc/include/fc/io/json_stream.hpp +++ b/libraries/libfc/include/fc/io/json_stream.hpp @@ -4,8 +4,11 @@ #include #include +#include #include #include +#include +#include #include #include #include @@ -38,7 +41,10 @@ class json_writer { { // Enough room for a reasonable response without reallocation; callers that know // the expected size can reserve() themselves before constructing the writer. - out_.reserve(out_.size() + 4096); + // Skip if the caller already has slack so we don't risk a spec-permitted shrink. + if (out_.capacity() - out_.size() < 4096) { + out_.reserve(out_.size() + 4096); + } } void begin_object() { @@ -49,6 +55,7 @@ class json_writer { void end_object() { assert(!stack_.empty() && stack_.back().ctx == context::object); + assert(!awaiting_value_); // key() without a value would leave a dangling colon out_.push_back('}'); stack_.pop_back(); } @@ -67,6 +74,7 @@ class json_writer { void key(std::string_view k) { assert(!stack_.empty() && stack_.back().ctx == context::object); + assert(!awaiting_value_); // two key() calls in a row would produce "a":,"b": (invalid JSON) if (stack_.back().has_item) { out_.push_back(','); } else { @@ -99,13 +107,20 @@ class json_writer { void value_uint8(uint8_t n) { value_uint64(n); } void value_double(double d) { + // NaN / +-inf have no JSON representation: any encoder would emit non-conforming + // tokens that no parser will accept. Reject before mutating the output buffer + // so the writer's frame state stays consistent on throw. + if (!std::isfinite(d)) { + throw std::invalid_argument("fc::json_writer::value_double: non-finite double cannot be JSON-encoded"); + } value_prefix(); + // std::to_chars is locale-independent (period radix always) and shortest-roundtrip: + // the parsed-back double is bit-identical to the input. snprintf with %g would + // honor LC_NUMERIC and emit comma radix in non-C locales -- invalid JSON. char buf[32]; - // "%.17g" preserves round-trip precision for IEEE 754 doubles. Callers that need - // bit-exact formatting should emit via raw_value(). - int n = std::snprintf(buf, sizeof(buf), "%.17g", d); - if (n > 0) out_.append(buf, static_cast(n)); - else out_.append("null", 4); // NaN/inf are not valid JSON; emit null defensively. + auto r = std::to_chars(buf, buf + sizeof(buf), d); + assert(r.ec == std::errc{}); // unreachable: finite double + 32-byte buffer always succeeds + out_.append(buf, static_cast(r.ptr - buf)); } void value_bool(bool b) { @@ -176,9 +191,35 @@ class json_writer { return *this; } - /// True when all begin_* have been paired with end_*. Asserted internally on destructor - /// usage via value_prefix() is not possible, so callers can check this in tests. - bool balanced() const { return stack_.empty(); } + /// True when all begin_* have been paired with end_* AND no key() is awaiting + /// its value. Used by tests/sinks that wrap the writer to confirm end-state + /// integrity after a streaming sequence completes. + bool balanced() const { return stack_.empty() && !awaiting_value_; } + + /// Snapshot of the writer's mutable state. Pair with rewind() to discard + /// any tokens emitted between the two calls. Cheap (~3 words copied) and + /// non-throwing. Used by the abi_serializer streaming path to roll back + /// half-written ABI fields when the inner unpack throws partway. + struct checkpoint_t { + size_t buf_size = 0; + size_t stack_size = 0; + bool surviving_top_has_item = false; // restored value of stack_[stack_size-1].has_item + bool awaiting_value = false; + }; + checkpoint_t checkpoint() const noexcept { + return { + out_.size(), + stack_.size(), + stack_.empty() ? false : stack_.back().has_item, + awaiting_value_ + }; + } + void rewind(const checkpoint_t& cp) noexcept { + out_.resize(cp.buf_size); + stack_.resize(cp.stack_size); + if (!stack_.empty()) stack_.back().has_item = cp.surviving_top_has_item; + awaiting_value_ = cp.awaiting_value; + } private: enum class context : uint8_t { object, array }; @@ -207,16 +248,17 @@ class json_writer { } // std::to_chars is non-throwing, locale-independent, and avoids the format-string - // parsing overhead that snprintf pays on every call. buf is sized for the longest - // possible decimal of int64/uint64 (20 chars + sign + slack). Failure case is - // unreachable for fixed-width integers; the unconditional out_.append handles it. + // parsing overhead that snprintf pays on every call. Buffer sized for the longest + // possible decimal of int64/uint64 (digits10+1 plus sign plus slack). + static constexpr size_t int64_decimal_buf = std::numeric_limits::digits10 + 3; + static_assert(int64_decimal_buf >= std::numeric_limits::digits10 + 1); void append_signed(int64_t n) { - char buf[24]; + char buf[int64_decimal_buf]; auto r = std::to_chars(buf, buf + sizeof(buf), n); out_.append(buf, static_cast(r.ptr - buf)); } void append_unsigned(uint64_t n) { - char buf[24]; + char buf[int64_decimal_buf]; auto r = std::to_chars(buf, buf + sizeof(buf), n); out_.append(buf, static_cast(r.ptr - buf)); } diff --git a/libraries/libfc/include/fc/io/raw_fwd.hpp b/libraries/libfc/include/fc/io/raw_fwd.hpp index 6a81239572..f789d35622 100644 --- a/libraries/libfc/include/fc/io/raw_fwd.hpp +++ b/libraries/libfc/include/fc/io/raw_fwd.hpp @@ -26,7 +26,6 @@ namespace fc { namespace ip { class endpoint; } namespace ecc { class public_key; class private_key; } - template class fixed_string; namespace raw { diff --git a/libraries/libfc/include/fc/reflect/json_stream.hpp b/libraries/libfc/include/fc/reflect/json_stream.hpp index 300afab68d..d090518abd 100644 --- a/libraries/libfc/include/fc/reflect/json_stream.hpp +++ b/libraries/libfc/include/fc/reflect/json_stream.hpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -31,6 +32,9 @@ void to_json_stream(const T& o, json_writer& w); // -- Scalar overloads --------------------------------------------------------------------- inline void to_json_stream(bool b, json_writer& w) { w.value_bool(b); } +// `char` emits as an int8 number (e.g. 'A' -> 65), matching the int8_t overload's shape. +// Single-character JSON strings would surprise reflected-struct consumers; the numeric +// form is consistent with how chars are typically read back via from_variant. inline void to_json_stream(char c, json_writer& w) { w.value_int8(static_cast(c)); } inline void to_json_stream(int8_t n, json_writer& w) { w.value_int8(n); } inline void to_json_stream(uint8_t n, json_writer& w) { w.value_uint8(n); } @@ -38,8 +42,31 @@ inline void to_json_stream(int16_t n, json_writer& w) { w.value_int inline void to_json_stream(uint16_t n, json_writer& w) { w.value_uint16(n); } inline void to_json_stream(int32_t n, json_writer& w) { w.value_int32(n); } inline void to_json_stream(uint32_t n, json_writer& w) { w.value_uint32(n); } -inline void to_json_stream(int64_t n, json_writer& w) { w.value_int64(n); } -inline void to_json_stream(uint64_t n, json_writer& w) { w.value_uint64(n); } +// 64-bit integers with magnitude > 0xffffffff are emitted as JSON strings to +// preserve precision past JS's 2^53 mantissa. Matches fc::variant's emission +// (libfc/src/io/json.cpp, int64_type / uint64_type cases). Writers stay raw; +// callers that want a literal big number use value_int64 / value_uint64 directly. +// Buffer size: 20 digits (uint64 max) + sign + slack. +inline constexpr size_t int64_decimal_buf_size = std::numeric_limits::digits10 + 3; + +inline void to_json_stream(int64_t n, json_writer& w) { + if (n > 0xffffffffLL || n < -0xffffffffLL) { + char buf[int64_decimal_buf_size]; + auto r = std::to_chars(buf, buf + sizeof(buf), n); + w.value_string(std::string_view(buf, r.ptr - buf)); + } else { + w.value_int64(n); + } +} +inline void to_json_stream(uint64_t n, json_writer& w) { + if (n > 0xffffffffULL) { + char buf[int64_decimal_buf_size]; + auto r = std::to_chars(buf, buf + sizeof(buf), n); + w.value_string(std::string_view(buf, r.ptr - buf)); + } else { + w.value_uint64(n); + } +} inline void to_json_stream(float f, json_writer& w) { w.value_double(static_cast(f)); } inline void to_json_stream(double d, json_writer& w) { w.value_double(d); } inline void to_json_stream(std::string_view s, json_writer& w) { w.value_string(s); } @@ -94,6 +121,8 @@ namespace detail { struct if_enum_json { template static void to_json_stream(const T& v, json_writer& w) { + static_assert(fc::reflector::is_defined::value, + "fc::to_json_stream: T must be FC_REFLECT'd or have a hand-written to_json_stream overload"); w.begin_object(); fc::reflector::visit(to_json_stream_visitor(w, v)); w.end_object(); diff --git a/libraries/libfc/include/fc/serialize_as_string.hpp b/libraries/libfc/include/fc/serialize_as_string.hpp index e9b02c54df..91a0a9c637 100644 --- a/libraries/libfc/include/fc/serialize_as_string.hpp +++ b/libraries/libfc/include/fc/serialize_as_string.hpp @@ -52,10 +52,19 @@ inline void to_json_stream(const T& v, fc::json_writer& w) { #define FC_SERIALIZE_AS_STRING(TYPE) \ namespace fc { template<> struct serialize_as_string : std::true_type {}; } -// Class-template variant of FC_SERIALIZE_AS_STRING. Wrap each argument in parens -// so internal commas (template parameter lists, template arguments) survive the -// preprocessor's argument splitting. -// FC_SERIALIZE_AS_STRING_TEMPLATE((typename T, typename U), (my_type)) +// Class-template variant of FC_SERIALIZE_AS_STRING. BOTH arguments must be wrapped +// in parens so internal commas (template parameter lists, template arguments) survive +// the preprocessor's argument splitting. No compile-time check exists for the parens +// themselves -- without them the expansion silently goes wrong. +// +// Correct: +// FC_SERIALIZE_AS_STRING_TEMPLATE((typename T, typename U), (my_type)) +// +// Wrong (preprocessor splits on the first comma; expansion is malformed): +// FC_SERIALIZE_AS_STRING_TEMPLATE(typename T, typename U, my_type) +// +// Wrong (single-arg form -- no parens needed but easy to forget the wrapper): +// FC_SERIALIZE_AS_STRING_TEMPLATE((typename T), my_type) // missing parens on TYPE #define FC_SERIALIZE_AS_STRING_TEMPLATE(TPL_PARAMS, TYPE) \ namespace fc { template \ struct serialize_as_string : std::true_type {}; } diff --git a/libraries/libfc/include/fc/variant.hpp b/libraries/libfc/include/fc/variant.hpp index 7541f2c3ce..0211988d02 100644 --- a/libraries/libfc/include/fc/variant.hpp +++ b/libraries/libfc/include/fc/variant.hpp @@ -48,11 +48,10 @@ namespace fc template struct safe; // Streaming JSON writer overloads for variant / variant_object; implemented in - // variant.cpp / variant_object.cpp. Provides a seamless path for reflected structs - // that embed a fc::variant or variant_object field. Performance is bounded by the - // underlying fc::json::to_string since variants can hold arbitrary shapes; callers - // that want the full allocation-free path should flatten variant usage out of their - // hot endpoints. + // variant.cpp / variant_object.cpp. Walks the variant tree directly into the writer + // -- no intermediate fc::json::to_string string allocation. Lets reflected structs + // that embed a variant or variant_object field stream through the same writer as + // their other members. void to_json_stream( const variant& v, json_writer& w ); void to_json_stream( const variant_object& vo, json_writer& w ); void to_json_stream( const mutable_variant_object& vo, json_writer& w ); @@ -766,6 +765,14 @@ namespace fc v = std::vector( static_cast(bi.data()), static_cast(bi.data()) + sizeof(bi) ); } + /// JSON shape mirrors to_variant: lowercase hex string (not the generic + /// std::array template's array-of-int8 form). + template + void to_json_stream( const std::array& bi, json_writer& w ) + { + w.value_hex( bi.data(), N ); + } + /** @ingroup Serializable */ template void from_variant( const variant& v, std::array& bi ) diff --git a/libraries/libfc/src/crypto/bls_signature.cpp b/libraries/libfc/src/crypto/bls_signature.cpp index 47e59e58bd..f260ddf220 100644 --- a/libraries/libfc/src/crypto/bls_signature.cpp +++ b/libraries/libfc/src/crypto/bls_signature.cpp @@ -60,4 +60,4 @@ std::string aggregate_signature::to_string() const { return bls::constants::signature_prefix + data_str; } -} // fc::crypto::bls \ No newline at end of file +} // fc::crypto::bls diff --git a/libraries/libfc/src/crypto/keccak256.cpp b/libraries/libfc/src/crypto/keccak256.cpp index fc4ac97838..5e7a8b1eee 100644 --- a/libraries/libfc/src/crypto/keccak256.cpp +++ b/libraries/libfc/src/crypto/keccak256.cpp @@ -59,4 +59,4 @@ void keccak256::encoder::reset() { data.clear(); } -} // namespace fc::crypto \ No newline at end of file +} // namespace fc::crypto diff --git a/libraries/libfc/src/variant.cpp b/libraries/libfc/src/variant.cpp index c6a9094bd9..a0bf7dc7b1 100644 --- a/libraries/libfc/src/variant.cpp +++ b/libraries/libfc/src/variant.cpp @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include diff --git a/libraries/libfc/test/io/test_json_stream.cpp b/libraries/libfc/test/io/test_json_stream.cpp index 4d29b31d48..13133d0828 100644 --- a/libraries/libfc/test/io/test_json_stream.cpp +++ b/libraries/libfc/test/io/test_json_stream.cpp @@ -1,19 +1,35 @@ #include +#include +#include +#include #include #include #include #include +#include +#include +#include #include +#include +#include #include #include +#include #include #include +#include + +#include +#include #include +#include #include #include +#include #include +#include #include namespace { @@ -69,6 +85,91 @@ BOOST_AUTO_TEST_CASE(primitives) { BOOST_CHECK_EQUAL(fc::to_json_string(std::string{"hi"}), "\"hi\""); } +BOOST_AUTO_TEST_CASE(array_of_char_emits_hex_string) { + // std::array serializes as a lowercase hex string in fc::variant + // (matches vector); streaming must produce the same shape, not an + // array of byte-numbers from the generic std::array template. + auto via_variant = [](const auto& v) { return fc::json::to_string(fc::variant(v), fc::json::yield_function_t()); }; + + std::array bytes{0x00, 0x01, 0x02, 0x03}; + BOOST_CHECK_EQUAL(fc::to_json_string(bytes), "\"00010203\""); + BOOST_CHECK_EQUAL(fc::to_json_string(bytes), via_variant(bytes)); + + std::array wider{}; + wider[0] = static_cast(0xff); wider[31] = 0x42; + BOOST_CHECK_EQUAL(fc::to_json_string(wider), via_variant(wider)); +} + +BOOST_AUTO_TEST_CASE(large_int_quoting_matches_variant) { + // Variant emits 64-bit integers with |v| > 0xffffffff as quoted strings to + // preserve precision past JS's 2^53 limit. Streaming must match. + auto via_variant = [](auto v) { return fc::json::to_string(fc::variant(v), fc::json::yield_function_t()); }; + + BOOST_CHECK_EQUAL(fc::to_json_string(int64_t{0xffffffff}), via_variant(int64_t{0xffffffff})); + BOOST_CHECK_EQUAL(fc::to_json_string(int64_t{0x100000000}), via_variant(int64_t{0x100000000})); + BOOST_CHECK_EQUAL(fc::to_json_string(int64_t{-0x100000000}), via_variant(int64_t{-0x100000000})); + BOOST_CHECK_EQUAL(fc::to_json_string(uint64_t{0xffffffff}), via_variant(uint64_t{0xffffffff})); + BOOST_CHECK_EQUAL(fc::to_json_string(uint64_t{0x100000000}), via_variant(uint64_t{0x100000000})); + BOOST_CHECK_EQUAL(fc::to_json_string(uint64_t{0xffffffffffffffff}), via_variant(uint64_t{0xffffffffffffffff})); +} + +BOOST_AUTO_TEST_CASE(value_double_locale_independent) { + // %g honors LC_NUMERIC: under de_DE the radix becomes ',' -> invalid JSON. + // value_double must use a locale-independent formatter. scoped_exit ensures + // the locale is restored even if a BOOST_CHECK throws. + const char* saved = std::setlocale(LC_NUMERIC, nullptr); + const std::string saved_locale = saved ? saved : "C"; + auto restore = fc::make_scoped_exit([&]{ std::setlocale(LC_NUMERIC, saved_locale.c_str()); }); + + const char* comma_locales[] = {"de_DE.UTF-8", "de_DE", "fr_FR.UTF-8", "fr_FR", "C-decimal-comma"}; + bool found = false; + for (const char* loc : comma_locales) { + if (std::setlocale(LC_NUMERIC, loc)) { found = true; break; } + } + if (!found) { + BOOST_TEST_MESSAGE("no comma-radix locale available; skipping locale-independence assertion"); + return; + } + BOOST_CHECK_EQUAL(fc::to_json_string(1.5), "1.5"); + BOOST_CHECK_EQUAL(fc::to_json_string(-2.25), "-2.25"); + // Round-trip: any finite double must parse back to its bit-exact value. + // strtod_l with the C locale avoids re-introducing the comma-radix issue. + auto round_trip = [](double d) { + const std::string s = fc::to_json_string(d); + double parsed = 0.0; + auto [_, ec] = std::from_chars(s.data(), s.data() + s.size(), parsed); + BOOST_REQUIRE(ec == std::errc{}); + BOOST_CHECK_EQUAL(parsed, d); + }; + round_trip(0.1); + round_trip(1.0 / 3.0); + round_trip(std::numeric_limits::min()); + round_trip(std::numeric_limits::max()); +} + +BOOST_AUTO_TEST_CASE(value_double_rejects_non_finite) { + // NaN / +-inf have no JSON representation; value_double must throw rather than + // emit unquoted nan/inf tokens that downstream parsers reject. + const auto nan = std::numeric_limits::quiet_NaN(); + const auto pos_inf = std::numeric_limits::infinity(); + const auto neg_inf = -std::numeric_limits::infinity(); + + std::string out; + { + fc::json_writer w(out); + BOOST_CHECK_THROW(w.value_double(nan), std::invalid_argument); + } + BOOST_CHECK(out.empty()); // nothing written, frame state untouched + { + fc::json_writer w(out); + BOOST_CHECK_THROW(w.value_double(pos_inf), std::invalid_argument); + BOOST_CHECK_THROW(w.value_double(neg_inf), std::invalid_argument); + } + // Finite extremes still round-trip. + BOOST_CHECK_EQUAL(fc::to_json_string(0.0), "0"); + BOOST_CHECK_EQUAL(fc::to_json_string(1.5), "1.5"); +} + BOOST_AUTO_TEST_CASE(string_escaping) { // Control characters must be JSON-escaped. BOOST_CHECK_EQUAL(fc::to_json_string(std::string{"a\"b"}), "\"a\\\"b\""); @@ -306,6 +407,75 @@ BOOST_AUTO_TEST_CASE(set_dispatches_via_to_json_stream) { BOOST_CHECK_EQUAL(out, expected); } +// Parity helper: streaming JSON for `v` must equal the variant-built JSON. +// Catches divergence between to_json_stream(T) and to_variant(T)+json::to_string +// for libfc leaf types whose HTTP-API consumers depend on the existing shape. +template +static void check_streaming_matches_variant(const T& v, const std::string& label) { + const std::string streamed = fc::to_json_string(v); + const std::string via_var = fc::json::to_string(fc::variant(v), fc::json::yield_function_t()); + BOOST_CHECK_MESSAGE(streamed == via_var, + label << ": streaming JSON differs from variant path\n streaming: " << streamed + << "\n variant: " << via_var); +} + +BOOST_AUTO_TEST_CASE(streaming_vs_variant_parity_libfc_leaf_types) { + // exception: shape comes from FC_REFLECT(fc::exception, ...) -- stack of log_messages. + { + fc::exception e(FC_LOG_MESSAGE(error, "test exception"), + fc::std_exception_code, "test_exception", "test what"); + check_streaming_matches_variant(e, "fc::exception"); + } + // log_message: timestamp + context + formatted message. + { + fc::log_message lm(FC_LOG_CONTEXT(warn), std::string("hello world")); + check_streaming_matches_variant(lm, "fc::log_message"); + } + // 128-bit integers: emitted as decimal strings. + { + check_streaming_matches_variant(fc::int128_t{-1234567890123456789LL}, "fc::int128 negative"); + check_streaming_matches_variant(fc::uint128_t{0xffffffffffffffffULL} * 2 + 1, "fc::uint128 large"); + } + // bls public_key + signature: derived from a deterministic seed via private_key::generate(). + // private_key itself has =delete'd to_json_stream so we don't include it here. + { + auto sk = fc::crypto::bls::private_key::generate(); + check_streaming_matches_variant(sk.get_public_key(), "fc::crypto::bls::public_key"); + const std::string msg = "ping"; + auto sig = sk.sign_raw(std::span(reinterpret_cast(msg.data()), msg.size())); + check_streaming_matches_variant(sig, "fc::crypto::bls::signature"); + } + // url: opaque string round-trip. + { + check_streaming_matches_variant(fc::url(std::string{"https://example.com/path?q=1"}), "fc::url"); + } + // bitset: '1'/'0' bit string via FC_SERIALIZE_AS_STRING. + { + check_streaming_matches_variant(fc::bitset(std::string_view{"1010110010"}), "fc::bitset"); + } + // enum_type<>: reflected enum should emit the member-name string, not the underlying integer. + { + using et = fc::enum_type; + check_streaming_matches_variant(et{color_t::green}, "fc::enum_type"); + } + // static_variant (std::variant alias): index + tagged value pair. + { + using sv_t = std::variant; + check_streaming_matches_variant(sv_t{int32_t{42}}, "static_variant alt 0"); + check_streaming_matches_variant(sv_t{std::string{"hi"}}, "static_variant alt 1"); + } + // vector: lowercase hex string. + { + std::vector v{0x00, 0x01, char(0xfe), char(0xff)}; + check_streaming_matches_variant(v, "std::vector"); + } + // blob: base64-encoded bytes. + { + fc::blob b{{'a', 'b', 'c'}}; + check_streaming_matches_variant(b, "fc::blob"); + } +} + BOOST_AUTO_TEST_CASE(set_raw_splices_preformatted_fragment) { // set_raw is "key + raw_value": for embedding a JSON fragment that was already // serialized elsewhere (eg the abi_serializer + json::to_string path). From 7e2cfd23b719aacbc04bc370e0cfe2961563703a Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Thu, 30 Apr 2026 11:57:55 -0500 Subject: [PATCH 52/71] chain: abi_serializer mid-stream throw rollback + cleanups The streaming sink had two mid-emit throw bugs: - add_value(action) called sink.key("data") BEFORE sink.unpack_action_data(...). If the inner unpack threw partway (truncated bytes / drifted ABI), the catch fell through to emit_action_bytes_field(sink, "data", ...) which called sink.key("data") AGAIN, producing duplicate keys + an orphan inner object frame. The variant_sink path didn't have this bug because emit_value commits the key+value atomically. - add_value(action_trace) had the same shape for return_value_data but the catch swallowed silently with NO fallback emit. After the throw the writer was left with a dangling key, then the subsequent end_object closed the orphaned inner frame instead of the outer action_trace -- truncating the entire enclosing JSON object. Fix: - New unpack_action_data_field(name, ...) helper on each sink: variant_sink builds the variant first (atomic emit on success; pending_key clear on throw) and stream_sink checkpoints the writer + frame_items_ before key emission so a throw rewinds both back to the pre-call state. Caller emits the hex fallback under the same key with no duplicate / orphan. - add_value(action) and add_value(action_trace) use the new helper. - Add abi_to_variant + abi_to_json_stream parity tests for both failure paths (action data truncated, action_result type mismatch). Other: - value_string(string_view) drops the std::string round-trip and uses fc::variant(string_view) directly (SSO-aware). - value_variant signature unified to const fc::variant& on both sinks (was const& on stream, by-value on variant; the if-constexpr workaround in _unpack_enum is gone). - Add stub add_specialized_stream_unpack with FC_ASSERT(not yet implemented) so future plugin/contract callers see the API gap at the call site rather than getting a silent variant-fallback (slow path) for their override. - symbol::from_string: parse precision via std::from_chars instead of fc::to_int64(std::string{prec_part}) -- locale-independent and drops the std::string round-trip. --- libraries/chain/abi_serializer.cpp | 73 +++++++++++-- .../include/sysio/chain/abi_serializer.hpp | 30 ++++-- .../chain/include/sysio/chain/abi_sinks.hpp | 20 +++- libraries/chain/symbol.cpp | 12 ++- unittests/abi_tests.cpp | 102 ++++++++++++++++++ 5 files changed, 215 insertions(+), 22 deletions(-) diff --git a/libraries/chain/abi_serializer.cpp b/libraries/chain/abi_serializer.cpp index 1cc822be1f..4c2589e5a8 100644 --- a/libraries/chain/abi_serializer.cpp +++ b/libraries/chain/abi_serializer.cpp @@ -256,6 +256,18 @@ namespace sysio::chain { built_in_types[name] = std::move( unpack_pack ); } + void abi_serializer::add_specialized_stream_unpack( const string& /*name*/, + stream_unpack_function /*unpack*/ ) { + // The streaming dispatch (get_built_in_stream_unpacks) is a static const map shared across + // instances. Wiring per-instance overrides would require restructuring that into an + // instance-owned map plus a fallback-to-static lookup, plus a parallel change in + // stream_sink::unpack_built_in to consult the instance map first. No callers need this + // today (add_specialized_unpack_pack itself has zero in-tree callers); intentional fail-loud + // until a real use case shows up. + FC_ASSERT(false, "abi_serializer::add_specialized_stream_unpack is not yet implemented; " + "see header doc for the restructuring required"); + } + const std::pair* abi_serializer::find_built_in(std::string_view type) const noexcept { auto it = built_in_types.find(type); @@ -1341,7 +1353,7 @@ void variant_sink::key(std::string_view k) { f.has_pending_key = true; } -void variant_sink::value_string(std::string_view s) { emit_value(fc::variant(std::string(s))); } +void variant_sink::value_string(std::string_view s) { emit_value(fc::variant(s)); } void variant_sink::value_int64(int64_t n) { emit_value(fc::variant(n)); } void variant_sink::value_uint64(uint64_t n) { emit_value(fc::variant(n)); } void variant_sink::value_double(double d) { emit_value(fc::variant(d)); } @@ -1401,6 +1413,28 @@ void variant_sink::unpack_action_data(const abi_serializer& abi, std::string_vie emit_value(abi._binary_to_variant(type, data, inner)); } +bool variant_sink::unpack_action_data_field(std::string_view key_name, const abi_serializer& abi, + std::string_view type, const bytes& data, + abi_traverse_context& ctx, bool short_path) { + // _binary_to_variant builds the variant before we touch the sink; if it throws + // pending_key was set by key() but never committed by emit_value, so we just + // clear it and the surviving frame is left as if neither call happened. + try { + key(key_name); + action_data_to_variant_context inner(abi, ctx, type); + inner.short_path = short_path; + emit_value(abi._binary_to_variant(type, data, inner)); + return true; + } catch(...) { + if (!stack_.empty()) { + auto& f = stack_.back(); + f.pending_key.clear(); + f.has_pending_key = false; + } + return false; + } +} + // -- stream_sink ------------------------------------------------------------------------- void stream_sink::begin_object() { @@ -1492,6 +1526,33 @@ void stream_sink::unpack_action_data(const abi_serializer& abi, std::string_view abi._binary_walk(type, ds, *this, inner); } +bool stream_sink::unpack_action_data_field(std::string_view key_name, const abi_serializer& abi, + std::string_view type, const bytes& data, + abi_traverse_context& ctx, bool short_path) { + // Snapshot writer + frame_items_ before the key emit; on throw we rewind both + // so the buffer / frame stack are byte-identical to the pre-call state. The + // walker may have opened nested object frames before throwing, so resizing + // frame_items_ back to its saved depth is required to keep it in lockstep + // with the json_writer's internal stack. + const auto writer_cp = w_.checkpoint(); + const size_t fi_save = frame_items_.size(); + const uint32_t items_save = frame_items_.empty() ? 0 : frame_items_.back(); + try { + w_.key(key_name); + action_data_to_variant_context inner(abi, ctx, type); + inner.short_path = short_path; + fc::datastream ds(data.data(), data.size()); + abi._binary_walk(type, ds, *this, inner); + on_value_emitted(); + return true; + } catch(...) { + w_.rewind(writer_cp); + frame_items_.resize(fi_save); + if (!frame_items_.empty()) frame_items_.back() = items_save; + return false; + } +} + } // namespace sysio::chain::impl // -- _binary_walk et al ------------------------------------------------------------ @@ -1515,13 +1576,9 @@ void abi_serializer::_unpack_enum( const enum_def& e_def, fc::datastream) { - sink.value_variant(std::move(int_var)); - } else { - sink.value_variant(int_var); - } + // Unknown value -- emit as the underlying integer. Both sinks take by const-ref + // and copy internally; the int_var temporary lives until end of expression. + sink.value_variant(int_var); } template diff --git a/libraries/chain/include/sysio/chain/abi_serializer.hpp b/libraries/chain/include/sysio/chain/abi_serializer.hpp index a87e2184df..f039f8f7ee 100644 --- a/libraries/chain/include/sysio/chain/abi_serializer.hpp +++ b/libraries/chain/include/sysio/chain/abi_serializer.hpp @@ -151,6 +151,15 @@ struct abi_serializer { void add_specialized_unpack_pack( const string& name, std::pair unpack_pack ); + /// Streaming-side companion to `add_specialized_unpack_pack`. Currently asserts + /// not-implemented: the streaming dispatch is a static const map and registering + /// per-instance overrides would require restructuring it into a per-instance map. + /// Surfaced now so future callers see the gap at the API level rather than getting + /// a silent variant-fallback (slow path) for their override. + using stream_unpack_function = std::function&, bool, bool, + const yield_function_t&, fc::json_writer&)>; + void add_specialized_stream_unpack( const string& name, stream_unpack_function unpack ); + /// Lookup helper used by the streaming walker sinks. Returns a pointer to the /// (unpack, pack) entry registered for `type`, or `nullptr` if it is not a /// built-in. Const view; the underlying map is shared with `binary_to_variant` @@ -627,6 +636,9 @@ namespace impl { return; } + // unpack_action_data_field is atomic: on failure the sink is rolled back + // so the hex fallback emits cleanly under the same key with no duplicate + // and no orphan inner frame. bool data_emitted = false; try { auto abi_optional = resolver(act.account); @@ -634,16 +646,10 @@ namespace impl { const abi_serializer& abi = *abi_optional; auto type = abi.get_action_type(act.name); if (!type.empty()) { - try { - sink.key("data"); - sink.unpack_action_data(abi, type, act.data, ctx, /*short_path*/ false); - data_emitted = true; - } catch(...) { - // serialization failure -- fall back to hex below - } + data_emitted = sink.unpack_action_data_field("data", abi, type, act.data, ctx, /*short_path*/ false); } } - } catch(...) { /* fall back to hex below */ } + } catch(...) { /* resolver lookup failure -- fall back to hex below */ } if( !data_emitted ) { emit_action_bytes_field( sink, "data", act.data, ctx ); } @@ -686,17 +692,19 @@ namespace impl { add( sink, "error_code", act_trace.error_code, resolver, ctx ); add( sink, "return_value_hex_data", act_trace.return_value, resolver, ctx ); + // Atomic key + unpack: on decode failure the field is silently omitted + // (matches existing variant-path behavior), with no orphan frame or + // dangling key left in the writer. try { auto abi_optional = resolver(act_trace.act.account); if (abi_optional) { const abi_serializer& abi = *abi_optional; auto type = abi.get_action_result_type(act_trace.act.name); if (!type.empty()) { - sink.key("return_value_data"); - sink.unpack_action_data(abi, type, act_trace.return_value, ctx, /*short_path*/ false); + sink.unpack_action_data_field("return_value_data", abi, type, act_trace.return_value, ctx, /*short_path*/ false); } } - } catch(...) {} + } catch(...) { /* resolver lookup failure -- omit field */ } sink.end_object(); } diff --git a/libraries/chain/include/sysio/chain/abi_sinks.hpp b/libraries/chain/include/sysio/chain/abi_sinks.hpp index c3047b9b17..58cd06d20f 100644 --- a/libraries/chain/include/sysio/chain/abi_sinks.hpp +++ b/libraries/chain/include/sysio/chain/abi_sinks.hpp @@ -83,7 +83,9 @@ class variant_sink { /// Inject an already-built `fc::variant` at the current value position. Used by /// `unpack_built_in` and `unpack_protobuf` to avoid re-tokenising values that the /// existing variant-side unpack functions produce as a single `fc::variant`. - void value_variant(fc::variant v) { emit_value(std::move(v)); } + /// By const-ref to match `stream_sink::value_variant`; copies before emit so the + /// sink owns its frame entry. + void value_variant(const fc::variant& v) { emit_value(v); } /// Generic field emit used by `abi_to_emit` for non-ABI types: builds a /// `fc::variant(v)` and emits it at the current value position. Symmetric with @@ -98,6 +100,14 @@ class variant_sink { void unpack_action_data(const abi_serializer& abi, std::string_view type, const bytes& data, abi_traverse_context& ctx, bool short_path); + /// Atomic key + unpack_action_data. Returns true on success. On unpack failure, + /// the sink is rolled back to the state before the call -- no partial key, + /// no orphan inner frame -- so the caller can emit a hex fallback under the + /// same key without producing duplicates or malformed JSON. + bool unpack_action_data_field(std::string_view key_name, const abi_serializer& abi, + std::string_view type, const bytes& data, + abi_traverse_context& ctx, bool short_path); + /// True if at least one item has been added to the current frame. Used by the walker /// to enforce the legacy "Unable to unpack '...' from stream" guard on empty structs. bool frame_has_items() const noexcept; @@ -190,6 +200,14 @@ class stream_sink { void unpack_action_data(const abi_serializer& abi, std::string_view type, const bytes& data, abi_traverse_context& ctx, bool short_path); + /// Atomic key + unpack_action_data. Returns true on success. On unpack failure, + /// rewinds the writer (truncates the buffer + restores frame state) so the caller + /// can emit a hex fallback under the same key without leaving a half-written + /// "data":{ ... orphan or producing duplicate keys. + bool unpack_action_data_field(std::string_view key_name, const abi_serializer& abi, + std::string_view type, const bytes& data, + abi_traverse_context& ctx, bool short_path); + bool frame_has_items() const noexcept; void unpack_built_in(std::string_view ftype, fc::datastream& stream, diff --git a/libraries/chain/symbol.cpp b/libraries/chain/symbol.cpp index 7a42fb6d8f..324b45490d 100644 --- a/libraries/chain/symbol.cpp +++ b/libraries/chain/symbol.cpp @@ -1,8 +1,11 @@ #include #include +#include +#include + namespace sysio::chain { - + symbol symbol::from_string(std::string_view from) { try { @@ -11,7 +14,12 @@ namespace sysio::chain { auto comma_pos = s.find(','); SYS_ASSERT(comma_pos != std::string_view::npos, symbol_type_exception, "missing comma in symbol"); std::string_view prec_part = s.substr(0, comma_pos); - uint8_t p = fc::to_int64(std::string{prec_part}); + // std::from_chars is locale-independent and avoids the std::string round-trip + // that fc::to_int64(const std::string&) would otherwise force. + uint8_t p = 0; + auto [ptr, ec] = std::from_chars(prec_part.data(), prec_part.data() + prec_part.size(), p); + SYS_ASSERT(ec == std::errc{} && ptr == prec_part.data() + prec_part.size(), + symbol_type_exception, "precision is not a valid unsigned integer"); std::string_view name_part = s.substr(comma_pos + 1); SYS_ASSERT( p <= max_precision, symbol_type_exception, "precision {} should be <= 18", p); return symbol(string_to_symbol(p, name_part)); diff --git a/unittests/abi_tests.cpp b/unittests/abi_tests.cpp index 869321023b..3d6efbd3a5 100644 --- a/unittests/abi_tests.cpp +++ b/unittests/abi_tests.cpp @@ -3520,6 +3520,108 @@ BOOST_AUTO_TEST_CASE(abi_to_variant__add_action__no_return_value) } } +// When the action_result decode fails partway, the field is silently omitted +// from the JSON (existing variant-path behavior). The streaming sink had a +// bug where sink.key("return_value_data") was emitted BEFORE the inner unpack, +// so a mid-stream throw left a dangling key + orphan frame and end_object() +// closed the wrong scope. Stream + variant must produce identical JSON. +BOOST_AUTO_TEST_CASE(abi_to_variant__add_action_trace__return_value_unpack_throws_silently_omitted) +{ + action_trace at; + std::string expected_json; + // Set up a return_value whose declared type is "string" (length-prefix + body) + // but the bytes are a single 0x09 length-prefix with no body -> unpack throws + // partway. generate_action_trace's parsable=false branch suppresses the + // return_value_data field in expected_json, which is what we want. + std::tie(at, expected_json) = generate_action_trace(std::optional{0x09}, "09", false); + + auto abi = R"({ + "version": "sysio::abi/1.0", + "structs": [ + {"name": "acttest", "base": "", "fields": [ + {"name": "str", "type": "string"} + ]} + ], + "action_results": [ + {"name": "acttest", "result_type": "string"} + ] + })"; + auto abidef = fc::json::from_string(abi).as(); + + // Variant path: known good, builds expected_json (return_value_data omitted). + fc::variant variant_v; + abi_serializer::to_variant(at, variant_v, get_resolver(abidef), yield_fn()); + mutable_variant_object variant_mvo; + variant_mvo("action_traces", std::move(variant_v)); + const std::string variant_json = fc::json::to_string(variant_mvo, get_deadline()); + BOOST_CHECK_EQUAL(variant_json, expected_json); + + // Streaming path: must emit byte-identical JSON, with no orphan frame. + std::string stream_json; + { + fc::json_writer w(stream_json); + w.begin_object(); + w.key("action_traces"); + abi_serializer::to_json_stream(at, w, get_resolver(abidef), yield_fn()); + w.end_object(); + BOOST_REQUIRE(w.balanced()); + } + BOOST_CHECK_EQUAL(stream_json, expected_json); + BOOST_CHECK_NO_THROW(fc::json::from_string(stream_json)); +} + +// When action data fails ABI decode partway through (truncated bytes, missing +// fields, etc.), both the variant path and the streaming path must fall back +// to the hex-only form. The streaming sink had a bug where sink.key("data") +// was emitted BEFORE the inner unpack, so a mid-stream throw left the writer +// in a half-written state and the catch-side hex fallback emitted a duplicate +// "data" key plus an orphan inner object frame, producing invalid JSON. +BOOST_AUTO_TEST_CASE(abi_to_variant__add_action__data_unpack_throws_falls_back_to_hex) +{ + auto abi = R"({ + "version": "sysio::abi/1.0", + "structs": [ + {"name": "acttest", "base": "", "fields": [ + {"name": "str", "type": "string"} + ]} + ], + "actions": [ + {"name": "acttest", "type": "acttest", "ricardian_contract": ""} + ] + })"; + auto abidef = fc::json::from_string(abi).as(); + + // String length prefix says 9 bytes but no body follows -> unpack throws partway. + action act( + std::vector{ {account_name{"acctest"}, permission_name{"active"}} }, + account_name{"acctest"}, + action_name{"acttest"}, + bytes{0x09}); + + const std::string expected_hex = "09"; + const std::string expected_json = + "{\"account\":\"acctest\",\"name\":\"acttest\"," + "\"authorization\":[{\"actor\":\"acctest\",\"permission\":\"active\"}]," + "\"data\":\"" + expected_hex + "\"," + "\"hex_data\":\"" + expected_hex + "\"}"; + + { + fc::variant v; + abi_serializer::to_variant(act, v, get_resolver(abidef), yield_fn()); + const std::string s = fc::json::to_string(v, get_deadline()); + BOOST_CHECK_EQUAL(s, expected_json); + } + { + std::string s; + fc::json_writer w(s); + abi_serializer::to_json_stream(act, w, get_resolver(abidef), yield_fn()); + BOOST_REQUIRE(w.balanced()); + BOOST_CHECK_EQUAL(s, expected_json); + // Round-trip parses cleanly (catches duplicate-key / orphan-frame regressions). + BOOST_CHECK_NO_THROW(fc::json::from_string(s)); + } +} + BOOST_AUTO_TEST_CASE(enum_types) { using sysio::testing::fc_exception_message_starts_with; From a2103b60d624fb8296ede05fa6e9c4d33058dd1a Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Thu, 30 Apr 2026 12:03:29 -0500 Subject: [PATCH 53/71] chain_plugin + http_plugin: PR-review followups chain_plugin: - streamed_processed_trace::to_json_stream: checkpoint the writer before the streaming abi_serializer call so a mid-emit throw can rewind to the pre-call state before the reflector-only fallback runs. Without this, the fallback continued writing into a partially-emitted object, producing malformed JSON. - read_only::get_account Phase-2 closure: re-wrap throws in account_query_exception so http clients see the same exception class they did pre-refactor (when the wrap was applied at the outer SYS_RETHROW_EXCEPTIONS). - push_recurse: replace assert(0) for the http_fwd alternative with SYS_THROW(plugin_exception, ...) so the failure is visible in NDEBUG builds; switch the inner lambda to rvalue-ref + std::move to match the next_function rvalue-only contract. - trx_retry tracked_transaction::memory_size: walk the action_traces vector to sum console + act.data + return_value sizes (plus the authorization / account_ram_deltas struct space) instead of just sizeof(*trace_ptr). Per-action heap dominates for chatty contracts; the prior heuristic undercounted and let the eviction loop fire too late. - trx_retry: fix the double-space in the storage-size error message. - test_trx_retry_db: update the lambda payload type from the stale unique_ptr to unique_ptr; switch lambda signatures to rvalue-ref to match the new next_function operator() contract. http_plugin: - handle_exception_stream: early-return + log when cb has been moved into a downstream lambda. Avoids a bad_function_call thrown on top of the original exception (which classify_current_exception would log instead, leaving the client hanging to its timeout). - add_handler / add_async_handler: assert disjointness against the streaming map (symmetric with the existing variant-side check in add_handler_stream / add_async_handler_stream). A path registered on the streaming map first would silently shadow a later variant registration since lookup checks streaming first. - bind_stream: tighten compute_deadline SFINAE to require start()'s return type be convertible to fc::time_point (was an unconstrained requires { handle.start(); }). - bind_stream: rvalue-only signature on the dispatch::async typed- result lambda, with std::get(std::move(result)) so the payload moves out of the variant instead of copying. Add a static_assert on is_move_constructible to document the contract. - bind_stream: split_path takes string_view; assert the "/api/call" shape so a malformed registration path is caught at startup. - bind_stream: prefer std::unreachable() over __builtin_unreachable. - bind_stream::async arity-1 path: invoke the method directly instead of the circuitous invoke_async(handle, std::monostate{}, ...) helper that drops the placeholder. - common.hpp: drop redundant body.reserve(4096); json_writer's ctor already reserves if there's no slack. - http_plugin: stream registration log uses the same format as the variant registration log (no more "(stream)" marker). - test_chain_plugin: drop duplicate using namespace sysio::chain_apis. --- plugins/chain_plugin/src/chain_plugin.cpp | 47 +++++++++++++------ plugins/chain_plugin/src/trx_retry_db.cpp | 25 +++++++--- .../chain_plugin/test/test_trx_retry_db.cpp | 46 +++++++++--------- .../include/sysio/http_plugin/bind_stream.hpp | 37 ++++++++++----- .../include/sysio/http_plugin/common.hpp | 1 - plugins/http_plugin/src/http_plugin.cpp | 19 +++++++- tests/test_chain_plugin.cpp | 1 - 7 files changed, 116 insertions(+), 60 deletions(-) diff --git a/plugins/chain_plugin/src/chain_plugin.cpp b/plugins/chain_plugin/src/chain_plugin.cpp index a670070394..fdd1c0f1c5 100644 --- a/plugins/chain_plugin/src/chain_plugin.cpp +++ b/plugins/chain_plugin/src/chain_plugin.cpp @@ -3055,22 +3055,27 @@ void read_write::push_transaction(const read_write::push_transaction_params& par } static void push_recurse(read_write* rw, int index, const std::shared_ptr& params, const std::shared_ptr& results, const next_function& next) { - auto wrapped_next = [=](const next_function_variant& result) { + auto wrapped_next = [=](next_function_variant&& result) { if (std::holds_alternative(result)) { const auto& e = std::get(result); results->emplace_back( read_write::push_transaction_results{ transaction_id_type(), fc::mutable_variant_object( "error", e->to_detail_string() ) } ); } else if (std::holds_alternative(result)) { - const auto& r = std::get(result); - results->emplace_back( r ); + results->emplace_back( std::get(std::move(result)) ); } else { - assert(0); + // push_transaction's next_function carries an http_fwd alternative for the streaming + // dispatch path; push_transaction itself has not been migrated to emit one, so reaching + // this branch indicates a future API change that broke this assumption. Throw rather + // than assert so the failure is visible in NDEBUG builds. + SYS_THROW(plugin_exception, + "push_recurse: unsupported next_function_variant alternative (http_fwd). " + "push_transaction must emit either a fc::exception_ptr or push_transaction_results."); } size_t next_index = index + 1; if (next_index < params->size()) { push_recurse(rw, next_index, params, results, next ); } else { - next(*results); + next(read_write::push_transactions_results{*results}); } }; @@ -3433,17 +3438,23 @@ read_only::get_account_return_t read_only::get_account( const get_account_params abi_serializer_max_time = abi_serializer_max_time]() mutable -> chain::t_or_exception { try { - abi_def abi; - if (abi_serializer::to_abi(abi_bytes, abi)) { - auto yield = [&]() { return abi_serializer::create_yield_function(abi_serializer_max_time); }; - abi_serializer abis(std::move(abi), yield()); - if (http_params.total_resources) { - result.total_resources = abis.binary_to_variant("reslimit", - *http_params.total_resources, - yield(), shorten_abi_errors); + // Mirror the outer SYS_RETHROW_EXCEPTIONS(account_query_exception, ...) so that a throw + // on the http thread is reported to the client with the same exception class clients + // were seeing pre-refactor. Without the inner wrap, CATCH_AND_RETURN would surface + // raw abi_exception / abi_serialization_deadline_exception instead. + try { + abi_def abi; + if (abi_serializer::to_abi(abi_bytes, abi)) { + auto yield = [&]() { return abi_serializer::create_yield_function(abi_serializer_max_time); }; + abi_serializer abis(std::move(abi), yield()); + if (http_params.total_resources) { + result.total_resources = abis.binary_to_variant("reslimit", + *http_params.total_resources, + yield(), shorten_abi_errors); + } } - } - return std::move(result); + return std::move(result); + } SYS_RETHROW_EXCEPTIONS(chain::account_query_exception, "unable to retrieve account info") } CATCH_AND_RETURN(chain::t_or_exception); }; } @@ -3594,10 +3605,16 @@ void to_json_stream(const sysio::chain_apis::streamed_processed_trace& t, fc::js w.value_null(); return; } + // Checkpoint before the streaming attempt so a mid-emit throw can rewind the writer + // (truncate buffer + restore frame state) before the reflector-only fallback runs. + // Without this, the fallback would continue writing into a partially-emitted object, + // producing malformed JSON. + const auto cp = w.checkpoint(); try { auto resolver = sysio::build_resolver_from_captured_abis(t.raw_abis, t.max_serialization_time); sysio::chain::abi_serializer::to_json_stream(*t.trace, w, resolver, t.max_serialization_time); } catch (sysio::chain::abi_exception&) { + w.rewind(cp); // Fall back: emit via reflector with no ABI-aware action data decode. Matches the prior variant path's // `output = *trx_trace_ptr;` fallback. to_json_stream(*t.trace, w); diff --git a/plugins/chain_plugin/src/trx_retry_db.cpp b/plugins/chain_plugin/src/trx_retry_db.cpp index b0f9a58ef5..d28858bd14 100644 --- a/plugins/chain_plugin/src/trx_retry_db.cpp +++ b/plugins/chain_plugin/src/trx_retry_db.cpp @@ -53,14 +53,27 @@ struct tracked_transaction { return block_num != 0; } - /// Heuristic for the memory budget: 2x the packed-trx size accounts for the action trace bulk (each action trace - /// mirrors the packed action plus per-action receipt/console/etc), plus the captured raw-ABI bytes, plus the trace - /// struct overhead. + /// Heuristic for the memory budget. Sums the dominant heap contributors so the + /// retry container's eviction loop fires when the real footprint -- not just the + /// packed-trx size -- exceeds the cap. Per-action heap (console / data / return_value) + /// dwarfs the packed bytes for chatty contracts, so we walk the action_traces. size_t memory_size()const { const size_t ptrx_sz = ptrx->get_estimated_size(); - return ptrx_sz * 2 + size_t trace_sz = 0; + if (trace_ptr) { + trace_sz = sizeof(*trace_ptr); + for (const auto& at : trace_ptr->action_traces) { + trace_sz += sizeof(at) + + at.console.size() + + at.act.data.size() + + at.return_value.size() + + at.act.authorization.size() * sizeof(chain::permission_level) + + at.account_ram_deltas.size() * sizeof(chain::account_delta); + } + } + return ptrx_sz + sysio::estimated_size(raw_abis) - + (trace_ptr ? sizeof(*trace_ptr) : 0) + + trace_sz + sizeof(*this); } }; @@ -118,7 +131,7 @@ struct trx_retry_db_impl { void track_transaction( packed_transaction_ptr ptrx, std::optional num_blocks, next_function> next ) { SYS_ASSERT( _tracked_trxs.memory_size() < _max_mem_usage_size, tx_resource_exhaustion, - "Transaction exceeded transaction-retry-max-storage-size-gb limit: {} bytes", _tracked_trxs.memory_size() ); + "Transaction exceeded transaction-retry-max-storage-size-gb limit: {} bytes", _tracked_trxs.memory_size() ); auto i = _tracked_trxs.index().get().find( ptrx->id() ); if( i == _tracked_trxs.index().end() ) { _tracked_trxs.insert( {std::move(ptrx), diff --git a/plugins/chain_plugin/test/test_trx_retry_db.cpp b/plugins/chain_plugin/test/test_trx_retry_db.cpp index 667f7a4fdf..e0984582c1 100644 --- a/plugins/chain_plugin/test/test_trx_retry_db.cpp +++ b/plugins/chain_plugin/test/test_trx_retry_db.cpp @@ -229,14 +229,14 @@ BOOST_AUTO_TEST_CASE(trx_retry_logic) { auto lib = std::optional{}; auto trx_1 = make_unique_trx(chain->get_chain_id(), fc::seconds(2), 1); bool trx_1_expired = false; - trx_retry.track_transaction( trx_1, lib, [&trx_1_expired](const next_function_variant>& result){ + trx_retry.track_transaction( trx_1, lib, [&trx_1_expired](next_function_variant>&& result){ BOOST_REQUIRE( std::holds_alternative(result) ); BOOST_CHECK_EQUAL( std::get(result)->code(), expired_tx_exception::code_value ); trx_1_expired = true; } ); auto trx_2 = make_unique_trx(chain->get_chain_id(), fc::seconds(4), 2); bool trx_2_expired = false; - trx_retry.track_transaction( trx_2, lib, [&trx_2_expired](const next_function_variant>& result){ + trx_retry.track_transaction( trx_2, lib, [&trx_2_expired](next_function_variant>&& result){ BOOST_REQUIRE( std::holds_alternative(result) ); BOOST_CHECK_EQUAL( std::get(result)->code(), expired_tx_exception::code_value ); trx_2_expired = true; @@ -275,7 +275,7 @@ BOOST_AUTO_TEST_CASE(trx_retry_logic) { // auto trx_3 = make_unique_trx(chain->get_chain_id(), fc::seconds(30), 3); bool trx_3_expired = false; - trx_retry.track_transaction( trx_3, lib, [&trx_3_expired](const next_function_variant>& result){ + trx_retry.track_transaction( trx_3, lib, [&trx_3_expired](next_function_variant>&& result){ BOOST_REQUIRE( std::holds_alternative(result) ); BOOST_CHECK_EQUAL( std::get(result)->code(), expired_tx_exception::code_value ); trx_3_expired = true; @@ -285,7 +285,7 @@ BOOST_AUTO_TEST_CASE(trx_retry_logic) { fc::mock_time_traits::set_now(pnow); auto trx_4 = make_unique_trx(chain->get_chain_id(), fc::seconds(30), 4); bool trx_4_expired = false; - trx_retry.track_transaction( trx_4, lib, [&trx_4_expired](const next_function_variant>& result){ + trx_retry.track_transaction( trx_4, lib, [&trx_4_expired](next_function_variant>&& result){ BOOST_REQUIRE( std::holds_alternative(result) ); BOOST_CHECK_EQUAL( std::get(result)->code(), expired_tx_exception::code_value ); trx_4_expired = true; @@ -328,9 +328,9 @@ BOOST_AUTO_TEST_CASE(trx_retry_logic) { // auto trx_5 = make_unique_trx(chain->get_chain_id(), fc::seconds(30), 5); bool trx_5_variant = false; - trx_retry.track_transaction( trx_5, lib, [&trx_5_variant](const next_function_variant>& result){ - BOOST_REQUIRE( std::holds_alternative>(result) ); - BOOST_CHECK( !!std::get>(result) ); + trx_retry.track_transaction( trx_5, lib, [&trx_5_variant](next_function_variant>&& result){ + BOOST_REQUIRE( std::holds_alternative>(result) ); + BOOST_CHECK( !!std::get>(result) ); trx_5_variant = true; } ); // increase time by 1 seconds, so trx_6 retry interval diff than 5 @@ -338,9 +338,9 @@ BOOST_AUTO_TEST_CASE(trx_retry_logic) { fc::mock_time_traits::set_now(pnow); auto trx_6 = make_unique_trx(chain->get_chain_id(), fc::seconds(30), 6); bool trx_6_variant = false; - trx_retry.track_transaction( trx_6, std::optional(2), [&trx_6_variant](const next_function_variant>& result){ - BOOST_REQUIRE( std::holds_alternative>(result) ); - BOOST_CHECK( !!std::get>(result) ); + trx_retry.track_transaction( trx_6, std::optional(2), [&trx_6_variant](next_function_variant>&& result){ + BOOST_REQUIRE( std::holds_alternative>(result) ); + BOOST_CHECK( !!std::get>(result) ); trx_6_variant = true; } ); // not in block 7, so not returned to user @@ -397,9 +397,9 @@ BOOST_AUTO_TEST_CASE(trx_retry_logic) { // auto trx_7 = make_unique_trx(chain->get_chain_id(), fc::seconds(30), 7); bool trx_7_variant = false; - trx_retry.track_transaction( trx_7, lib, [&trx_7_variant](const next_function_variant>& result){ - BOOST_REQUIRE( std::holds_alternative>(result) ); - BOOST_CHECK( !!std::get>(result) ); + trx_retry.track_transaction( trx_7, lib, [&trx_7_variant](next_function_variant>&& result){ + BOOST_REQUIRE( std::holds_alternative>(result) ); + BOOST_CHECK( !!std::get>(result) ); trx_7_variant = true; } ); // increase time by 1 seconds, so trx_8 retry interval diff than 7 @@ -407,15 +407,15 @@ BOOST_AUTO_TEST_CASE(trx_retry_logic) { fc::mock_time_traits::set_now(pnow); auto trx_8 = make_unique_trx(chain->get_chain_id(), fc::seconds(30), 8); bool trx_8_variant = false; - trx_retry.track_transaction( trx_8, std::optional(3), [&trx_8_variant](const next_function_variant>& result){ - BOOST_REQUIRE( std::holds_alternative>(result) ); - BOOST_CHECK( !!std::get>(result) ); + trx_retry.track_transaction( trx_8, std::optional(3), [&trx_8_variant](next_function_variant>&& result){ + BOOST_REQUIRE( std::holds_alternative>(result) ); + BOOST_CHECK( !!std::get>(result) ); trx_8_variant = true; } ); // one to expire, will be forked out never to return auto trx_9 = make_unique_trx(chain->get_chain_id(), fc::seconds(30), 9); bool trx_9_expired = false; - trx_retry.track_transaction( trx_9, lib, [&trx_9_expired](const next_function_variant>& result){ + trx_retry.track_transaction( trx_9, lib, [&trx_9_expired](next_function_variant>&& result){ BOOST_REQUIRE( std::holds_alternative(result) ); BOOST_CHECK_EQUAL( std::get(result)->code(), expired_tx_exception::code_value ); trx_9_expired = true; @@ -556,16 +556,16 @@ BOOST_AUTO_TEST_CASE(trx_retry_logic) { // auto trx_10 = make_unique_trx(chain->get_chain_id(), fc::seconds(30), 10); bool trx_10_variant = false; - trx_retry.track_transaction( trx_10, std::optional(0), [&trx_10_variant](const next_function_variant>& result){ - BOOST_REQUIRE( std::holds_alternative>(result) ); - BOOST_CHECK( !!std::get>(result) ); + trx_retry.track_transaction( trx_10, std::optional(0), [&trx_10_variant](next_function_variant>&& result){ + BOOST_REQUIRE( std::holds_alternative>(result) ); + BOOST_CHECK( !!std::get>(result) ); trx_10_variant = true; } ); auto trx_11 = make_unique_trx(chain->get_chain_id(), fc::seconds(30), 11); bool trx_11_variant = false; - trx_retry.track_transaction( trx_11, std::optional(1), [&trx_11_variant](const next_function_variant>& result){ - BOOST_REQUIRE( std::holds_alternative>(result) ); - BOOST_CHECK( !!std::get>(result) ); + trx_retry.track_transaction( trx_11, std::optional(1), [&trx_11_variant](next_function_variant>&& result){ + BOOST_REQUIRE( std::holds_alternative>(result) ); + BOOST_CHECK( !!std::get>(result) ); trx_11_variant = true; } ); // seen in block immediately diff --git a/plugins/http_plugin/include/sysio/http_plugin/bind_stream.hpp b/plugins/http_plugin/include/sysio/http_plugin/bind_stream.hpp index a414c78639..06bb4bcc7d 100644 --- a/plugins/http_plugin/include/sysio/http_plugin/bind_stream.hpp +++ b/plugins/http_plugin/include/sysio/http_plugin/bind_stream.hpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -101,12 +102,13 @@ namespace http_detail { /// Compute the per-request deadline. Chain api handles expose `start()` which both /// returns the deadline and registers the in-flight call; other handles fall back to - /// `http_plugin::get_max_response_time()`. + /// `http_plugin::get_max_response_time()`. Constraint requires `start()` return to + /// be convertible to fc::time_point so an unrelated `start()` does not match. template inline fc::time_point compute_deadline(Handle& handle, http_plugin& http) { - if constexpr (requires { handle.start(); }) { + if constexpr (requires { { handle.start() } -> std::convertible_to; }) { return handle.start(); - } else if constexpr (requires { handle->start(); }) { + } else if constexpr (requires { { handle->start() } -> std::convertible_to; }) { return handle->start(); } else { const fc::microseconds m = http.get_max_response_time(); @@ -127,17 +129,20 @@ namespace http_detail { case http_params_types::possible_no_params: return parse_params(body); } - __builtin_unreachable(); + std::unreachable(); } /// Parse a registration path of the form "/v1//" into the - /// (api_name, call_name) labels used by `handle_exception_stream`. - inline std::pair split_path(const std::string& path) { + /// (api_name, call_name) labels used by `handle_exception_stream`. Returns + /// owning std::string because the caller captures the labels into a long-lived + /// registration lambda. + inline std::pair split_path(std::string_view path) { auto last_slash = path.find_last_of('/'); - auto prev_slash = path.find_last_of('/', last_slash > 0 ? last_slash - 1 : 0); - std::string api_name = path.substr(prev_slash + 1, last_slash - prev_slash - 1); - std::string call_name = path.substr(last_slash + 1); - return { std::move(api_name), std::move(call_name) }; + assert(last_slash != std::string_view::npos && last_slash > 0); // expects "/api/call" shape + auto prev_slash = path.find_last_of('/', last_slash - 1); + assert(prev_slash != std::string_view::npos); + return { std::string(path.substr(prev_slash + 1, last_slash - prev_slash - 1)), + std::string(path.substr(last_slash + 1)) }; } /// Invoke `MethodPtr` against `handle` with the right argument shape: @@ -357,10 +362,15 @@ api_entry_stream bind_stream(http_plugin& http, Handle handle, using payload_t = typename is_next_fn< std::remove_cvref_t>>::payload; using http_fwd_t = std::function()>; + // The next_function operator() is rvalue-only; move out of the variant into + // the response closure so the heavy payload (raw ABIs, traces) is moved, not + // copied, before the closure runs on the http pool. + static_assert(std::is_move_constructible_v, + "bind_stream::async requires the next_function payload to be move-constructible."); auto next = [&http, cb = std::move(cb), body = std::move(body), api_name, call_name, resp_code] - (const chain::plugin_interface::next_function_variant& result) mutable { + (chain::plugin_interface::next_function_variant&& result) mutable { if (std::holds_alternative(result)) { try { throw *std::get(result); } catch (...) { @@ -369,7 +379,7 @@ api_entry_stream bind_stream(http_plugin& http, Handle handle, } } else if (std::holds_alternative(result)) { cb(resp_code, - [r = std::get(result)] + [r = std::get(std::move(result))] (fc::json_writer& w) mutable { fc::to_json_stream(r, w); }); } else { http.post_http_thread_pool( @@ -404,7 +414,8 @@ api_entry_stream bind_stream(http_plugin& http, Handle handle, if constexpr (has_params) { invoke_async(handle, std::move(params), std::move(next)); } else { - invoke_async(handle, std::monostate{}, std::move(next)); + // Arity-1 async: no params slot, invoke the method directly. + std::invoke(MethodPtr, handle, std::move(next)); } } } catch (...) { diff --git a/plugins/http_plugin/include/sysio/http_plugin/common.hpp b/plugins/http_plugin/include/sysio/http_plugin/common.hpp index 772c9d685f..559a9029c9 100644 --- a/plugins/http_plugin/include/sysio/http_plugin/common.hpp +++ b/plugins/http_plugin/include/sysio/http_plugin/common.hpp @@ -256,7 +256,6 @@ inline auto make_http_stream_response_handler(http_plugin_state& plugin_state, d [&plugin_state, session_ptr{std::move(session_ptr)}, code, emitter{std::move(emitter)}]() mutable { try { std::string body; - body.reserve(4096); { fc::json_writer w(body); emitter(w); diff --git a/plugins/http_plugin/src/http_plugin.cpp b/plugins/http_plugin/src/http_plugin.cpp index fa6cfda50b..01b5205941 100644 --- a/plugins/http_plugin/src/http_plugin.cpp +++ b/plugins/http_plugin/src/http_plugin.cpp @@ -575,6 +575,11 @@ namespace sysio { void http_plugin::add_handler(api_entry&& entry, appbase::exec_queue q, int priority, http_content_type content_type) { log_add_handler(my.get(), entry); std::string path = entry.path; + // Symmetric with add_handler_stream: a path registered on the streaming map + // would silently shadow this variant registration since lookup checks the + // streaming map first. + SYS_ASSERT( !my->plugin_state->url_handlers_stream.contains(path), chain::plugin_config_exception, + "http url {} is not unique - already registered on the streaming cb path", path ); auto p = my->plugin_state->url_handlers.emplace(path, my->make_app_thread_url_handler(std::move(entry), q, priority, content_type)); SYS_ASSERT( p.second, chain::plugin_config_exception, "http url {} is not unique", path ); } @@ -582,6 +587,8 @@ namespace sysio { void http_plugin::add_async_handler(api_entry&& entry, http_content_type content_type) { log_add_handler(my.get(), entry); std::string path = entry.path; + SYS_ASSERT( !my->plugin_state->url_handlers_stream.contains(path), chain::plugin_config_exception, + "http url {} is not unique - already registered on the streaming cb path", path ); auto p = my->plugin_state->url_handlers.emplace(path, my->make_http_thread_url_handler(std::move(entry), content_type)); SYS_ASSERT( p.second, chain::plugin_config_exception, "http url {} is not unique", path ); } @@ -593,7 +600,7 @@ namespace sysio { addrs = "on " + addrs; else addrs = "disabled for category address not configured"; - fc_ilog(logger(), "add {} api url (stream): {} {}", + fc_ilog(logger(), "add {} api url: {} {}", from_category(entry.category), entry.path, addrs); } } @@ -693,6 +700,16 @@ namespace sysio { } void http_plugin::handle_exception_stream( const char* api_name, const char* call_name, const string& body, url_response_stream_callback& cb) { + // bind_stream's outer catch can land here AFTER cb has been moved into a + // downstream lambda (eg if post_http_thread_pool throws after the move). + // Calling an empty move_only_function would throw bad_function_call on top + // of the original exception; classify_current_exception would log THAT one, + // and the client would never get a response. Log the original exception + // and return so the original failure is at least visible. + if (!cb) { + classify_current_exception(api_name, call_name, body, [](int, error_results){}); + return; + } classify_current_exception(api_name, call_name, body, [&cb](int code, error_results r) { cb(code, [r = std::move(r)](fc::json_writer& w) mutable { fc::to_json_stream(r, w); diff --git a/tests/test_chain_plugin.cpp b/tests/test_chain_plugin.cpp index 0e9e99c440..caf211b929 100644 --- a/tests/test_chain_plugin.cpp +++ b/tests/test_chain_plugin.cpp @@ -19,7 +19,6 @@ using namespace sysio; using namespace sysio::chain; using namespace sysio::chain_apis; using namespace sysio::testing; -using namespace sysio::chain_apis; using namespace fc; using mvo = fc::mutable_variant_object; From cc0e85a092b4f24795d2ad5ba7df0f5a67971e6f Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Thu, 30 Apr 2026 13:44:13 -0500 Subject: [PATCH 54/71] libfc: to_json_stream(double/float) emits variant-equivalent quoted fixed-precision string Reflector-driven double / float fields were emitting unquoted shortest-roundtrip numbers via value_double, while the fc::variant path emits a JSON-quoted fixed-precision string (digits10 + 2 fractional digits, std::fixed). A real shape regression today: get_producers_result.total_producer_vote_weight is a double field exposed via /v1/chain/get_producers, which is bound on the streaming path (chain_api_plugin.cpp). Pre-PR clients see e.g. "123.45000000000000000"; post-PR they would have seen 123.45, silently breaking any consumer parsing the string form. Change the to_json_stream(double) overload to emit via value_string with a locale-independent std::to_chars(chars_format::fixed, digits10+2) format. to_json_stream(float) routes to the double overload. Non-finite values emit the literal "nan" / "inf" / "-inf" string to match variant. value_double itself is unchanged (raw number primitive; throws on non-finite for direct callers that want an unquoted number). Adds a double / float case to streaming_vs_variant_parity_libfc_leaf_types. Updates two existing tests that had locked in the unquoted shape. --- .../libfc/include/fc/reflect/json_stream.hpp | 27 +++++++++++++++++-- libraries/libfc/test/io/test_json_stream.cpp | 21 ++++++++++++--- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/libraries/libfc/include/fc/reflect/json_stream.hpp b/libraries/libfc/include/fc/reflect/json_stream.hpp index d090518abd..02f14904bd 100644 --- a/libraries/libfc/include/fc/reflect/json_stream.hpp +++ b/libraries/libfc/include/fc/reflect/json_stream.hpp @@ -5,6 +5,9 @@ #include #include +#include +#include +#include #include #include #include @@ -67,8 +70,28 @@ inline void to_json_stream(uint64_t n, json_writer& w) { w.value_uint64(n); } } -inline void to_json_stream(float f, json_writer& w) { w.value_double(static_cast(f)); } -inline void to_json_stream(double d, json_writer& w) { w.value_double(d); } +// double / float emit as a JSON-quoted fixed-precision string to match fc::variant's emission shape +// (variant.cpp s_fc_to_string + json.cpp double_type case). Reflector-driven struct fields (e.g. +// get_producers_result.total_producer_vote_weight) depend on this shape; emitting a bare JSON number +// would silently break clients parsing the string form. std::to_chars with chars_format::fixed is +// locale-independent (unlike stringstream / snprintf, which honor LC_NUMERIC). +inline constexpr int double_fixed_precision = std::numeric_limits::digits10 + 2; +// Largest finite double in fixed format: max_exponent10 integer digits + '.' + precision frac + sign + NUL slack. +inline constexpr size_t double_fixed_buf_size = std::numeric_limits::max_exponent10 + double_fixed_precision + 8; +inline void to_json_stream(double d, json_writer& w) { + if (!std::isfinite(d)) { + // Variant emits the literal "nan" / "inf" / "-inf" string for non-finite doubles; match that shape. + w.value_string(std::isnan(d) ? "nan" : (d > 0 ? "inf" : "-inf")); + return; + } + char buf[double_fixed_buf_size]; + auto r = std::to_chars(buf, buf + sizeof(buf), d, std::chars_format::fixed, double_fixed_precision); + assert(r.ec == std::errc{}); + w.value_string(std::string_view(buf, r.ptr - buf)); +} +inline void to_json_stream(float f, json_writer& w) { + to_json_stream(static_cast(f), w); +} inline void to_json_stream(std::string_view s, json_writer& w) { w.value_string(s); } inline void to_json_stream(const std::string& s, json_writer& w) { w.value_string(s); } inline void to_json_stream(const char* s, json_writer& w) { w.value_string(s ? std::string_view(s) : std::string_view()); } diff --git a/libraries/libfc/test/io/test_json_stream.cpp b/libraries/libfc/test/io/test_json_stream.cpp index 13133d0828..6a460881a0 100644 --- a/libraries/libfc/test/io/test_json_stream.cpp +++ b/libraries/libfc/test/io/test_json_stream.cpp @@ -165,9 +165,10 @@ BOOST_AUTO_TEST_CASE(value_double_rejects_non_finite) { BOOST_CHECK_THROW(w.value_double(pos_inf), std::invalid_argument); BOOST_CHECK_THROW(w.value_double(neg_inf), std::invalid_argument); } - // Finite extremes still round-trip. - BOOST_CHECK_EQUAL(fc::to_json_string(0.0), "0"); - BOOST_CHECK_EQUAL(fc::to_json_string(1.5), "1.5"); + // Finite values: to_json_stream(double) emits the quoted fixed-precision form + // matching the variant path's emission shape. + BOOST_CHECK_EQUAL(fc::to_json_string(0.0), "\"0.00000000000000000\""); + BOOST_CHECK_EQUAL(fc::to_json_string(1.5), "\"1.50000000000000000\""); } BOOST_AUTO_TEST_CASE(string_escaping) { @@ -374,7 +375,7 @@ BOOST_AUTO_TEST_CASE(set_chained_object) { .set("d", 3.5); w.end_object(); } - BOOST_CHECK_EQUAL(out, R"({"a":1,"b":"two","c":true,"d":3.5})"); + BOOST_CHECK_EQUAL(out, R"({"a":1,"b":"two","c":true,"d":"3.50000000000000000"})"); } BOOST_AUTO_TEST_CASE(set_dispatches_via_to_json_stream) { @@ -436,6 +437,18 @@ BOOST_AUTO_TEST_CASE(streaming_vs_variant_parity_libfc_leaf_types) { check_streaming_matches_variant(fc::int128_t{-1234567890123456789LL}, "fc::int128 negative"); check_streaming_matches_variant(fc::uint128_t{0xffffffffffffffffULL} * 2 + 1, "fc::uint128 large"); } + // double / float: variant emits a JSON-quoted fixed-precision string + // (digits10 + 2, std::fixed) so wire-format clients see a stable shape + // regardless of magnitude. Reflector-driven struct fields (e.g. + // get_producers_result.total_producer_vote_weight) depend on this. + { + check_streaming_matches_variant(double{1.5}, "double 1.5"); + check_streaming_matches_variant(double{-2.25}, "double -2.25"); + check_streaming_matches_variant(double{0.1}, "double 0.1 (round-trip)"); + check_streaming_matches_variant(double{1e10}, "double 1e10"); + check_streaming_matches_variant(double{0.0}, "double 0.0"); + check_streaming_matches_variant(float{3.5f}, "float 3.5"); + } // bls public_key + signature: derived from a deterministic seed via private_key::generate(). // private_key itself has =delete'd to_json_stream so we don't include it here. { From f904fe49e22035a37d5c0000f1cbed50dd01402f Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Thu, 30 Apr 2026 14:16:24 -0500 Subject: [PATCH 55/71] abi_serializer: inline restructuring summary into add_specialized_stream_unpack assert message The body comment already documents what restructuring is needed (split the static const map into an instance-owned map with a static fallback, plus a parallel change in stream_sink::unpack_built_in to consult the instance map first). The FC_ASSERT message just said "see header doc", forcing the next implementer to go hunting for the plan. Quote the relevant sentence inline so the failure mode is self-documenting. --- libraries/chain/abi_serializer.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libraries/chain/abi_serializer.cpp b/libraries/chain/abi_serializer.cpp index 4c2589e5a8..6884a69b78 100644 --- a/libraries/chain/abi_serializer.cpp +++ b/libraries/chain/abi_serializer.cpp @@ -264,8 +264,9 @@ namespace sysio::chain { // stream_sink::unpack_built_in to consult the instance map first. No callers need this // today (add_specialized_unpack_pack itself has zero in-tree callers); intentional fail-loud // until a real use case shows up. - FC_ASSERT(false, "abi_serializer::add_specialized_stream_unpack is not yet implemented; " - "see header doc for the restructuring required"); + FC_ASSERT(false, "abi_serializer::add_specialized_stream_unpack is not yet implemented: " + "the streaming dispatch is a static const map; per-instance overrides " + "require restructuring it into an instance-owned map with a static fallback"); } const std::pair* From e4b19bd14a617d70ba00a06826dfee64dce22116 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Thu, 30 Apr 2026 14:37:05 -0500 Subject: [PATCH 56/71] libfc: rename json_yield.hpp -> json_escape.hpp (escape_string is the API; yield is the parameter type) The header holds two things: a yield-callback typedef and the escape_string declarations. escape_string is the actual public API; the yield function is the callback parameter. Old name only described the secondary item. Pure rename + include update in two consumers (json.hpp, json_stream.hpp). File comment updated to lead with the escape primitives. --- libraries/libfc/include/fc/io/json.hpp | 2 +- .../fc/io/{json_yield.hpp => json_escape.hpp} | 13 +++++++++---- libraries/libfc/include/fc/io/json_stream.hpp | 2 +- 3 files changed, 11 insertions(+), 6 deletions(-) rename libraries/libfc/include/fc/io/{json_yield.hpp => json_escape.hpp} (61%) diff --git a/libraries/libfc/include/fc/io/json.hpp b/libraries/libfc/include/fc/io/json.hpp index 65cd0521c8..7700060d34 100644 --- a/libraries/libfc/include/fc/io/json.hpp +++ b/libraries/libfc/include/fc/io/json.hpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #define DEFAULT_MAX_RECURSION_DEPTH 200 diff --git a/libraries/libfc/include/fc/io/json_yield.hpp b/libraries/libfc/include/fc/io/json_escape.hpp similarity index 61% rename from libraries/libfc/include/fc/io/json_yield.hpp rename to libraries/libfc/include/fc/io/json_escape.hpp index 396b613104..073638b031 100644 --- a/libraries/libfc/include/fc/io/json_yield.hpp +++ b/libraries/libfc/include/fc/io/json_escape.hpp @@ -1,9 +1,14 @@ #pragma once -// Lightweight header carrying just the bits of the JSON layer that fc/io/json_stream.hpp -// (and downstream variant.hpp / flat.hpp container streaming overloads) need to compile, -// without dragging in fc/io/json.hpp's class json. Pulling in class json would create -// a cycle: variant.hpp -> json_stream.hpp -> json.hpp -> variant.hpp. +// JSON string-escape primitives shared by both the legacy variant-tree path and +// the streaming json_writer path. Decoupled from class json so that +// fc/io/json_stream.hpp (and downstream variant.hpp / flat.hpp container +// streaming overloads) can call escape_string without pulling in class json -- +// avoiding the cycle: variant.hpp -> json_stream.hpp -> json.hpp -> variant.hpp. +// +// The yield-function alias is the parameter type used by escape_string; +// class json::yield_function_t is an alias to this type so existing callers +// continue to compile unchanged. #include #include diff --git a/libraries/libfc/include/fc/io/json_stream.hpp b/libraries/libfc/include/fc/io/json_stream.hpp index fc114dea66..16cc1e3912 100644 --- a/libraries/libfc/include/fc/io/json_stream.hpp +++ b/libraries/libfc/include/fc/io/json_stream.hpp @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include From bb05817ab1b37b95df5b3194d7ae249a907d8394 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Wed, 6 May 2026 11:43:05 -0500 Subject: [PATCH 57/71] trace_api: migrate get_transaction_trace to streaming JSON Previously /v1/trace_api/get_transaction_trace called request_handler::get_transaction_trace, which built the entire block as fc::variant via process_block / process_transactions and then walked the variant tree to extract just the one matching transaction. That duplicates the work of get_block (already streaming) and discards the rest. Extract write_transaction out of write_transactions in request_handler.cpp so a single transaction can be emitted standalone; pure refactor, the block-level output is byte-identical. Add response_formatter::process_transaction_to_json that walks the block's transaction list and emits just the matching transaction's JSON object via fc::json_writer, no surrounding block wrapper, returning an empty string on miss. Add request_handler::get_transaction_trace_json mirroring get_block_trace_json. Swap the plugin handler to call the streaming peer and return the pre-serialized body wrapped in fc::variant(string) with http_content_type::json_raw, matching the get_block handler pattern. Add streaming_vs_variant_transaction_response_parity in test_responses covering the hit case (round-trip the streamed JSON through fc::json::from_string and parity-check key-by-key against the variant path via to_kv) and the miss case (variant returns null, stream returns empty string). Locks the parity invariant for the per-transaction emitter, which the existing block-level parity test could mask via the surrounding wrapper. --- .../sysio/trace_api/request_handler.hpp | 34 +++++++ .../trace_api_plugin/src/request_handler.cpp | 56 +++++++---- .../trace_api_plugin/src/trace_api_plugin.cpp | 10 +- .../trace_api_plugin/test/test_responses.cpp | 93 +++++++++++++++++++ 4 files changed, 171 insertions(+), 22 deletions(-) diff --git a/plugins/trace_api_plugin/include/sysio/trace_api/request_handler.hpp b/plugins/trace_api_plugin/include/sysio/trace_api/request_handler.hpp index d3e9d6b2db..b9adb70392 100644 --- a/plugins/trace_api_plugin/include/sysio/trace_api/request_handler.hpp +++ b/plugins/trace_api_plugin/include/sysio/trace_api/request_handler.hpp @@ -18,6 +18,12 @@ namespace sysio::trace_api { // fc::mutable_variant_object tree entirely. Output is byte-for-byte identical // to fc::json::to_string(process_block(...)). static std::string process_block_to_json( const data_log_entry& trace, bool irreversible, const data_handler_function& data_handler ); + + // Streaming counterpart for a single transaction inside a block trace. Walks the block's transaction list and, + // for the first entry whose id matches trxid, emits just that transaction's JSON object (no surrounding block + // wrapper). Returns an empty string if no transaction in the block matches. Output for a matching trx is + // byte-for-byte identical to fc::json::to_string(get_transaction_trace(...)). + static std::string process_transaction_to_json( const data_log_entry& trace, const chain::transaction_id_type& trxid, const data_handler_function& data_handler ); }; } @@ -119,6 +125,34 @@ namespace sysio::trace_api { return result; } + /** + * Streaming variant of get_transaction_trace: emits the matching transaction's JSON directly into a std::string + * via fc::json_writer instead of building the full block's fc::variant tree only to extract one transaction. + * Returned string is empty if the block does not exist or the trxid is not present in the block; callers should + * check empty() before dispatching the 200 response. Skips the fc::variant -> fc::json::to_string copy on the + * response path when paired with http_content_type::json_raw. + */ + std::string get_transaction_trace_json( const chain::transaction_id_type& trxid, uint32_t block_height ) { + _log("get_transaction_trace_json called"); + auto data = logfile_provider.get_block(block_height); + if (!data) { + _log("No block found at block height " + std::to_string(block_height)); + return {}; + } + + auto data_handler = [this](const std::variant& action) -> std::tuple> { + return std::visit([&](const auto& a) { + return data_handler_provider.serialize_to_variant(a); + }, action); + }; + + auto resp = detail::response_formatter::process_transaction_to_json(std::get<0>(*data), trxid, data_handler); + if (resp.empty()) { + _log("No transaction with id " + trxid.str() + " found in block " + std::to_string(block_height)); + } + return resp; + } + private: LogfileProvider logfile_provider; DataHandlerProvider data_handler_provider; diff --git a/plugins/trace_api_plugin/src/request_handler.cpp b/plugins/trace_api_plugin/src/request_handler.cpp index b20e2ac06c..c00ba450aa 100644 --- a/plugins/trace_api_plugin/src/request_handler.cpp +++ b/plugins/trace_api_plugin/src/request_handler.cpp @@ -166,29 +166,35 @@ namespace { w.end_array(); } + void write_transaction(fc::json_writer& w, + const transaction_trace_v0& t, + const data_handler_function& data_handler) { + w.begin_object(); + w.set("id", t.id) + .set("block_num", t.block_num) + // FC_SERIALIZE_AS_STRING-reflected; emits the ISO date string. + .set("block_time", t.block_time) + .set("producer_block_id", t.producer_block_id); + w.key("actions"); write_actions(w, t.actions, data_handler); + // FC_REFLECT_ENUM-reflected; emits the member-name string via to_json_stream(enum_type). + w.set("status", t.status) + .set("cpu_usage_us", t.cpu_usage_us) + .set("net_usage_words", t.net_usage_words.value) + .set("signatures", t.signatures); + // transaction_header is a reflected struct composed of fc::time_point_sec, + // uint16/uint32 and unsigned_ints. No native to_json_stream path yet so + // fall back via the variant bridge for just that field. + w.key("transaction_header"); + fc::to_json_stream_via_variant(t.trx_header, w); + w.end_object(); + } + void write_transactions(fc::json_writer& w, const std::vector& transactions, const data_handler_function& data_handler) { w.begin_array(); for (const auto& t : transactions) { - w.begin_object(); - w.set("id", t.id) - .set("block_num", t.block_num) - // FC_SERIALIZE_AS_STRING-reflected; emits the ISO date string. - .set("block_time", t.block_time) - .set("producer_block_id", t.producer_block_id); - w.key("actions"); write_actions(w, t.actions, data_handler); - // FC_REFLECT_ENUM-reflected; emits the member-name string via to_json_stream(enum_type). - w.set("status", t.status) - .set("cpu_usage_us", t.cpu_usage_us) - .set("net_usage_words", t.net_usage_words.value) - .set("signatures", t.signatures); - // transaction_header is a reflected struct composed of fc::time_point_sec, - // uint16/uint32 and unsigned_ints. No native to_json_stream path yet so - // fall back via the variant bridge for just that field. - w.key("transaction_header"); - fc::to_json_stream_via_variant(t.trx_header, w); - w.end_object(); + write_transaction(w, t, data_handler); } w.end_array(); } @@ -214,4 +220,18 @@ namespace sysio::trace_api::detail { }, trace); return out; } + + std::string response_formatter::process_transaction_to_json( const data_log_entry& trace, const chain::transaction_id_type& trxid, const data_handler_function& data_handler ) { + std::string out; + std::visit([&](auto&& bt) { + for (const auto& t : bt.transactions) { + if (t.id == trxid) { + fc::json_writer w(out); + write_transaction(w, t, data_handler); + return; + } + } + }, trace); + return out; + } } diff --git a/plugins/trace_api_plugin/src/trace_api_plugin.cpp b/plugins/trace_api_plugin/src/trace_api_plugin.cpp index e1d7ac054a..8734a97dea 100644 --- a/plugins/trace_api_plugin/src/trace_api_plugin.cpp +++ b/plugins/trace_api_plugin/src/trace_api_plugin.cpp @@ -322,18 +322,20 @@ struct trace_api_rpc_plugin_impl : public std::enable_shared_from_thisget_transaction_trace(*trx_id, *blk_num); - if (resp.is_null()) { + std::string resp = req_handler->get_transaction_trace_json(*trx_id, *blk_num); + if (resp.empty()) { error_results results{404, "Trace API: transaction trace missing"}; cb( 404, fc::variant( results )); } else { - cb( 200, std::move(resp) ); + // Pre-serialized JSON body: wrap in a string-valued variant so the + // json_raw content-type path sends it verbatim, mirroring get_block. + cb( 200, fc::variant( std::move(resp) ) ); } } } catch (...) { http_plugin::handle_exception("trace_api", "get_transaction", body, cb); } - }}); + }}, http_content_type::json_raw); } void plugin_shutdown() { diff --git a/plugins/trace_api_plugin/test/test_responses.cpp b/plugins/trace_api_plugin/test/test_responses.cpp index 649123d8ad..b5ae39c870 100644 --- a/plugins/trace_api_plugin/test/test_responses.cpp +++ b/plugins/trace_api_plugin/test/test_responses.cpp @@ -69,6 +69,14 @@ struct response_test_fixture { return response_impl.get_block_trace_json( block_height ); } + fc::variant get_transaction_trace( const chain::transaction_id_type& trxid, uint32_t block_height ) { + return response_impl.get_transaction_trace( trxid, block_height ); + } + + std::string get_transaction_trace_json( const chain::transaction_id_type& trxid, uint32_t block_height ) { + return response_impl.get_transaction_trace_json( trxid, block_height ); + } + // fixture data and methods std::function mock_get_block; std::function>(const action_trace_v0&)> mock_data_handler = default_mock_data_handler; @@ -546,4 +554,89 @@ BOOST_AUTO_TEST_SUITE(trace_responses) BOOST_TEST(to_kv(variant_response) == to_kv(streamed_as_variant), boost::test_tools::per_element()); } + // The streaming JSON response (process_transaction_to_json) must produce the same observable shape as the variant + // response (get_transaction_trace) for a matching trxid. Round-trip the streaming bytes through fc::json::from_string + // and compare key-by-key with the variant path. Catches silent shape regressions in the per-transaction emitter that + // the block-level parity test would not detect because they are masked by the surrounding block wrapper. + BOOST_FIXTURE_TEST_CASE(streaming_vs_variant_transaction_response_parity, response_test_fixture) + { + auto trx_id_a = "0000000000000000000000000000000000000000000000000000000000000001"_h; + auto trx_id_b = "0000000000000000000000000000000000000000000000000000000000000002"_h; + + auto block_trace = block_trace_v0 { + "b000000000000000000000000000000000000000000000000000000000000001"_h, + 1, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + chain::block_timestamp_type(0), + "bp.one"_n, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + std::vector { + { + trx_id_a, + std::vector { + { + 0, + "receiver"_n, "contract"_n, "action"_n, + {{ "alice"_n, "active"_n }}, + { 0x00, 0x01, 0x02, 0x03 }, + { 0x04, 0x05, 0x06, 0x07 } + } + }, + fc::enum_type{chain::transaction_receipt_header::status_enum::executed}, + 10, + 5, + std::vector{ chain::signature_type() }, + { chain::time_point_sec(), 1, 0, 100, 50, 0 }, + 1, + chain::block_timestamp_type(0), + std::optional{} + }, + { + trx_id_b, + std::vector { + { + 1, + "receiver"_n, "contract"_n, "action"_n, + {{ "bob"_n, "active"_n }}, + { 0x10, 0x11 }, + {} + } + }, + fc::enum_type{chain::transaction_receipt_header::status_enum::executed}, + 20, + 6, + std::vector{ chain::signature_type() }, + { chain::time_point_sec(), 2, 0, 200, 60, 0 }, + 1, + chain::block_timestamp_type(0), + std::optional{} + } + } + }; + + mock_get_block = [&block_trace]( uint32_t height ) -> get_block_t { + return std::make_tuple(data_log_entry(block_trace), false); + }; + + // Hit case: trxid present in the block. Both paths must agree on every field. + { + fc::variant variant_response = get_transaction_trace( trx_id_b, 1 ); + std::string streamed_response = get_transaction_trace_json( trx_id_b, 1 ); + BOOST_TEST(!variant_response.is_null()); + BOOST_TEST(!streamed_response.empty()); + fc::variant streamed_as_variant = fc::json::from_string( streamed_response ); + BOOST_TEST(to_kv(variant_response) == to_kv(streamed_as_variant), boost::test_tools::per_element()); + } + + // Miss case: trxid not in the block. Variant path returns null variant; streaming path returns empty string. + { + auto missing_id = "00000000000000000000000000000000000000000000000000000000000000ff"_h; + fc::variant variant_response = get_transaction_trace( missing_id, 1 ); + std::string streamed_response = get_transaction_trace_json( missing_id, 1 ); + BOOST_TEST(variant_response.is_null()); + BOOST_TEST(streamed_response.empty()); + } + } + BOOST_AUTO_TEST_SUITE_END() From 1d737c8082e594de889cd3b497370322b41bdb7f Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Wed, 6 May 2026 13:07:19 -0500 Subject: [PATCH 58/71] trace_api: stream params/return_data via abi_serializer::binary_to_json_stream write_actions used to bridge from the streaming json_writer back to fc::variant just to ABI-decode action "params" and "return_data": call serialize_to_variant, build an fc::variant tree via binary_to_variant, re-serialise through fc::json::to_string, and splice the result into the writer. The variant tree was pure allocation tax that only existed to bridge the two APIs. Add abi_data_handler::serialize_to_json_stream that calls abi_serializer::binary_to_json_stream directly into the writer, no intermediate variant. Each field is wrapped in its own json_writer::checkpoint() / rewind() pair so a partial decode failure leaves the writer balanced. Restructure serialize_to_variant the same way: per-field try/catch instead of the previous single try wrapping both decodes. Per-field independence rule (both paths): a parse failure on one field is logged via except_handler and does NOT prevent the other field from being attempted. The lone short-circuit case is chain::abi_recursion_depth_exception thrown during the params decode -- the ABI is structurally bad, so retrying return_data with the same ABI cannot succeed and the function returns early with neither field emitted. In the streaming path the half-written params tokens are rewound; in the variant path both fields return null/nullopt. return_data has no "next" field so abi_recursion_depth_exception in the return_data decode is treated like any other failure: log, leave that field empty, keep the already-decoded params. Add a parallel stream_data_handler_function typedef and route the streaming response paths (process_block_to_json, process_transaction_to_json, and the write_actions / write_transaction / write_transactions helpers) through it. get_block_trace_json and get_transaction_trace_json construct a streaming callback that calls data_handler_provider.serialize_to_json_stream. The variant-only process_block path still uses data_handler_function. Tests in abi_data_handler_tests: - streaming_basic_abi_with_return_value_parity: both fields decode to JSON byte-equivalent to the variant path, checked via to_kv after a json::from_string round-trip. - streaming_no_abi_emits_nothing: writer stays balanced and produces "{}" when no ABI is registered. - streaming_basic_abi_insufficient_data_rolls_back: truncated bytes throw inside binary_to_json_stream; rewind brings the writer back to empty, except_handler is logged. - variant_basic_abi_params_throws_return_data_succeeds and the streaming counterpart: truncated params bytes, well-formed return_data bytes -- params decode logs and falls through, return_data decodes cleanly, only return_data is emitted. Pre-restructure the single catch swallowed both fields. - variant_basic_abi_params_succeeds_return_data_throws and the streaming counterpart: well-formed params, truncated return_data -- return_data decode logs and is rolled back, params remains. Pre-restructure return_data's exception lost the already-successful params. The fixture parity tests in test_responses already cover the integration path through the streaming mock_data_handler_provider, which gains a serialize_to_json_stream peer that delegates to the variant mock and re-emits via fc::to_json_stream so byte parity holds without a real ABI. --- .../sysio/trace_api/abi_data_handler.hpp | 24 ++ .../sysio/trace_api/request_handler.hpp | 23 +- .../trace_api_plugin/src/abi_data_handler.cpp | 114 +++++-- .../trace_api_plugin/src/request_handler.cpp | 28 +- .../test/test_data_handlers.cpp | 277 ++++++++++++++++++ .../trace_api_plugin/test/test_responses.cpp | 17 ++ 6 files changed, 438 insertions(+), 45 deletions(-) diff --git a/plugins/trace_api_plugin/include/sysio/trace_api/abi_data_handler.hpp b/plugins/trace_api_plugin/include/sysio/trace_api/abi_data_handler.hpp index 5def5b8ac4..b40a671d11 100644 --- a/plugins/trace_api_plugin/include/sysio/trace_api/abi_data_handler.hpp +++ b/plugins/trace_api_plugin/include/sysio/trace_api/abi_data_handler.hpp @@ -4,6 +4,8 @@ #include #include +namespace fc { class json_writer; } + namespace sysio { namespace chain { struct abi_serializer; @@ -38,6 +40,24 @@ namespace sysio { */ std::tuple> serialize_to_variant(const std::variant& action); + /** + * Streaming counterpart of serialize_to_variant: emits the ABI-decoded "params" and (when applicable) + * "return_data" key/value pairs directly into `w` via fc::json_writer, without ever materialising an + * fc::variant tree. Pre-conditions: `w` is positioned inside the action's enclosing JSON object such that + * additional `key(...) value(...)` pairs are valid (i.e. caller has already written global_sequence, receiver, + * etc.). No-op if the action's account has no registered ABI or the action is not in the ABI. + * + * Fields decode independently: a parse failure on one is logged via `except_handler` and rolled back via + * `w.rewind()` to its own per-field checkpoint, but does NOT prevent the other from being attempted. The + * lone short-circuit case is `chain::abi_recursion_depth_exception` thrown during the params decode -- the + * ABI is structurally bad, so retrying the other field with the same ABI cannot succeed in a useful way + * and the function returns early with neither field emitted. serialize_to_variant uses the same rule. + * + * @param action - trace of the action including metadata necessary for finding the ABI + * @param w - writer to emit into; on return, zero, one, or both of "params" / "return_data" have been added + */ + void serialize_to_json_stream(const std::variant& action, fc::json_writer& w); + /** * Utility class that allows multiple request_handlers to share the same abi_data_handler */ @@ -51,6 +71,10 @@ namespace sysio { return handler->serialize_to_variant(action); } + void serialize_to_json_stream( const std::variant& action, fc::json_writer& w ) { + handler->serialize_to_json_stream(action, w); + } + std::shared_ptr handler; }; diff --git a/plugins/trace_api_plugin/include/sysio/trace_api/request_handler.hpp b/plugins/trace_api_plugin/include/sysio/trace_api/request_handler.hpp index b9adb70392..69ea61b334 100644 --- a/plugins/trace_api_plugin/include/sysio/trace_api/request_handler.hpp +++ b/plugins/trace_api_plugin/include/sysio/trace_api/request_handler.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -8,6 +9,12 @@ namespace sysio::trace_api { using data_handler_function = std::function>( const std::variant& action)>; + // Streaming counterpart of data_handler_function: instead of returning a variant tuple that the caller then + // re-serializes to JSON, the callback emits the action's "params" and (when applicable) "return_data" + // key/value pairs directly into the writer. Decoded bytes flow straight from abi_serializer through + // binary_to_json_stream into the output buffer with no intermediate fc::variant allocation. + using stream_data_handler_function = std::function& action, fc::json_writer& w)>; + namespace detail { class response_formatter { public: @@ -17,13 +24,13 @@ namespace sysio::trace_api { // directly into an output std::string via fc::json_writer, skipping the // fc::mutable_variant_object tree entirely. Output is byte-for-byte identical // to fc::json::to_string(process_block(...)). - static std::string process_block_to_json( const data_log_entry& trace, bool irreversible, const data_handler_function& data_handler ); + static std::string process_block_to_json( const data_log_entry& trace, bool irreversible, const stream_data_handler_function& data_handler ); // Streaming counterpart for a single transaction inside a block trace. Walks the block's transaction list and, // for the first entry whose id matches trxid, emits just that transaction's JSON object (no surrounding block // wrapper). Returns an empty string if no transaction in the block matches. Output for a matching trx is // byte-for-byte identical to fc::json::to_string(get_transaction_trace(...)). - static std::string process_transaction_to_json( const data_log_entry& trace, const chain::transaction_id_type& trxid, const data_handler_function& data_handler ); + static std::string process_transaction_to_json( const data_log_entry& trace, const chain::transaction_id_type& trxid, const stream_data_handler_function& data_handler ); }; } @@ -78,9 +85,9 @@ namespace sysio::trace_api { return {}; } - auto data_handler = [this](const std::variant& action) -> std::tuple> { - return std::visit([&](const auto& a) { - return data_handler_provider.serialize_to_variant(a); + auto data_handler = [this](const std::variant& action, fc::json_writer& w) { + std::visit([&](const auto& a) { + data_handler_provider.serialize_to_json_stream(a, w); }, action); }; @@ -140,9 +147,9 @@ namespace sysio::trace_api { return {}; } - auto data_handler = [this](const std::variant& action) -> std::tuple> { - return std::visit([&](const auto& a) { - return data_handler_provider.serialize_to_variant(a); + auto data_handler = [this](const std::variant& action, fc::json_writer& w) { + std::visit([&](const auto& a) { + data_handler_provider.serialize_to_json_stream(a, w); }, action); }; diff --git a/plugins/trace_api_plugin/src/abi_data_handler.cpp b/plugins/trace_api_plugin/src/abi_data_handler.cpp index 338e665aa8..c32f6b014d 100644 --- a/plugins/trace_api_plugin/src/abi_data_handler.cpp +++ b/plugins/trace_api_plugin/src/abi_data_handler.cpp @@ -1,8 +1,21 @@ #include #include +#include namespace sysio::trace_api { + namespace { + // ABIs are user-provided; the yield function only enforces the recursion-depth cap and + // intentionally has no wall-clock deadline. Shared between the variant and streaming paths + // so they stay in lock-step on what counts as "too deep". + auto make_abi_yield() { + return [](size_t recursion_depth) { + SYS_ASSERT( recursion_depth < chain::abi_serializer::max_recursion_depth, chain::abi_recursion_depth_exception, + "exceeded max_recursion_depth {} ", chain::abi_serializer::max_recursion_depth ); + }; + } + } + void abi_data_handler::add_abi( const chain::name& name, chain::abi_def&& abi ) { // currently abis are operator provided so no need to protect against abuse abi_serializer_by_account.emplace(name, @@ -11,33 +24,94 @@ namespace sysio::trace_api { std::tuple> abi_data_handler::serialize_to_variant(const std::variant& action) { return std::visit([&](const auto& a) -> std::tuple> { - if (abi_serializer_by_account.count(a.account) > 0) { - const auto &serializer_p = abi_serializer_by_account.at(a.account); - auto type_name = serializer_p->get_action_type(a.action); + auto it = abi_serializer_by_account.find(a.account); + if (it == abi_serializer_by_account.end()) return {}; + const auto& serializer_p = it->second; + auto type_name = serializer_p->get_action_type(a.action); + if (type_name.empty()) return {}; + + auto abi_yield = make_abi_yield(); + + // Each field is decoded independently. A parse failure on one (truncated bytes, type + // mismatch, etc.) is logged but does not prevent the other from being attempted. The one + // exception is abi_recursion_depth_exception: the ABI itself is structurally bad, retrying + // the other field with the same ABI cannot succeed in a useful way, so bail entirely with + // both fields empty. Deadline exceptions are not raised here -- this yield does not carry + // a deadline. + fc::variant params; + try { + params = serializer_p->binary_to_variant(type_name, a.data, abi_yield); + } catch (const chain::abi_recursion_depth_exception&) { + except_handler(MAKE_EXCEPTION_WITH_CONTEXT(std::current_exception())); + return {}; + } catch (...) { + except_handler(MAKE_EXCEPTION_WITH_CONTEXT(std::current_exception())); + // params stays default-constructed (null variant); fall through to return_data. + } - if (!type_name.empty()) { + std::optional ret_data; + if (a.return_value.size() > 0) { + auto return_type_name = serializer_p->get_action_result_type(a.action); + if (!return_type_name.empty()) { try { - // abi_serializer expects a yield function that takes a recursion depth - // abis are user provided, do not use a deadline - auto abi_yield = [](size_t recursion_depth) { - SYS_ASSERT( recursion_depth < chain::abi_serializer::max_recursion_depth, chain::abi_recursion_depth_exception, - "exceeded max_recursion_depth {} ", chain::abi_serializer::max_recursion_depth ); - }; - std::optional ret_data; - auto params = serializer_p->binary_to_variant(type_name, a.data, abi_yield); - if(a.return_value.size() > 0) { - auto return_type_name = serializer_p->get_action_result_type(a.action); - if (!return_type_name.empty()) { - ret_data = serializer_p->binary_to_variant(return_type_name, a.return_value, abi_yield); - } - } - return {std::move(params), std::move(ret_data)}; + ret_data = serializer_p->binary_to_variant(return_type_name, a.return_value, abi_yield); } catch (...) { + // No "next" field after return_data, so abi_recursion_depth_exception is treated + // the same as any other failure: log and leave ret_data nullopt. params remains. except_handler(MAKE_EXCEPTION_WITH_CONTEXT(std::current_exception())); } } } - return {}; + + return {std::move(params), std::move(ret_data)}; + }, action); + } + + void abi_data_handler::serialize_to_json_stream(const std::variant& action, fc::json_writer& w) { + std::visit([&](const auto& a) { + auto it = abi_serializer_by_account.find(a.account); + if (it == abi_serializer_by_account.end()) return; + const auto& serializer_p = it->second; + auto type_name = serializer_p->get_action_type(a.action); + if (type_name.empty()) return; + + auto abi_yield = make_abi_yield(); + + // Same independence rule as serialize_to_variant: a parse failure on one field is logged + // but does not prevent the other from being attempted; abi_recursion_depth_exception is + // the lone short-circuit case (structurally bad ABI -- retrying the other field cannot + // help). Each field has its own json_writer::checkpoint() / rewind() so a partial emit + // is rolled back to keep the writer balanced. + { + auto cp = w.checkpoint(); + try { + w.key("params"); + serializer_p->binary_to_json_stream(type_name, a.data, w, abi_yield); + } catch (const chain::abi_recursion_depth_exception&) { + w.rewind(cp); + except_handler(MAKE_EXCEPTION_WITH_CONTEXT(std::current_exception())); + return; // bail; do not attempt return_data + } catch (...) { + w.rewind(cp); + except_handler(MAKE_EXCEPTION_WITH_CONTEXT(std::current_exception())); + // fall through and attempt return_data independently + } + } + + if (a.return_value.empty()) return; + auto return_type_name = serializer_p->get_action_result_type(a.action); + if (return_type_name.empty()) return; + + auto cp = w.checkpoint(); + try { + w.key("return_data"); + serializer_p->binary_to_json_stream(return_type_name, a.return_value, w, abi_yield); + } catch (...) { + // No "next" field after return_data, so abi_recursion_depth_exception is treated the + // same as any other failure: rewind, log, leave params (if any) intact. + w.rewind(cp); + except_handler(MAKE_EXCEPTION_WITH_CONTEXT(std::current_exception())); + } }, action); } } diff --git a/plugins/trace_api_plugin/src/request_handler.cpp b/plugins/trace_api_plugin/src/request_handler.cpp index c00ba450aa..55501a19df 100644 --- a/plugins/trace_api_plugin/src/request_handler.cpp +++ b/plugins/trace_api_plugin/src/request_handler.cpp @@ -128,7 +128,7 @@ namespace { void write_actions(fc::json_writer& w, const std::vector& actions, - const data_handler_function& data_handler) { + const stream_data_handler_function& data_handler) { // Iteration order matches process_actions: sort indices by global_sequence so // clients see execution order (notifications run before the inline actions that // queued them, so action-slot order is not global_sequence order). @@ -149,18 +149,12 @@ namespace { w.key("authorization"); write_authorizations(w, a.authorization); w.key("data"); w.value_hex(a.data.data(), a.data.size()); w.key("return_value"); w.value_hex(a.return_value.data(), a.return_value.size()); - // The abi decode path still produces fc::variant (structure is ABI-dependent). - // TODO: route through abi_serializer::binary_to_json_stream (already in use for - // table rows + streamed_processed_trace) so the action body stays allocation-free - // end-to-end; data_handler currently builds the variant tree and we re-emit it. - // For now, splice its serialized form as raw JSON. - auto [params, return_data] = data_handler(a); - if (!params.is_null()) { - w.set_raw("params", fc::json::to_string(params, fc::json::yield_function_t())); - } - if (return_data.has_value()) { - w.set_raw("return_data", fc::json::to_string(*return_data, fc::json::yield_function_t())); - } + // ABI-decoded "params" / "return_data" are emitted directly into w by the streaming + // data_handler -- no fc::variant tree, no fc::json::to_string splice. The handler + // is responsible for emitting zero, one, or both keys depending on what's available + // and whether the decode succeeds; on failure it rolls back via json_writer::rewind + // so the action object's preceding fields stay intact. + data_handler(a, w); w.end_object(); } w.end_array(); @@ -168,7 +162,7 @@ namespace { void write_transaction(fc::json_writer& w, const transaction_trace_v0& t, - const data_handler_function& data_handler) { + const stream_data_handler_function& data_handler) { w.begin_object(); w.set("id", t.id) .set("block_num", t.block_num) @@ -191,7 +185,7 @@ namespace { void write_transactions(fc::json_writer& w, const std::vector& transactions, - const data_handler_function& data_handler) { + const stream_data_handler_function& data_handler) { w.begin_array(); for (const auto& t : transactions) { write_transaction(w, t, data_handler); @@ -201,7 +195,7 @@ namespace { } namespace sysio::trace_api::detail { - std::string response_formatter::process_block_to_json( const data_log_entry& trace, bool irreversible, const data_handler_function& data_handler ) { + std::string response_formatter::process_block_to_json( const data_log_entry& trace, bool irreversible, const stream_data_handler_function& data_handler ) { std::string out; fc::json_writer w(out); std::visit([&](auto&& bt) { @@ -221,7 +215,7 @@ namespace sysio::trace_api::detail { return out; } - std::string response_formatter::process_transaction_to_json( const data_log_entry& trace, const chain::transaction_id_type& trxid, const data_handler_function& data_handler ) { + std::string response_formatter::process_transaction_to_json( const data_log_entry& trace, const chain::transaction_id_type& trxid, const stream_data_handler_function& data_handler ) { std::string out; std::visit([&](auto&& bt) { for (const auto& t : bt.transactions) { diff --git a/plugins/trace_api_plugin/test/test_data_handlers.cpp b/plugins/trace_api_plugin/test/test_data_handlers.cpp index 32aa14a172..adf158e34a 100644 --- a/plugins/trace_api_plugin/test/test_data_handlers.cpp +++ b/plugins/trace_api_plugin/test/test_data_handlers.cpp @@ -1,5 +1,8 @@ #include +#include +#include + #include #include @@ -229,5 +232,279 @@ BOOST_AUTO_TEST_SUITE(abi_data_handler_tests) BOOST_REQUIRE(!std::get<1>(actual)); } + // Streaming serialize_to_json_stream must emit byte-equivalent JSON to the variant path + // when the same ABI is supplied. Drive the streamer into a wrapped object so the result + // can be parsed back as a variant and compared field-by-field. + BOOST_AUTO_TEST_CASE(streaming_basic_abi_with_return_value_parity) + { + auto action = action_trace_v0 { + 0, "alice"_n, "alice"_n, "foo"_n, {}, {0x00, 0x01, 0x02, 0x03}, {0x04, 0x05, 0x06} + }; + std::variant action_trace_t = action; + + auto abi = chain::abi_def ( {}, + { + { "foo", "", { {"a", "varuint32"}, {"b", "varuint32"}, {"c", "varuint32"}, {"d", "varuint32"} } }, + { "foor", "", { {"e", "varuint32"}, {"f", "varuint32"}, {"g", "varuint32"} } } + }, + { + { "foo"_n, "foo", ""} + }, + {}, {}, {} + ); + abi.version = "sysio::abi/1."; + abi.action_results = { std::vector{ chain::action_result_def{ "foo"_n, "foor"} } }; + + abi_data_handler handler(exception_handler{}); + handler.add_abi("alice"_n, std::move(abi)); + + auto [params_v, return_v] = handler.serialize_to_variant(action_trace_t); + + std::string out; + { + fc::json_writer w(out); + w.begin_object(); + handler.serialize_to_json_stream(action_trace_t, w); + w.end_object(); + BOOST_REQUIRE(w.balanced()); + } + + fc::variant streamed = fc::json::from_string(out); + const auto& streamed_obj = streamed.get_object(); + BOOST_REQUIRE(streamed_obj.contains("params")); + BOOST_REQUIRE(streamed_obj.contains("return_data")); + + BOOST_TEST(to_kv(params_v) == to_kv(streamed_obj["params"]), boost::test_tools::per_element()); + BOOST_REQUIRE(return_v.has_value()); + BOOST_TEST(to_kv(*return_v) == to_kv(streamed_obj["return_data"]), boost::test_tools::per_element()); + } + + // No ABI registered: the streaming path must emit zero key/value pairs and leave the writer + // in a balanced, key/value-pair-empty state so the surrounding object closes cleanly. + BOOST_AUTO_TEST_CASE(streaming_no_abi_emits_nothing) + { + auto action = action_trace_v0 { + 0, "alice"_n, "alice"_n, "foo"_n, {}, {0x00, 0x01, 0x02, 0x03}, {0x04, 0x05, 0x06} + }; + std::variant action_trace_t = action; + + abi_data_handler handler(exception_handler{}); + + std::string out; + { + fc::json_writer w(out); + w.begin_object(); + handler.serialize_to_json_stream(action_trace_t, w); + w.end_object(); + BOOST_REQUIRE(w.balanced()); + } + + BOOST_TEST(out == "{}"); + } + + // Per-field independence (variant path, params throws / return_data succeeds): truncated params + // bytes raise inside binary_to_variant; serialize_to_variant logs and falls through to attempt + // return_data with the same yield. return_data decodes cleanly, so the result is + // {null_variant, ret_data}: call sites that check `!params.is_null()` skip params and emit + // just return_data. Pre-PR (single try wrapping both) the catch swallowed both fields. + BOOST_AUTO_TEST_CASE(variant_basic_abi_params_throws_return_data_succeeds) + { + auto action = action_trace_v0 { + 0, "alice"_n, "alice"_n, "foo"_n, {}, {0x00, 0x01, 0x02}, {0x04, 0x05, 0x06} + }; + std::variant action_trace_t = action; + + auto abi = chain::abi_def ( {}, + { + { "foo", "", { {"a", "varuint32"}, {"b", "varuint32"}, {"c", "varuint32"}, {"d", "varuint32"} } }, + { "foor", "", { {"e", "varuint32"}, {"f", "varuint32"}, {"g", "varuint32"} } } + }, + { { "foo"_n, "foo", ""} }, + {}, {}, {} + ); + abi.version = "sysio::abi/1."; + abi.action_results = { std::vector{ chain::action_result_def{ "foo"_n, "foor"} } }; + + bool log_called = false; + abi_data_handler handler([&log_called](const exception_with_context&){ log_called = true; }); + handler.add_abi("alice"_n, std::move(abi)); + + fc::variant expected_return = fc::mutable_variant_object() + ("e", 4)("f", 5)("g", 6); + + auto actual = handler.serialize_to_variant(action_trace_t); + + BOOST_TEST(log_called); + BOOST_TEST(std::get<0>(actual).is_null()); + BOOST_REQUIRE(std::get<1>(actual).has_value()); + BOOST_TEST(to_kv(expected_return) == to_kv(*std::get<1>(actual)), boost::test_tools::per_element()); + } + + // Per-field independence (variant path, params succeeds / return_data throws): params decodes + // cleanly, return_data bytes are truncated. serialize_to_variant logs the return_data failure + // and returns the params-decoded variant alongside an empty optional. Pre-PR the catch on + // return_data discarded the already-successful params; the per-field rewrite preserves it. + BOOST_AUTO_TEST_CASE(variant_basic_abi_params_succeeds_return_data_throws) + { + auto action = action_trace_v0 { + 0, "alice"_n, "alice"_n, "foo"_n, {}, {0x00, 0x01, 0x02, 0x03}, {0x04, 0x05} + }; + std::variant action_trace_t = action; + + auto abi = chain::abi_def ( {}, + { + { "foo", "", { {"a", "varuint32"}, {"b", "varuint32"}, {"c", "varuint32"}, {"d", "varuint32"} } }, + { "foor", "", { {"e", "varuint32"}, {"f", "varuint32"}, {"g", "varuint32"} } } + }, + { { "foo"_n, "foo", ""} }, + {}, {}, {} + ); + abi.version = "sysio::abi/1."; + abi.action_results = { std::vector{ chain::action_result_def{ "foo"_n, "foor"} } }; + + bool log_called = false; + abi_data_handler handler([&log_called](const exception_with_context&){ log_called = true; }); + handler.add_abi("alice"_n, std::move(abi)); + + fc::variant expected_params = fc::mutable_variant_object() + ("a", 0)("b", 1)("c", 2)("d", 3); + + auto actual = handler.serialize_to_variant(action_trace_t); + + BOOST_TEST(log_called); + BOOST_TEST(to_kv(expected_params) == to_kv(std::get<0>(actual)), boost::test_tools::per_element()); + BOOST_REQUIRE(!std::get<1>(actual).has_value()); + } + + // Per-field independence (streaming path, params throws / return_data succeeds): truncated + // params bytes throw after "params" is half-written; the params catch rewinds and logs but + // does NOT short-circuit (the throw is not abi_recursion_depth_exception), so return_data is + // attempted with a fresh checkpoint and decodes successfully. Output contains return_data only. + BOOST_AUTO_TEST_CASE(streaming_basic_abi_params_throws_return_data_succeeds) + { + auto action = action_trace_v0 { + 0, "alice"_n, "alice"_n, "foo"_n, {}, {0x00, 0x01, 0x02}, {0x04, 0x05, 0x06} + }; + std::variant action_trace_t = action; + + auto abi = chain::abi_def ( {}, + { + { "foo", "", { {"a", "varuint32"}, {"b", "varuint32"}, {"c", "varuint32"}, {"d", "varuint32"} } }, + { "foor", "", { {"e", "varuint32"}, {"f", "varuint32"}, {"g", "varuint32"} } } + }, + { { "foo"_n, "foo", ""} }, + {}, {}, {} + ); + abi.version = "sysio::abi/1."; + abi.action_results = { std::vector{ chain::action_result_def{ "foo"_n, "foor"} } }; + + bool log_called = false; + abi_data_handler handler([&log_called](const exception_with_context&){ log_called = true; }); + handler.add_abi("alice"_n, std::move(abi)); + + std::string out; + { + fc::json_writer w(out); + w.begin_object(); + handler.serialize_to_json_stream(action_trace_t, w); + w.end_object(); + BOOST_REQUIRE(w.balanced()); + } + + BOOST_TEST(log_called); + fc::variant streamed = fc::json::from_string(out); + const auto& streamed_obj = streamed.get_object(); + BOOST_TEST(!streamed_obj.contains("params")); + BOOST_REQUIRE(streamed_obj.contains("return_data")); + + fc::variant expected_return = fc::mutable_variant_object() + ("e", 4)("f", 5)("g", 6); + BOOST_TEST(to_kv(expected_return) == to_kv(streamed_obj["return_data"]), boost::test_tools::per_element()); + } + + // Per-field independence (streaming path, params succeeds / return_data throws): params is + // emitted cleanly; the return_data decode throws on truncated bytes; the return_data catch + // rewinds only its own tokens and logs. Output contains params only; the writer stays + // balanced and the surrounding object closes correctly. + BOOST_AUTO_TEST_CASE(streaming_basic_abi_params_succeeds_return_data_throws) + { + auto action = action_trace_v0 { + 0, "alice"_n, "alice"_n, "foo"_n, {}, {0x00, 0x01, 0x02, 0x03}, {0x04, 0x05} + }; + std::variant action_trace_t = action; + + auto abi = chain::abi_def ( {}, + { + { "foo", "", { {"a", "varuint32"}, {"b", "varuint32"}, {"c", "varuint32"}, {"d", "varuint32"} } }, + { "foor", "", { {"e", "varuint32"}, {"f", "varuint32"}, {"g", "varuint32"} } } + }, + { { "foo"_n, "foo", ""} }, + {}, {}, {} + ); + abi.version = "sysio::abi/1."; + abi.action_results = { std::vector{ chain::action_result_def{ "foo"_n, "foor"} } }; + + bool log_called = false; + abi_data_handler handler([&log_called](const exception_with_context&){ log_called = true; }); + handler.add_abi("alice"_n, std::move(abi)); + + std::string out; + { + fc::json_writer w(out); + w.begin_object(); + handler.serialize_to_json_stream(action_trace_t, w); + w.end_object(); + BOOST_REQUIRE(w.balanced()); + } + + BOOST_TEST(log_called); + fc::variant streamed = fc::json::from_string(out); + const auto& streamed_obj = streamed.get_object(); + BOOST_REQUIRE(streamed_obj.contains("params")); + BOOST_TEST(!streamed_obj.contains("return_data")); + + fc::variant expected_params = fc::mutable_variant_object() + ("a", 0)("b", 1)("c", 2)("d", 3); + BOOST_TEST(to_kv(expected_params) == to_kv(streamed_obj["params"]), boost::test_tools::per_element()); + } + + // Truncated action data triggers an exception inside binary_to_json_stream after "params" has + // already been written. The handler must rewind, log via except_handler, and leave the writer + // observably empty (no half-written "params": fragment) so the surrounding object closes cleanly. + BOOST_AUTO_TEST_CASE(streaming_basic_abi_insufficient_data_rolls_back) + { + auto action = action_trace_v0 { + 0, "alice"_n, "alice"_n, "foo"_n, {}, {0x00, 0x01, 0x02}, {} + }; + std::variant action_trace_t = action; + + auto abi = chain::abi_def ( {}, + { + { "foo", "", { {"a", "varuint32"}, {"b", "varuint32"}, {"c", "varuint32"}, {"d", "varuint32"} } } + }, + { + { "foo"_n, "foo", ""} + }, + {}, {}, {} + ); + abi.version = "sysio::abi/1."; + + bool log_called = false; + abi_data_handler handler([&log_called](const exception_with_context&){ log_called = true; }); + handler.add_abi("alice"_n, std::move(abi)); + + std::string out; + { + fc::json_writer w(out); + w.begin_object(); + handler.serialize_to_json_stream(action_trace_t, w); + w.end_object(); + BOOST_REQUIRE(w.balanced()); + } + + BOOST_TEST(log_called); + BOOST_TEST(out == "{}"); + } + BOOST_AUTO_TEST_SUITE_END() diff --git a/plugins/trace_api_plugin/test/test_responses.cpp b/plugins/trace_api_plugin/test/test_responses.cpp index b5ae39c870..39546ee4eb 100644 --- a/plugins/trace_api_plugin/test/test_responses.cpp +++ b/plugins/trace_api_plugin/test/test_responses.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include @@ -49,6 +50,22 @@ struct response_test_fixture { return fixture.mock_data_handler(action); } + // Streaming peer for the mock: delegates to the same mock_data_handler that the variant path uses, + // then walks the resulting variant via fc::to_json_stream so the streaming output matches the + // variant output byte-for-byte. Production uses abi_serializer::binary_to_json_stream end-to-end; + // here we don't have a real ABI, so we fake parity by re-using the variant tuple. + void serialize_to_json_stream(const action_trace_v0& action, fc::json_writer& w) { + auto [params, return_data] = fixture.mock_data_handler(action); + if (!params.is_null()) { + w.key("params"); + fc::to_json_stream(params, w); + } + if (return_data.has_value()) { + w.key("return_data"); + fc::to_json_stream(*return_data, w); + } + } + response_test_fixture& fixture; }; From e559105ef62b74be44b143554812f239f2c00526 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Mon, 13 Jul 2026 16:01:16 -0500 Subject: [PATCH 59/71] fix streaming http fallback, status codes, and request-body accounting - get_table_rows_stream: rewind the json_writer before the hex fallback so a mid-emission ABI decode failure cannot leave a half-written fragment in the response (structurally invalid JSON on the wire) - bind_stream: deliver stored exceptions via rethrow() so their dynamic type reaches the http status classifier; throw *ptr sliced to fc::exception and turned tx_duplicate(409)/unknown_block(400)/unauthorized(401) into 500s - http_plugin: reserve queued streaming request-body bytes against bytes_in_flight in make_app_thread_url_handler_stream, mirroring the variant handler added for WSA-176; queued streaming bodies previously evaded the accounting entirely - bind_stream: include controller.hpp ahead of plugin_interface.hpp so the header is self-contained (block_signal_params) tests: stream_decode_failure_falls_back_to_hex_valid_json (plugin_test), stream_stored_exception_status_codes and request_body_bytes_in_flight_stream (http_plugin_unit_tests); each fails without its fix --- plugins/chain_plugin/src/chain_plugin.cpp | 8 +- .../include/sysio/http_plugin/bind_stream.hpp | 16 +- plugins/http_plugin/src/http_plugin.cpp | 9 +- plugins/http_plugin/test/CMakeLists.txt | 12 +- plugins/http_plugin/test/unit_tests.cpp | 142 ++++++++++++++++++ tests/test_get_table_rows_page.cpp | 45 ++++++ 6 files changed, 225 insertions(+), 7 deletions(-) diff --git a/plugins/chain_plugin/src/chain_plugin.cpp b/plugins/chain_plugin/src/chain_plugin.cpp index 6b00109ac7..008fecdb48 100644 --- a/plugins/chain_plugin/src/chain_plugin.cpp +++ b/plugins/chain_plugin/src/chain_plugin.cpp @@ -2640,13 +2640,19 @@ read_only::get_table_rows_stream( const read_only::get_table_rows_params& p, con (fc::json_writer& w) mutable { auto emit_value = [&](const std::vector& data) { if (hp.json && !value_type.empty() && !data.empty()) { + // binary_to_json_stream writes tokens into `w` as it walks the ABI, so a + // mid-decode throw (truncated value, ABI/type drift) leaves a half-written + // fragment behind. Rewind every byte written before falling back so the hex + // string below lands at a clean value position instead of producing + // structurally invalid JSON on the wire. + const auto cp = w.checkpoint(); try { abis->binary_to_json_stream(value_type, data, w, abi_serializer::create_yield_function(abi_serializer_max_time), shorten_abi_errors); return; } catch (...) { - // fall through to hex + w.rewind(cp); } } // Empty value or non-json mode: emit hex (matches the variant path's diff --git a/plugins/http_plugin/include/sysio/http_plugin/bind_stream.hpp b/plugins/http_plugin/include/sysio/http_plugin/bind_stream.hpp index 06bb4bcc7d..b74f81865d 100644 --- a/plugins/http_plugin/include/sysio/http_plugin/bind_stream.hpp +++ b/plugins/http_plugin/include/sysio/http_plugin/bind_stream.hpp @@ -2,6 +2,10 @@ #include #include +// controller.hpp must precede plugin_interface.hpp: the latter uses block_signal_params +// (declared in controller.hpp) without including it, so this header includes it to stay +// self-contained regardless of the caller's include order. +#include #include #include @@ -318,7 +322,11 @@ api_entry_stream bind_stream(http_plugin& http, Handle handle, try { chain::t_or_exception result = http_fwd(); if (std::holds_alternative(result)) { - try { throw *std::get(result); } + // rethrow() (virtual) preserves the stored exception's dynamic type so + // classify_current_exception's specific-type catches map it to the right + // status code (tx_duplicate -> 409, unknown_block -> 400, ...). A plain + // `throw *ptr` would slice to fc::exception and turn them all into 500s. + try { std::get(result)->rethrow(); } catch (...) { http_plugin::handle_exception_stream( api_name.c_str(), call_name.c_str(), body, cb); @@ -344,7 +352,7 @@ api_entry_stream bind_stream(http_plugin& http, Handle handle, try { chain::t_or_exception result = http_fwd(); if (std::holds_alternative(result)) { - try { throw *std::get(result); } + try { std::get(result)->rethrow(); } catch (...) { http_plugin::handle_exception_stream( api_name.c_str(), call_name.c_str(), body, cb); @@ -372,7 +380,7 @@ api_entry_stream bind_stream(http_plugin& http, Handle handle, api_name, call_name, resp_code] (chain::plugin_interface::next_function_variant&& result) mutable { if (std::holds_alternative(result)) { - try { throw *std::get(result); } + try { std::get(result)->rethrow(); } catch (...) { http_plugin::handle_exception_stream( api_name.c_str(), call_name.c_str(), body, cb); @@ -389,7 +397,7 @@ api_entry_stream bind_stream(http_plugin& http, Handle handle, try { chain::t_or_exception r = http_fwd(); if (std::holds_alternative(r)) { - try { throw *std::get(r); } + try { std::get(r)->rethrow(); } catch (...) { http_plugin::handle_exception_stream( api_name.c_str(), call_name.c_str(), body, cb); diff --git a/plugins/http_plugin/src/http_plugin.cpp b/plugins/http_plugin/src/http_plugin.cpp index 61db33f5d0..74da7bcd8e 100644 --- a/plugins/http_plugin/src/http_plugin.cpp +++ b/plugins/http_plugin/src/http_plugin.cpp @@ -246,8 +246,15 @@ namespace sysio { return; } + // Mirror the variant-cb handler above: the body is about to sit in the (unbounded) + // app-thread queue, so reserve its bytes against bytes_in_flight until the posted + // work is destroyed (run, thrown, or discarded). Without this, queued streaming + // request bodies accumulate uncounted and evade verify_max_bytes_in_flight. + auto body_in_flight_guard = std::make_shared(conn, b.size()); + app().executor().post( priority, to_queue, - [next_ptr, conn=std::move(conn), r=std::move(r), b=std::move(b), then=std::move(then)]() mutable { + [next_ptr, conn=std::move(conn), r=std::move(r), b=std::move(b), then=std::move(then), + body_in_flight_guard=std::move(body_in_flight_guard)]() mutable { try { if( app().is_quiting() ) return; (*next_ptr)( std::move(r), std::move(b), std::move(then) ); diff --git a/plugins/http_plugin/test/CMakeLists.txt b/plugins/http_plugin/test/CMakeLists.txt index 0b2b76f5af..525ddbd93a 100644 --- a/plugins/http_plugin/test/CMakeLists.txt +++ b/plugins/http_plugin/test/CMakeLists.txt @@ -4,11 +4,21 @@ add_executable( http_plugin_unit_tests ${UNIT_TESTS}) target_link_libraries( http_plugin_unit_tests PRIVATE appbase - PRIVATE http_plugin + PRIVATE http_plugin ${CMAKE_DL_LIBS} ${PLATFORM_SPECIFIC_LIBS} ) +# Header-only chain dependencies for the stored-exception status-code test: +# bind_stream.hpp needs sysio/chain/plugin_interface.hpp (chain_interface has no +# CMake target -- see producer_plugin's CMakeLists for the same pattern) plus the +# chain exception / t_or_exception headers. Everything used is inline, so include +# dirs suffice; linking sysio_chain would drag fc's PUBLIC Boost::unit_test_framework +# onto the link line and collide with this test's header-only Boost.Test. target_include_directories( http_plugin_unit_tests PRIVATE ${CMAKE_SOURCE_DIR}/plugins/http_plugin/include + ${CMAKE_SOURCE_DIR}/plugins/chain_interface/include + ${CMAKE_SOURCE_DIR}/libraries/chain/include + ${CMAKE_BINARY_DIR}/libraries/chain/include + ${CMAKE_SOURCE_DIR}/libraries/chaindb/include ${CMAKE_SOURCE_DIR}/tests ) add_np_test( NAME http_plugin_unit_tests COMMAND http_plugin_unit_tests ) diff --git a/plugins/http_plugin/test/unit_tests.cpp b/plugins/http_plugin/test/unit_tests.cpp index cc8eb49202..284cdc0577 100644 --- a/plugins/http_plugin/test/unit_tests.cpp +++ b/plugins/http_plugin/test/unit_tests.cpp @@ -1,6 +1,10 @@ #include #include #include +#include +#include + +#include #include #include @@ -41,6 +45,8 @@ constexpr uint32_t bytes_in_flight_index = 3; constexpr uint32_t requests_in_flight_index = 4; constexpr uint32_t request_body_bytes_in_flight_index = 5; constexpr uint32_t category_uw_index = 6; +constexpr uint32_t stream_status_codes_index = 7; +constexpr uint32_t stream_request_body_index = 8; // Keeps unsharded IPv6 probe coverage on the historical port 9999. constexpr uint32_t ipv6_probe_index = 2; @@ -916,5 +922,141 @@ BOOST_FIXTURE_TEST_CASE(requests_in_flight, http_plugin_test_fixture) { wait_for_no_requests_in_flight(); } +namespace { + /// Minimal api handle for driving bind_stream's stored-exception delivery paths end-to-end. + /// Both methods deliver a DERIVED exception through the stored fc::exception_ptr alternative + /// (never a live throw), which is the path where a `throw *ptr` would slice the dynamic type + /// and defeat classify_current_exception's specific-type -> status-code mapping. + struct stored_exception_api { + /// dispatch::async -- stored tx_duplicate must classify as 409 Conflict. + void duplicate_trx(sysio::chain::plugin_interface::next_function next) { + next(sysio::chain::plugin_interface::next_function_variant{ + fc::exception_ptr{std::make_shared( + FC_LOG_MESSAGE(error, "duplicate transaction"))}}); + } + + /// dispatch::post -- Phase-2 closure returning a stored unknown_block_exception must + /// classify as 400 Unknown Block. + std::function()> missing_block() { + return []() -> sysio::chain::t_or_exception { + return fc::exception_ptr{std::make_shared( + FC_LOG_MESSAGE(error, "no such block"))}; + }; + } + }; +} + +// Stored exceptions delivered through bind_stream (async next_function and post http_fwd +// alternatives) must reach classify_current_exception with their dynamic type intact so the +// specific-type catches map them to the right HTTP status. A regression to `throw *ptr` +// (slicing to fc::exception) turns both of these into 500 Internal Service Error. +BOOST_FIXTURE_TEST_CASE(stream_stored_exception_status_codes, http_plugin_test_fixture) { + const std::string endpoint = test_http_endpoint("127.0.0.1", stream_status_codes_index); + const std::string server_address = "--http-server-address=" + endpoint; + const std::string port = test_http_port(stream_status_codes_index); + + http_plugin* http_plugin = init({"--plugin=sysio::http_plugin", server_address.c_str()}); + BOOST_REQUIRE(http_plugin); + + http_plugin->add_api_stream({ + bind_stream<&stored_exception_api::duplicate_trx, dispatch::async>( + *http_plugin, stored_exception_api{}, "/v1/test/duplicate_trx", + api_category::node, http_params_types::no_params, 200), + bind_stream<&stored_exception_api::missing_block, dispatch::post>( + *http_plugin, stored_exception_api{}, "/v1/test/missing_block", + api_category::node, http_params_types::no_params, 200), + }, appbase::exec_queue::read_write); + + boost::asio::io_context ctx; + boost::asio::ip::tcp::resolver resolver(ctx); + + auto get_status = [&](const char* path) { + boost::asio::ip::tcp::socket s(ctx, boost::asio::ip::tcp::v4()); + boost::asio::connect(s, resolver.resolve("127.0.0.1", port)); + boost::beast::http::request req(boost::beast::http::verb::get, path, 11); + req.set(http::field::host, endpoint); + boost::beast::http::write(s, req); + + boost::beast::http::response resp; + boost::beast::flat_buffer buffer; + boost::beast::http::read(s, buffer, resp); + return resp.result(); + }; + + BOOST_CHECK(get_status("/v1/test/duplicate_trx") == boost::beast::http::status::conflict); // 409 + BOOST_CHECK(get_status("/v1/test/missing_block") == boost::beast::http::status::bad_request); // 400 +} + +// Streaming-path twin of request_body_bytes_in_flight: request bodies queued to the app thread +// through the streaming-cb registration path (make_app_thread_url_handler_stream) must take the +// same bytes_in_flight reservation the variant path takes. Without it the sampled value below +// reads 0 (queued streaming bodies uncounted -- the WSA-176 vector); with it, >= body size. The +// budget must also drain to zero afterwards, proving the reservation releases exactly once. +BOOST_FIXTURE_TEST_CASE(request_body_bytes_in_flight_stream, http_plugin_test_fixture) { + const std::string endpoint = test_http_endpoint("127.0.0.1", stream_request_body_index); + const std::string server_address = "--http-server-address=" + endpoint; + const std::string port = test_http_port(stream_request_body_index); + + http_plugin* http_plugin = init({"--plugin=sysio::http_plugin", + server_address.c_str(), + "--http-max-bytes-in-flight-mb=64"}); + BOOST_REQUIRE(http_plugin); + + std::atomic observed_body_size{0}; + std::atomic observed_bytes_in_flight{0}; + http_plugin->add_api_stream({{std::string("/echo_body_stream"), api_category::node, + [&](string&&, string&& body, url_response_stream_callback&& cb) { + // Sampled on the app thread while the request body is still in + // flight and before any response bytes are accounted, so it + // reflects only the request-body reservation. + observed_body_size.store(body.size()); + observed_bytes_in_flight.store(http_plugin->bytes_in_flight()); + cb(200, [](fc::json_writer& w) { w.value_string("ok"); }); + }}}, appbase::exec_queue::read_write); + + boost::asio::io_context ctx; + boost::asio::ip::tcp::resolver resolver(ctx); + + // 1 MiB body: comfortably under the 2 MiB max-body-size and the 64 MiB in-flight budget. + const size_t body_size = 1024 * 1024; + + auto post_body = [&]() { + boost::asio::ip::tcp::socket s(ctx, boost::asio::ip::tcp::v4()); + boost::asio::connect(s, resolver.resolve("127.0.0.1", port)); + boost::beast::http::request req(boost::beast::http::verb::post, "/echo_body_stream", 11); + req.keep_alive(true); + req.set(http::field::host, endpoint); + req.body() = std::string(body_size, 'x'); + req.prepare_payload(); + boost::beast::http::write(s, req); + + boost::beast::http::response resp; + boost::beast::flat_buffer buffer; + boost::beast::http::read(s, buffer, resp); + return resp.result(); + }; + + auto wait_for_no_bytes_in_flight = [&]() { + uint16_t max = std::numeric_limits::max(); + while (http_plugin->requests_in_flight() > 0 && --max) + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + BOOST_CHECK(max > 0); + BOOST_CHECK_EQUAL(http_plugin->bytes_in_flight(), 0u); + }; + + BOOST_REQUIRE(post_body() == boost::beast::http::status::ok); + BOOST_CHECK_EQUAL(observed_body_size.load(), body_size); + // Regression assertion: 0 without the reservation in make_app_thread_url_handler_stream. + BOOST_CHECK_GE(observed_bytes_in_flight.load(), body_size); + wait_for_no_bytes_in_flight(); + + // A second request must be admitted and counted the same way: no leaked or double-counted + // reservation. + observed_bytes_in_flight.store(0); + BOOST_REQUIRE(post_body() == boost::beast::http::status::ok); + BOOST_CHECK_GE(observed_bytes_in_flight.load(), body_size); + wait_for_no_bytes_in_flight(); +} + //A warning for future tests: destruction of http_plugin_test_fixture sometimes does not destroy http_plugin's listeners. Tests // added in the future should avoid reusing ports of other tests in http_plugin_unit_tests. diff --git a/tests/test_get_table_rows_page.cpp b/tests/test_get_table_rows_page.cpp index 58779fe72d..1aad01e834 100644 --- a/tests/test_get_table_rows_page.cpp +++ b/tests/test_get_table_rows_page.cpp @@ -384,4 +384,49 @@ BOOST_FIXTURE_TEST_CASE(stream_direct_vs_variant_byte_identical, validating_test } } FC_LOG_AND_RETHROW() +// A row whose value fails ABI decode mid-emission must fall back to a hex string without +// corrupting the response. The streaming path writes tokens as it walks the ABI, so a decode +// failure after some fields were already emitted has to rewind before the hex fallback -- a +// regression here produces structurally invalid JSON on the wire (a hex token appended after a +// half-written value object). Force the failure by re-setting the contract ABI with an extra +// trailing field on `numobj`: stored rows then under-run the declared struct and throw +// "stream unexpectedly ended" after the five real fields were emitted. +BOOST_FIXTURE_TEST_CASE(stream_decode_failure_falls_back_to_hex_valid_json, validating_tester) try { + deploy_contract(*this); + populate_numobjs(*this, 3); + + abi_def abi = fc::json::from_string( + std::string(test_contracts::get_table_test_abi().begin(), + test_contracts::get_table_test_abi().end())).as(); + for (auto& st : abi.structs) { + if (st.name == "numobj") { + st.fields.push_back({"extra_field", "uint64"}); + } + } + set_abi("test"_n, fc::json::to_string(fc::variant(abi), fc::time_point::maximum())); + produce_block(); + + auto ro = make_read_only(*this); + auto p = numobjs_params(); + p.show_payer = true; + + // The streamed body must be parseable JSON even though every row's value decode threw. + const std::string stream_json = run_stream(ro, p); + fc::variant parsed; + BOOST_REQUIRE_NO_THROW(parsed = fc::json::from_string(stream_json)); + + // Every row fell back to the bare hex-string form (not a half-decoded object). + const auto& rows = parsed.get_object()["rows"].get_array(); + BOOST_REQUIRE_EQUAL(rows.size(), 3u); + for (const auto& row : rows) { + BOOST_CHECK(row.get_object()["value"].is_string()); + } + + // And the fallback shape stays byte-identical to the variant path's fallback. + auto via_variant = run(ro, p); + const std::string variant_json = + fc::json::to_string(fc::variant(via_variant), fc::time_point::maximum()); + BOOST_CHECK_EQUAL(variant_json, stream_json); +} FC_LOG_AND_RETHROW() + BOOST_AUTO_TEST_SUITE_END() From 8a49a3eaefcaf990676f9805b31114a2b527b88a Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Mon, 13 Jul 2026 17:28:46 -0500 Subject: [PATCH 60/71] apply pre-pr review cleanups to the streaming json feature - bound the fc::variant/variant_object streaming walkers at json_stream_max_depth (matches json.cpp's DEFAULT_MAX_RECURSION_DEPTH) so a pathological tree throws instead of overflowing the stack - drop value_double from the abi sinks: no walker path calls it and the two sinks disagreed on shape (quoted fixed-precision vs bare number) - replace the abi stream dispatch's custom int64/uint64/float32/float64 emitters with the shared fc::to_json_stream scalars; deletes the stringstream double formatter (locale-sensitive, per-field allocation) - document on add_specialized_unpack_pack that overrides are not reflected on the streaming dispatch; delete the fail-loud stream-side placeholder - hash from_string factories (sha1/224/256/512, sha3, ripemd160, keccak256, blake3) reject odd-length hex, restoring the strictness the pre-trait from_variant vector path enforced - log per-account abi parse failures in build_resolver_from_captured_abis instead of swallowing them silently - emit container elements unqualified in to_json_stream_from_map/_from_set: the qualified call suppressed ADL, so late-declared element overloads (e.g. fc::crypto::public_key in flat_set responses) fell into the reflector primary template - shared constants for the json quoting threshold, int64 decimal buffer, and writer reserve slack; rewind() asserts its append-only contract; writer frame members default-initialized; value_double throws fc::assert_exception; to_chars error codes asserted - dedupe the softfloat128 hex spelling, the variant_object/mutable walkers, and the flat_set/multiset emit loops; remove dead to_json_stream(log_context) - benchmark harness reports and skips a throwing scenario instead of dying on an uncaught exception - url::from_string takes string_view; stray trailing space removed from the recursion-depth message; unnecessary .template on a concrete sink; unused json.hpp include dropped; missing added to abi_sinks.hpp tests: fc_variant_walker_depth_guard pins the new depth cap; odd-length-hex negative checks folded into every hash roundtrip case; value_double expectations moved to fc::assert_exception --- benchmark/benchmark.cpp | 13 ++++- libraries/chain/abi_serializer.cpp | 56 +++---------------- .../include/sysio/chain/abi_serializer.hpp | 15 ++--- .../chain/include/sysio/chain/abi_sinks.hpp | 3 +- .../include/sysio/chain/database_utils.hpp | 27 +++++---- .../include/fc/container/container_detail.hpp | 18 +++++- libraries/libfc/include/fc/container/flat.hpp | 10 +--- libraries/libfc/include/fc/crypto/blake3.hpp | 8 ++- .../libfc/include/fc/crypto/keccak256.hpp | 4 +- .../libfc/include/fc/crypto/ripemd160.hpp | 4 +- libraries/libfc/include/fc/crypto/sha1.hpp | 4 +- libraries/libfc/include/fc/crypto/sha224.hpp | 4 +- libraries/libfc/include/fc/crypto/sha256.hpp | 4 +- libraries/libfc/include/fc/crypto/sha3.hpp | 4 +- libraries/libfc/include/fc/crypto/sha512.hpp | 4 +- libraries/libfc/include/fc/io/json_stream.hpp | 55 ++++++++++++------ .../libfc/include/fc/log/log_message.hpp | 1 - libraries/libfc/include/fc/network/url.hpp | 2 +- .../libfc/include/fc/reflect/json_stream.hpp | 15 ++--- libraries/libfc/include/fc/variant.hpp | 10 ++-- libraries/libfc/src/crypto/keccak256.cpp | 8 +++ libraries/libfc/src/crypto/ripemd160.cpp | 9 +++ libraries/libfc/src/crypto/sha1.cpp | 9 +++ libraries/libfc/src/crypto/sha224.cpp | 9 +++ libraries/libfc/src/crypto/sha256.cpp | 8 +++ libraries/libfc/src/crypto/sha3.cpp | 8 +++ libraries/libfc/src/crypto/sha512.cpp | 9 +++ libraries/libfc/src/log/log_message.cpp | 5 -- libraries/libfc/src/variant.cpp | 12 ++-- libraries/libfc/src/variant_object.cpp | 44 ++++++++------- .../libfc/test/crypto/test_hash_functions.cpp | 7 +++ libraries/libfc/test/io/test_json_stream.cpp | 33 ++++++++++- .../sysio/chain_plugin/chain_plugin.hpp | 13 ++++- plugins/chain_plugin/src/chain_plugin.cpp | 2 +- .../include/sysio/http_plugin/bind_stream.hpp | 19 ++++--- .../trace_api_plugin/src/abi_data_handler.cpp | 2 +- 36 files changed, 292 insertions(+), 166 deletions(-) diff --git a/benchmark/benchmark.cpp b/benchmark/benchmark.cpp index 58384ce500..914c511919 100644 --- a/benchmark/benchmark.cpp +++ b/benchmark/benchmark.cpp @@ -80,7 +80,18 @@ void benchmarking(const std::string& name, const std::function& func, for (auto i = 0U; i < runs; ++i) { auto start_time = std::chrono::high_resolution_clock::now(); - func(); + // A throwing scenario (e.g. an abi-serialization deadline tripped by a transient + // machine stall during a long -r run) reports and skips the scenario instead of + // taking the whole benchmark process down with an uncaught exception. + try { + func(); + } catch (const std::exception& e) { + std::cout << name << ": SKIPPED after run " << i << " -- " << e.what() << "\n"; + return; + } catch (...) { + std::cout << name << ": SKIPPED after run " << i << " -- unknown exception\n"; + return; + } auto end_time = std::chrono::high_resolution_clock::now(); uint64_t duration = std::chrono::duration_cast(end_time - start_time).count(); diff --git a/libraries/chain/abi_serializer.cpp b/libraries/chain/abi_serializer.cpp index a7f587ccc1..cd0b232057 100644 --- a/libraries/chain/abi_serializer.cpp +++ b/libraries/chain/abi_serializer.cpp @@ -156,15 +156,6 @@ namespace sysio::chain { const abi_serializer::yield_function_t&, fc::json_writer&)>; - // Format double with the same fixed-precision spelling used by `variant::as_string()` - // for double_type variants (digits10 + 2, std::fixed). The variant path emits this - // form quoted via to_json_stream(variant) -> w.value_string(v.as_string()). - inline std::string double_as_string_for_abi(double d) { - std::stringstream ss; - ss << std::setprecision(std::numeric_limits::digits10 + 2) << std::fixed << d; - return ss.str(); - } - // Static dispatch table for the streaming side of `unpack_built_in`. Built once on // first access and then read-only. Variant-side overrides via // `add_specialized_unpack_pack` are not reflected here; the streaming consumers @@ -173,29 +164,11 @@ namespace sysio::chain { static const auto map = []() { std::map> m; + // fc::to_json_stream provides the variant-path shapes for every scalar here: + // int64/uint64 quote past fc::json_integer_quote_magnitude, float32/float64 + // quote with the digits10+2 fixed-precision to_chars form. auto generic = [](const auto& v, fc::json_writer& w) { fc::to_json_stream(v, w); }; - // int64/uint64: variant path quotes when |v| > 0xffffffff (JS precision guard). - constexpr int64_t i64_quote_above = 0xffffffffLL; - constexpr int64_t i64_quote_below = -0xffffffffLL; - constexpr uint64_t u64_quote_above = 0xffffffffULL; - auto emit_int64 = [=](int64_t v, fc::json_writer& w) { - if( v > i64_quote_above || v < i64_quote_below ) w.value_string(std::to_string(v)); - else w.value_int64(v); - }; - auto emit_uint64 = [=](uint64_t v, fc::json_writer& w) { - if( v > u64_quote_above ) w.value_string(std::to_string(v)); - else w.value_uint64(v); - }; - - // float / double: variant path always quotes with digits10+2 fixed precision. - auto emit_double = [](double v, fc::json_writer& w) { - w.value_string(double_as_string_for_abi(v)); - }; - auto emit_float = [](float v, fc::json_writer& w) { - w.value_string(double_as_string_for_abi(static_cast(v))); - }; - m.emplace("bool", pack_unpack_stream(generic)); m.emplace("int8", pack_unpack_stream(generic)); m.emplace("uint8", pack_unpack_stream(generic)); @@ -203,15 +176,15 @@ namespace sysio::chain { m.emplace("uint16", pack_unpack_stream(generic)); m.emplace("int32", pack_unpack_stream(generic)); m.emplace("uint32", pack_unpack_stream(generic)); - m.emplace("int64", pack_unpack_stream(emit_int64)); - m.emplace("uint64", pack_unpack_stream(emit_uint64)); + m.emplace("int64", pack_unpack_stream(generic)); + m.emplace("uint64", pack_unpack_stream(generic)); m.emplace("int128", pack_unpack_stream(generic)); m.emplace("uint128", pack_unpack_stream(generic)); m.emplace("varint32", pack_unpack_stream(generic)); m.emplace("varuint32", pack_unpack_stream(generic)); - m.emplace("float32", pack_unpack_stream(emit_float)); - m.emplace("float64", pack_unpack_stream(emit_double)); + m.emplace("float32", pack_unpack_stream(generic)); + m.emplace("float64", pack_unpack_stream(generic)); m.emplace("float128", pack_unpack_stream(generic)); m.emplace("time_point", pack_unpack_stream(generic)); @@ -256,19 +229,6 @@ namespace sysio::chain { built_in_types[name] = std::move( unpack_pack ); } - void abi_serializer::add_specialized_stream_unpack( const string& /*name*/, - stream_unpack_function /*unpack*/ ) { - // The streaming dispatch (get_built_in_stream_unpacks) is a static const map shared across - // instances. Wiring per-instance overrides would require restructuring that into an - // instance-owned map plus a fallback-to-static lookup, plus a parallel change in - // stream_sink::unpack_built_in to consult the instance map first. No callers need this - // today (add_specialized_unpack_pack itself has zero in-tree callers); intentional fail-loud - // until a real use case shows up. - FC_ASSERT(false, "abi_serializer::add_specialized_stream_unpack is not yet implemented: " - "the streaming dispatch is a static const map; per-instance overrides " - "require restructuring it into an instance-owned map with a static fallback"); - } - const std::pair* abi_serializer::find_built_in(std::string_view type) const noexcept { auto it = built_in_types.find(type); @@ -1365,7 +1325,6 @@ void variant_sink::key(std::string_view k) { void variant_sink::value_string(std::string_view s) { emit_value(fc::variant(s)); } void variant_sink::value_int64(int64_t n) { emit_value(fc::variant(n)); } void variant_sink::value_uint64(uint64_t n) { emit_value(fc::variant(n)); } -void variant_sink::value_double(double d) { emit_value(fc::variant(d)); } void variant_sink::value_bool(bool b) { emit_value(fc::variant(b)); } void variant_sink::value_null() { emit_value(fc::variant()); } @@ -1475,7 +1434,6 @@ void stream_sink::key(std::string_view k) { w_.key(k); } void stream_sink::value_string(std::string_view s) { w_.value_string(s); on_value_emitted(); } void stream_sink::value_int64(int64_t n) { w_.value_int64(n); on_value_emitted(); } void stream_sink::value_uint64(uint64_t n) { w_.value_uint64(n); on_value_emitted(); } -void stream_sink::value_double(double d) { w_.value_double(d); on_value_emitted(); } void stream_sink::value_bool(bool b) { w_.value_bool(b); on_value_emitted(); } void stream_sink::value_null() { w_.value_null(); on_value_emitted(); } diff --git a/libraries/chain/include/sysio/chain/abi_serializer.hpp b/libraries/chain/include/sysio/chain/abi_serializer.hpp index f039f8f7ee..fb2645c6d4 100644 --- a/libraries/chain/include/sysio/chain/abi_serializer.hpp +++ b/libraries/chain/include/sysio/chain/abi_serializer.hpp @@ -149,17 +149,14 @@ struct abi_serializer { typedef std::function&, bool, bool, const abi_serializer::yield_function_t&)> unpack_function; typedef std::function&, bool, bool, const abi_serializer::yield_function_t&)> pack_function; + /// Registers a per-instance (unpack, pack) override for a built-in type on the + /// VARIANT dispatch only. The streaming dispatch (`binary_to_json_stream`) uses a + /// static table consulted first, so an override registered here for a type that + /// table covers is NOT reflected on the streaming path -- the two paths would emit + /// different shapes for that type. No in-tree callers exist today; if one appears, + /// the streaming dispatch needs a per-instance map with a static fallback first. void add_specialized_unpack_pack( const string& name, std::pair unpack_pack ); - /// Streaming-side companion to `add_specialized_unpack_pack`. Currently asserts - /// not-implemented: the streaming dispatch is a static const map and registering - /// per-instance overrides would require restructuring it into a per-instance map. - /// Surfaced now so future callers see the gap at the API level rather than getting - /// a silent variant-fallback (slow path) for their override. - using stream_unpack_function = std::function&, bool, bool, - const yield_function_t&, fc::json_writer&)>; - void add_specialized_stream_unpack( const string& name, stream_unpack_function unpack ); - /// Lookup helper used by the streaming walker sinks. Returns a pointer to the /// (unpack, pack) entry registered for `type`, or `nullptr` if it is not a /// built-in. Const view; the underlying map is shared with `binary_to_variant` diff --git a/libraries/chain/include/sysio/chain/abi_sinks.hpp b/libraries/chain/include/sysio/chain/abi_sinks.hpp index 58cd06d20f..014fdcea91 100644 --- a/libraries/chain/include/sysio/chain/abi_sinks.hpp +++ b/libraries/chain/include/sysio/chain/abi_sinks.hpp @@ -18,6 +18,7 @@ #include #include #include +#include namespace sysio::chain { struct abi_serializer; @@ -74,7 +75,6 @@ class variant_sink { void value_uint16(uint16_t n) { value_uint64(n); } void value_int8(int8_t n) { value_int64(n); } void value_uint8(uint8_t n) { value_uint64(n); } - void value_double(double d); void value_bool(bool b); void value_null(); void value_hex(const char* data, size_t size); @@ -173,7 +173,6 @@ class stream_sink { void value_uint16(uint16_t n) { value_uint64(n); } void value_int8(int8_t n) { value_int64(n); } void value_uint8(uint8_t n) { value_uint64(n); } - void value_double(double d); void value_bool(bool b); void value_null(); void value_hex(const char* data, size_t size); diff --git a/libraries/chain/include/sysio/chain/database_utils.hpp b/libraries/chain/include/sysio/chain/database_utils.hpp index 55638a2f25..babf8b4efd 100644 --- a/libraries/chain/include/sysio/chain/database_utils.hpp +++ b/libraries/chain/include/sysio/chain/database_utils.hpp @@ -658,24 +658,29 @@ namespace fc { double_to_float64(double_f, f); } + namespace detail { + /// fc's canonical softfloat128 spelling: "0x" + 16 little-endian hex bytes. + /// Assumes platform is little endian and hex representation of the 128-bit + /// integer is in little endian order. Shared by to_variant / to_json_stream + /// so the two emission paths cannot drift. + inline std::string softfloat128_to_hex_string( const softfloat128_t& f ) { + char as_bytes[sizeof(sysio::chain::uint128_t)]; + memcpy(as_bytes, &f, sizeof(as_bytes)); + std::string s = "0x"; + s.append( to_hex( as_bytes, sizeof(as_bytes) ) ); + return s; + } + } + inline void to_variant( const softfloat128_t& f, variant& v ) { - // Assumes platform is little endian and hex representation of 128-bit integer is in little endian order. - char as_bytes[sizeof(sysio::chain::uint128_t)]; - memcpy(as_bytes, &f, sizeof(as_bytes)); - std::string s = "0x"; - s.append( to_hex( as_bytes, sizeof(as_bytes) ) ); - v = s; + v = detail::softfloat128_to_hex_string( f ); } /// JSON shape mirrors to_variant: "0x" with little-endian bytes. inline void to_json_stream( const softfloat128_t& f, json_writer& w ) { - char as_bytes[sizeof(sysio::chain::uint128_t)]; - memcpy(as_bytes, &f, sizeof(as_bytes)); - std::string s = "0x"; - s.append( to_hex( as_bytes, sizeof(as_bytes) ) ); - w.value_string(s); + w.value_string( detail::softfloat128_to_hex_string( f ) ); } inline diff --git a/libraries/libfc/include/fc/container/container_detail.hpp b/libraries/libfc/include/fc/container/container_detail.hpp index 32bd5fa18b..6c36d7c073 100644 --- a/libraries/libfc/include/fc/container/container_detail.hpp +++ b/libraries/libfc/include/fc/container/container_detail.hpp @@ -131,12 +131,28 @@ namespace fc { } /// JSON shape mirrors to_variant_from_map: an array of [key, value] pairs. + /// The element emit is deliberately UNQUALIFIED: a qualified fc::to_json_stream + /// call would fix the overload set at this header's definition point and + /// suppress ADL, so element types whose overload is declared later (e.g. + /// fc::crypto::public_key) would wrongly fall into the reflector primary. template class Map, typename K, typename V, typename... U > void to_json_stream_from_map( const Map< K, V, U... >& m, fc::json_writer& w ) { FC_ASSERT( m.size() <= MAX_NUM_ARRAY_ELEMENTS ); w.begin_array(); for( const auto& item : m ) { - fc::to_json_stream( item, w ); + to_json_stream( item, w ); + } + w.end_array(); + } + + /// JSON shape mirrors to_variant_from_set: an array of elements. Same + /// unqualified-emit rule as to_json_stream_from_map above. + template class Set, typename T, typename... U> + void to_json_stream_from_set( const Set& s, fc::json_writer& w ) { + FC_ASSERT( s.size() <= MAX_NUM_ARRAY_ELEMENTS ); + w.begin_array(); + for( const auto& item : s ) { + to_json_stream( item, w ); } w.end_array(); } diff --git a/libraries/libfc/include/fc/container/flat.hpp b/libraries/libfc/include/fc/container/flat.hpp index d2fec70cc5..9f8cffdce3 100644 --- a/libraries/libfc/include/fc/container/flat.hpp +++ b/libraries/libfc/include/fc/container/flat.hpp @@ -146,10 +146,7 @@ namespace fc { template void to_json_stream( const flat_set< T, U... >& s, json_writer& w ) { - FC_ASSERT( s.size() <= MAX_NUM_ARRAY_ELEMENTS ); - w.begin_array(); - for( const auto& e : s ) to_json_stream( e, w ); - w.end_array(); + detail::to_json_stream_from_set( s, w ); } template void to_variant( const flat_set< T, U... >& s, fc::variant& vo ) { @@ -162,10 +159,7 @@ namespace fc { template void to_json_stream( const flat_multiset< T, U... >& s, json_writer& w ) { - FC_ASSERT( s.size() <= MAX_NUM_ARRAY_ELEMENTS ); - w.begin_array(); - for( const auto& e : s ) to_json_stream( e, w ); - w.end_array(); + detail::to_json_stream_from_set( s, w ); } template void to_variant( const flat_multiset< T, U... >& s, fc::variant& vo ) { diff --git a/libraries/libfc/include/fc/crypto/blake3.hpp b/libraries/libfc/include/fc/crypto/blake3.hpp index 880955e301..6eb7892e13 100644 --- a/libraries/libfc/include/fc/crypto/blake3.hpp +++ b/libraries/libfc/include/fc/crypto/blake3.hpp @@ -35,7 +35,13 @@ class blake3 { std::string str() const { return to_hex(to_char_span()); } std::string to_string() const { return str(); } - static blake3 from_string(std::string_view s) { return blake3(s); } + /// Validating parse used by the FC_SERIALIZE_AS_STRING trait: rejects odd-length + /// hex (a 63-char string would otherwise round up to 32 decoded bytes and pass the + /// constructor's size check with a zero-extended final nibble). + static blake3 from_string(std::string_view s) { + FC_ASSERT(s.size() % 2 == 0, "blake3 hex string length must be even, got {}", s.size()); + return blake3(s); + } uint8_t* data() { return _hash; } const uint8_t* data() const { return _hash; } diff --git a/libraries/libfc/include/fc/crypto/keccak256.hpp b/libraries/libfc/include/fc/crypto/keccak256.hpp index c518cfe73e..9106b10705 100644 --- a/libraries/libfc/include/fc/crypto/keccak256.hpp +++ b/libraries/libfc/include/fc/crypto/keccak256.hpp @@ -25,7 +25,9 @@ class keccak256 { // in hex std::string str() const; std::string to_string() const { return str(); } - static keccak256 from_string(std::string_view s) { return keccak256(s); } + /// Validating parse used by the FC_SERIALIZE_AS_STRING trait: rejects odd-length + /// hex (the strictness the pre-trait from_variant vector path enforced). + static keccak256 from_string(std::string_view s); const uint8_t* data() const { return _hash; } constexpr size_t data_size() const { return byte_size; } diff --git a/libraries/libfc/include/fc/crypto/ripemd160.hpp b/libraries/libfc/include/fc/crypto/ripemd160.hpp index 8df0c87a0a..697ee52a4e 100644 --- a/libraries/libfc/include/fc/crypto/ripemd160.hpp +++ b/libraries/libfc/include/fc/crypto/ripemd160.hpp @@ -18,7 +18,9 @@ class ripemd160 : public add_packhash_to_hash std::string str()const; std::string to_string() const { return str(); } - static ripemd160 from_string(std::string_view s) { return ripemd160(s); } + /// Validating parse used by the FC_SERIALIZE_AS_STRING trait: rejects odd-length + /// hex (the strictness the pre-trait from_variant vector path enforced). + static ripemd160 from_string(std::string_view s); char* data()const; size_t data_size()const { return 160/8; } diff --git a/libraries/libfc/include/fc/crypto/sha1.hpp b/libraries/libfc/include/fc/crypto/sha1.hpp index 3af68e3ba0..6e534a3588 100644 --- a/libraries/libfc/include/fc/crypto/sha1.hpp +++ b/libraries/libfc/include/fc/crypto/sha1.hpp @@ -14,7 +14,9 @@ class sha1 : public add_packhash_to_hash std::string str()const; std::string to_string() const { return str(); } - static sha1 from_string(std::string_view s) { return sha1(s); } + /// Validating parse used by the FC_SERIALIZE_AS_STRING trait: rejects odd-length + /// hex (the strictness the pre-trait from_variant vector path enforced). + static sha1 from_string(std::string_view s); operator std::string()const; char* data(); diff --git a/libraries/libfc/include/fc/crypto/sha224.hpp b/libraries/libfc/include/fc/crypto/sha224.hpp index 9985f25191..5662fb882e 100644 --- a/libraries/libfc/include/fc/crypto/sha224.hpp +++ b/libraries/libfc/include/fc/crypto/sha224.hpp @@ -17,7 +17,9 @@ class sha224 : public add_packhash_to_hash std::string str()const; std::string to_string() const { return str(); } - static sha224 from_string(std::string_view s) { return sha224(s); } + /// Validating parse used by the FC_SERIALIZE_AS_STRING trait: rejects odd-length + /// hex (the strictness the pre-trait from_variant vector path enforced). + static sha224 from_string(std::string_view s); operator std::string()const; char* data(); diff --git a/libraries/libfc/include/fc/crypto/sha256.hpp b/libraries/libfc/include/fc/crypto/sha256.hpp index 1a34100696..ab18c36661 100644 --- a/libraries/libfc/include/fc/crypto/sha256.hpp +++ b/libraries/libfc/include/fc/crypto/sha256.hpp @@ -29,7 +29,9 @@ class sha256 : public add_packhash_to_hash std::string str()const; std::string to_string() const { return str(); } - static sha256 from_string(std::string_view s) { return sha256(s); } + /// Validating parse used by the FC_SERIALIZE_AS_STRING trait: rejects odd-length + /// hex (the strictness the pre-trait from_variant vector path enforced). + static sha256 from_string(std::string_view s); std::string short_id()const; const char* data()const; diff --git a/libraries/libfc/include/fc/crypto/sha3.hpp b/libraries/libfc/include/fc/crypto/sha3.hpp index 1c484c8211..0b87183ee0 100644 --- a/libraries/libfc/include/fc/crypto/sha3.hpp +++ b/libraries/libfc/include/fc/crypto/sha3.hpp @@ -20,7 +20,9 @@ class sha3 std::string str() const; std::string to_string() const { return str(); } - static sha3 from_string(std::string_view s) { return sha3(s); } + /// Validating parse used by the FC_SERIALIZE_AS_STRING trait: rejects odd-length + /// hex (the strictness the pre-trait from_variant vector path enforced). + static sha3 from_string(std::string_view s); operator std::string() const; const char *data() const; diff --git a/libraries/libfc/include/fc/crypto/sha512.hpp b/libraries/libfc/include/fc/crypto/sha512.hpp index 5b2c3489bd..c2f3205f1f 100644 --- a/libraries/libfc/include/fc/crypto/sha512.hpp +++ b/libraries/libfc/include/fc/crypto/sha512.hpp @@ -15,7 +15,9 @@ class sha512 : public add_packhash_to_hash std::string str()const; std::string to_string() const { return str(); } - static sha512 from_string(std::string_view s) { return sha512(s); } + /// Validating parse used by the FC_SERIALIZE_AS_STRING trait: rejects odd-length + /// hex (the strictness the pre-trait from_variant vector path enforced). + static sha512 from_string(std::string_view s); operator std::string()const; char* data(); diff --git a/libraries/libfc/include/fc/io/json_stream.hpp b/libraries/libfc/include/fc/io/json_stream.hpp index 16cc1e3912..6fcf28684b 100644 --- a/libraries/libfc/include/fc/io/json_stream.hpp +++ b/libraries/libfc/include/fc/io/json_stream.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -8,13 +9,33 @@ #include #include #include -#include #include #include #include namespace fc { +/// Slack the writer guarantees in the output buffer at construction so typical responses +/// append without an early reallocation. +inline constexpr size_t json_writer_initial_reserve = 4096; + +/// Decimal-digit buffer that fits any int64/uint64 (uint64 max = 20 digits, plus sign and slack). +inline constexpr size_t int64_decimal_buf_size = std::numeric_limits::digits10 + 3; +static_assert(int64_decimal_buf_size >= std::numeric_limits::digits10 + 1); + +/// Shortest-roundtrip std::to_chars of a finite double fits well within this. +inline constexpr size_t double_shortest_buf_size = 32; + +/// 64-bit integers whose magnitude exceeds this are emitted as quoted JSON strings to +/// preserve precision past JS's 2^53 mantissa. Single source of truth for the streaming +/// emitters; matches fc::json::to_string's emission (libfc/src/io/json.cpp int64/uint64 cases). +inline constexpr int64_t json_integer_quote_magnitude = 0xffffffff; + +/// Depth cap for the fc::variant / variant_object streaming walkers; matches +/// DEFAULT_MAX_RECURSION_DEPTH in fc/io/json.hpp so the streaming path throws where +/// fc::json::to_string's yield would. +inline constexpr uint32_t json_stream_max_depth = 200; + /** * Streaming JSON writer that emits tokens directly into an output std::string. * @@ -42,8 +63,8 @@ class json_writer { // Enough room for a reasonable response without reallocation; callers that know // the expected size can reserve() themselves before constructing the writer. // Skip if the caller already has slack so we don't risk a spec-permitted shrink. - if (out_.capacity() - out_.size() < 4096) { - out_.reserve(out_.size() + 4096); + if (out_.capacity() - out_.size() < json_writer_initial_reserve) { + out_.reserve(out_.size() + json_writer_initial_reserve); } } @@ -110,16 +131,14 @@ class json_writer { // NaN / +-inf have no JSON representation: any encoder would emit non-conforming // tokens that no parser will accept. Reject before mutating the output buffer // so the writer's frame state stays consistent on throw. - if (!std::isfinite(d)) { - throw std::invalid_argument("fc::json_writer::value_double: non-finite double cannot be JSON-encoded"); - } + FC_ASSERT(std::isfinite(d), "fc::json_writer::value_double: non-finite double cannot be JSON-encoded"); value_prefix(); // std::to_chars is locale-independent (period radix always) and shortest-roundtrip: // the parsed-back double is bit-identical to the input. snprintf with %g would // honor LC_NUMERIC and emit comma radix in non-C locales -- invalid JSON. - char buf[32]; + char buf[double_shortest_buf_size]; auto r = std::to_chars(buf, buf + sizeof(buf), d); - assert(r.ec == std::errc{}); // unreachable: finite double + 32-byte buffer always succeeds + assert(r.ec == std::errc{}); // unreachable: finite double + buffer sized for shortest-roundtrip out_.append(buf, static_cast(r.ptr - buf)); } @@ -215,6 +234,11 @@ class json_writer { }; } void rewind(const checkpoint_t& cp) noexcept { + // rewind() only discards tokens APPENDED since checkpoint(); a caller that closed + // frames past the checkpoint and re-opened new ones would resize the stack UP here + // with value-initialized frames -- silent corruption, so the contract is asserted. + assert(out_.size() >= cp.buf_size); + assert(stack_.size() >= cp.stack_size); out_.resize(cp.buf_size); stack_.resize(cp.stack_size); if (!stack_.empty()) stack_.back().has_item = cp.surviving_top_has_item; @@ -224,8 +248,8 @@ class json_writer { private: enum class context : uint8_t { object, array }; struct frame { - context ctx; - bool has_item; // true once one element or key:value has been emitted in this frame + context ctx = context::object; + bool has_item = false; // true once one element or key:value has been emitted in this frame }; void value_prefix() { @@ -248,18 +272,17 @@ class json_writer { } // std::to_chars is non-throwing, locale-independent, and avoids the format-string - // parsing overhead that snprintf pays on every call. Buffer sized for the longest - // possible decimal of int64/uint64 (digits10+1 plus sign plus slack). - static constexpr size_t int64_decimal_buf = std::numeric_limits::digits10 + 3; - static_assert(int64_decimal_buf >= std::numeric_limits::digits10 + 1); + // parsing overhead that snprintf pays on every call. void append_signed(int64_t n) { - char buf[int64_decimal_buf]; + char buf[int64_decimal_buf_size]; auto r = std::to_chars(buf, buf + sizeof(buf), n); + assert(r.ec == std::errc{}); // unreachable: buffer sized for any int64 decimal out_.append(buf, static_cast(r.ptr - buf)); } void append_unsigned(uint64_t n) { - char buf[int64_decimal_buf]; + char buf[int64_decimal_buf_size]; auto r = std::to_chars(buf, buf + sizeof(buf), n); + assert(r.ec == std::errc{}); // unreachable: buffer sized for any uint64 decimal out_.append(buf, static_cast(r.ptr - buf)); } diff --git a/libraries/libfc/include/fc/log/log_message.hpp b/libraries/libfc/include/fc/log/log_message.hpp index b55cd981f7..5402a923aa 100644 --- a/libraries/libfc/include/fc/log/log_message.hpp +++ b/libraries/libfc/include/fc/log/log_message.hpp @@ -156,7 +156,6 @@ namespace fc /// (log_message has a private impl pointer rather than reflected fields, so we can't walk it directly). class json_writer; void to_json_stream( const log_message& m, json_writer& w ); - void to_json_stream( const log_context& l, json_writer& w ); typedef std::vector log_messages; diff --git a/libraries/libfc/include/fc/network/url.hpp b/libraries/libfc/include/fc/network/url.hpp index 9a7ff4eb81..56c9b84e78 100644 --- a/libraries/libfc/include/fc/network/url.hpp +++ b/libraries/libfc/include/fc/network/url.hpp @@ -41,7 +41,7 @@ namespace fc { operator std::string()const; std::string to_string() const { return std::string(*this); } - static url from_string(const std::string& s) { return url(s); } + static url from_string(std::string_view s) { return url(std::string(s)); } //// file, ssh, tcp, http, ssl, etc... std::string proto()const; diff --git a/libraries/libfc/include/fc/reflect/json_stream.hpp b/libraries/libfc/include/fc/reflect/json_stream.hpp index 02f14904bd..7b3c228df2 100644 --- a/libraries/libfc/include/fc/reflect/json_stream.hpp +++ b/libraries/libfc/include/fc/reflect/json_stream.hpp @@ -45,15 +45,12 @@ inline void to_json_stream(int16_t n, json_writer& w) { w.value_int inline void to_json_stream(uint16_t n, json_writer& w) { w.value_uint16(n); } inline void to_json_stream(int32_t n, json_writer& w) { w.value_int32(n); } inline void to_json_stream(uint32_t n, json_writer& w) { w.value_uint32(n); } -// 64-bit integers with magnitude > 0xffffffff are emitted as JSON strings to -// preserve precision past JS's 2^53 mantissa. Matches fc::variant's emission -// (libfc/src/io/json.cpp, int64_type / uint64_type cases). Writers stay raw; -// callers that want a literal big number use value_int64 / value_uint64 directly. -// Buffer size: 20 digits (uint64 max) + sign + slack. -inline constexpr size_t int64_decimal_buf_size = std::numeric_limits::digits10 + 3; - +// 64-bit integers with magnitude > json_integer_quote_magnitude are emitted as JSON +// strings to preserve precision past JS's 2^53 mantissa. Matches fc::variant's emission +// (libfc/src/io/json.cpp, int64_type / uint64_type cases). Writers stay raw; callers +// that want a literal big number use value_int64 / value_uint64 directly. inline void to_json_stream(int64_t n, json_writer& w) { - if (n > 0xffffffffLL || n < -0xffffffffLL) { + if (n > json_integer_quote_magnitude || n < -json_integer_quote_magnitude) { char buf[int64_decimal_buf_size]; auto r = std::to_chars(buf, buf + sizeof(buf), n); w.value_string(std::string_view(buf, r.ptr - buf)); @@ -62,7 +59,7 @@ inline void to_json_stream(int64_t n, json_writer& w) { } } inline void to_json_stream(uint64_t n, json_writer& w) { - if (n > 0xffffffffULL) { + if (n > static_cast(json_integer_quote_magnitude)) { char buf[int64_decimal_buf_size]; auto r = std::to_chars(buf, buf + sizeof(buf), n); w.value_string(std::string_view(buf, r.ptr - buf)); diff --git a/libraries/libfc/include/fc/variant.hpp b/libraries/libfc/include/fc/variant.hpp index fcb400ec1c..3c26c45486 100644 --- a/libraries/libfc/include/fc/variant.hpp +++ b/libraries/libfc/include/fc/variant.hpp @@ -51,10 +51,12 @@ namespace fc // variant.cpp / variant_object.cpp. Walks the variant tree directly into the writer // -- no intermediate fc::json::to_string string allocation. Lets reflected structs // that embed a variant or variant_object field stream through the same writer as - // their other members. - void to_json_stream( const variant& v, json_writer& w ); - void to_json_stream( const variant_object& vo, json_writer& w ); - void to_json_stream( const mutable_variant_object& vo, json_writer& w ); + // their other members. `max_depth` bounds the recursion the same way + // fc::json::to_string's yield does; a deeper tree throws instead of overflowing the + // stack. + void to_json_stream( const variant& v, json_writer& w, uint32_t max_depth = json_stream_max_depth ); + void to_json_stream( const variant_object& vo, json_writer& w, uint32_t max_depth = json_stream_max_depth ); + void to_json_stream( const mutable_variant_object& vo, json_writer& w, uint32_t max_depth = json_stream_max_depth ); struct blob { std::vector data; }; diff --git a/libraries/libfc/src/crypto/keccak256.cpp b/libraries/libfc/src/crypto/keccak256.cpp index 5e7a8b1eee..7a01c16cef 100644 --- a/libraries/libfc/src/crypto/keccak256.cpp +++ b/libraries/libfc/src/crypto/keccak256.cpp @@ -59,4 +59,12 @@ void keccak256::encoder::reset() { data.clear(); } + keccak256 keccak256::from_string(std::string_view s) { + // The pre-trait from_variant path (vector) rejected odd-length hex; + // keep that strictness so a trailing lone nibble is malformed input, not + // silently zero-extended. + FC_ASSERT(s.size() % 2 == 0, "keccak256 hex string length must be even, got {}", s.size()); + return keccak256(s); + } + } // namespace fc::crypto diff --git a/libraries/libfc/src/crypto/ripemd160.cpp b/libraries/libfc/src/crypto/ripemd160.cpp index 4216bf1406..daf99fa634 100644 --- a/libraries/libfc/src/crypto/ripemd160.cpp +++ b/libraries/libfc/src/crypto/ripemd160.cpp @@ -9,6 +9,7 @@ #include #include #include "_digest_common.hpp" +#include namespace fc { @@ -98,4 +99,12 @@ bool operator == ( const ripemd160& h1, const ripemd160& h2 ) { return memcmp( h1._hash, h2._hash, sizeof(h1._hash) ) == 0; } + ripemd160 ripemd160::from_string(std::string_view s) { + // The pre-trait from_variant path (vector) rejected odd-length hex; + // keep that strictness so a trailing lone nibble is malformed input, not + // silently zero-extended. + FC_ASSERT(s.size() % 2 == 0, "ripemd160 hex string length must be even, got {}", s.size()); + return ripemd160(s); + } + } // fc diff --git a/libraries/libfc/src/crypto/sha1.cpp b/libraries/libfc/src/crypto/sha1.cpp index 2dcab62e70..71f35ee88e 100644 --- a/libraries/libfc/src/crypto/sha1.cpp +++ b/libraries/libfc/src/crypto/sha1.cpp @@ -6,6 +6,7 @@ #include #include #include "_digest_common.hpp" +#include namespace fc { @@ -86,4 +87,12 @@ bool operator == ( const sha1& h1, const sha1& h2 ) { return memcmp( h1._hash, h2._hash, sizeof(h1._hash) ) == 0; } + sha1 sha1::from_string(std::string_view s) { + // The pre-trait from_variant path (vector) rejected odd-length hex; + // keep that strictness so a trailing lone nibble is malformed input, not + // silently zero-extended. + FC_ASSERT(s.size() % 2 == 0, "sha1 hex string length must be even, got {}", s.size()); + return sha1(s); + } + } // fc diff --git a/libraries/libfc/src/crypto/sha224.cpp b/libraries/libfc/src/crypto/sha224.cpp index 6c8991c030..2215caeb46 100644 --- a/libraries/libfc/src/crypto/sha224.cpp +++ b/libraries/libfc/src/crypto/sha224.cpp @@ -6,6 +6,7 @@ #include #include #include "_digest_common.hpp" +#include namespace fc { @@ -83,4 +84,12 @@ namespace fc { template<> unsigned int hmac::internal_block_size() const { return 64; } + sha224 sha224::from_string(std::string_view s) { + // The pre-trait from_variant path (vector) rejected odd-length hex; + // keep that strictness so a trailing lone nibble is malformed input, not + // silently zero-extended. + FC_ASSERT(s.size() % 2 == 0, "sha224 hex string length must be even, got {}", s.size()); + return sha224(s); + } + } diff --git a/libraries/libfc/src/crypto/sha256.cpp b/libraries/libfc/src/crypto/sha256.cpp index 1ff206dda0..62360aaada 100644 --- a/libraries/libfc/src/crypto/sha256.cpp +++ b/libraries/libfc/src/crypto/sha256.cpp @@ -205,4 +205,12 @@ namespace fc { template<> unsigned int hmac::internal_block_size() const { return 64; } + sha256 sha256::from_string(std::string_view s) { + // The pre-trait from_variant path (vector) rejected odd-length hex; + // keep that strictness so a trailing lone nibble is malformed input, not + // silently zero-extended. + FC_ASSERT(s.size() % 2 == 0, "sha256 hex string length must be even, got {}", s.size()); + return sha256(s); + } + } //end namespace fc diff --git a/libraries/libfc/src/crypto/sha3.cpp b/libraries/libfc/src/crypto/sha3.cpp index 8305bf9143..4462d9b7ca 100644 --- a/libraries/libfc/src/crypto/sha3.cpp +++ b/libraries/libfc/src/crypto/sha3.cpp @@ -283,4 +283,12 @@ bool operator==(const sha3 &h1, const sha3 &h2) h1._hash[3] == h2._hash[3]; } + sha3 sha3::from_string(std::string_view s) { + // The pre-trait from_variant path (vector) rejected odd-length hex; + // keep that strictness so a trailing lone nibble is malformed input, not + // silently zero-extended. + FC_ASSERT(s.size() % 2 == 0, "sha3 hex string length must be even, got {}", s.size()); + return sha3(s); + } + } // namespace fc diff --git a/libraries/libfc/src/crypto/sha512.cpp b/libraries/libfc/src/crypto/sha512.cpp index a7980dbb19..120dd1a3ad 100644 --- a/libraries/libfc/src/crypto/sha512.cpp +++ b/libraries/libfc/src/crypto/sha512.cpp @@ -6,6 +6,7 @@ #include #include #include "_digest_common.hpp" +#include namespace fc { @@ -90,4 +91,12 @@ namespace fc { template<> unsigned int hmac::internal_block_size() const { return 128; } + sha512 sha512::from_string(std::string_view s) { + // The pre-trait from_variant path (vector) rejected odd-length hex; + // keep that strictness so a trailing lone nibble is malformed input, not + // silently zero-extended. + FC_ASSERT(s.size() % 2 == 0, "sha512 hex string length must be even, got {}", s.size()); + return sha512(s); + } + } diff --git a/libraries/libfc/src/log/log_message.cpp b/libraries/libfc/src/log/log_message.cpp index d2d1bbecdd..f5502e5d66 100644 --- a/libraries/libfc/src/log/log_message.cpp +++ b/libraries/libfc/src/log/log_message.cpp @@ -114,11 +114,6 @@ namespace fc to_json_stream(v, w); } - void to_json_stream( const log_context& l, json_writer& w ) - { - to_json_stream(l.to_variant(), w); - } - void to_variant( log_level e, variant& v ) { switch( e ) diff --git a/libraries/libfc/src/variant.cpp b/libraries/libfc/src/variant.cpp index 5313ae31a5..622e06f7f7 100644 --- a/libraries/libfc/src/variant.cpp +++ b/libraries/libfc/src/variant.cpp @@ -1274,18 +1274,18 @@ std::string format_string( const std::string& frmt, const variant_object& args, } - void to_json_stream( const variant& v, json_writer& w ) { + void to_json_stream( const variant& v, json_writer& w, uint32_t max_depth ) { // Walk the variant tree token-by-token directly into json_writer instead of // building a JSON string via fc::json::to_string and splicing it back in. Mirrors // the compact (indent=0) form of fc::json::to_string for byte-identical output. + FC_ASSERT( max_depth > 0, "to_json_stream: variant nesting exceeds max depth" ); switch( v.get_type() ) { case variant::null_type: w.value_null(); return; case variant::int64_type: { int64_t i = v.as_int64(); - constexpr int64_t max_value(0xffffffff); - if( i > max_value || i < -max_value ) { + if( i > json_integer_quote_magnitude || i < -json_integer_quote_magnitude ) { // Quote out-of-range integers so 64-bit JS clients don't lose precision. w.value_string( v.as_string() ); } else { @@ -1295,7 +1295,7 @@ std::string format_string( const std::string& frmt, const variant_object& args, } case variant::uint64_type: { uint64_t i = v.as_uint64(); - if( i > 0xffffffff ) { + if( i > static_cast(json_integer_quote_magnitude) ) { w.value_string( v.as_string() ); } else { w.value_uint64( i ); @@ -1328,12 +1328,12 @@ std::string format_string( const std::string& frmt, const variant_object& args, case variant::array_type: { const variants& a = v.get_array(); w.begin_array(); - for( const auto& e : a ) to_json_stream( e, w ); + for( const auto& e : a ) to_json_stream( e, w, max_depth - 1 ); w.end_array(); return; } case variant::object_type: - to_json_stream( v.get_object(), w ); + to_json_stream( v.get_object(), w, max_depth - 1 ); return; default: FC_THROW_EXCEPTION( fc::invalid_arg_exception, "Unsupported variant type: {}", std::to_string( v.get_type() ) ); diff --git a/libraries/libfc/src/variant_object.cpp b/libraries/libfc/src/variant_object.cpp index 663f0cd7a8..483eac2eef 100644 --- a/libraries/libfc/src/variant_object.cpp +++ b/libraries/libfc/src/variant_object.cpp @@ -1,6 +1,5 @@ #include #include -#include #include namespace fc @@ -505,29 +504,34 @@ namespace fc vo = variant(var); } - void to_json_stream( const variant_object& vo, json_writer& w ) - { - // Walk the object's key/value pairs directly into json_writer. Mirrors the - // compact (indent=0) form of fc::json::to_string for byte-identical output; - // values recurse into to_json_stream(variant, w) which handles nested arrays / - // objects without going through fc::json::to_string at any level. - w.begin_object(); - for( const auto& kv : vo ) { - w.key( kv.key() ); - to_json_stream( kv.value(), w ); + namespace { + // Shared walker for both object flavors (they iterate the same entry shape). + // Walks the key/value pairs directly into json_writer, mirroring the compact + // (indent=0) form of fc::json::to_string for byte-identical output; values + // recurse into to_json_stream(variant, w) which handles nested arrays / objects + // without going through fc::json::to_string at any level. `max_depth` bounds + // the recursion in lock-step with the variant walker. + template + void object_to_json_stream( const Object& vo, json_writer& w, uint32_t max_depth ) + { + FC_ASSERT( max_depth > 0, "to_json_stream: variant_object nesting exceeds max depth" ); + w.begin_object(); + for( const auto& kv : vo ) { + w.key( kv.key() ); + to_json_stream( kv.value(), w, max_depth - 1 ); + } + w.end_object(); } - w.end_object(); } - void to_json_stream( const mutable_variant_object& vo, json_writer& w ) + void to_json_stream( const variant_object& vo, json_writer& w, uint32_t max_depth ) { - // mutable_variant_object iterates the same pair shape as variant_object. - w.begin_object(); - for( const auto& kv : vo ) { - w.key( kv.key() ); - to_json_stream( kv.value(), w ); - } - w.end_object(); + object_to_json_stream( vo, w, max_depth ); + } + + void to_json_stream( const mutable_variant_object& vo, json_writer& w, uint32_t max_depth ) + { + object_to_json_stream( vo, w, max_depth ); } void from_variant( const variant& var, mutable_variant_object& vo ) diff --git a/libraries/libfc/test/crypto/test_hash_functions.cpp b/libraries/libfc/test/crypto/test_hash_functions.cpp index c23467e286..7b21c2f779 100644 --- a/libraries/libfc/test/crypto/test_hash_functions.cpp +++ b/libraries/libfc/test/crypto/test_hash_functions.cpp @@ -46,6 +46,13 @@ namespace { const std::string expected_json = std::string{ "\"" } + std::string{ hex } + "\""; BOOST_CHECK_EQUAL(fc::to_json_string(h), expected_json); + + // Odd-length hex is malformed input: from_string (and therefore the trait-routed + // fc::from_variant) must reject it rather than zero-extend the trailing nibble. + const std::string odd{ hex.substr(0, hex.size() - 1) }; + BOOST_CHECK_THROW(Hash::from_string(odd), fc::assert_exception); + Hash h4; + BOOST_CHECK_THROW(fc::from_variant(fc::variant{ odd }, h4), fc::assert_exception); } } diff --git a/libraries/libfc/test/io/test_json_stream.cpp b/libraries/libfc/test/io/test_json_stream.cpp index 6a460881a0..bd3a3b955e 100644 --- a/libraries/libfc/test/io/test_json_stream.cpp +++ b/libraries/libfc/test/io/test_json_stream.cpp @@ -157,13 +157,13 @@ BOOST_AUTO_TEST_CASE(value_double_rejects_non_finite) { std::string out; { fc::json_writer w(out); - BOOST_CHECK_THROW(w.value_double(nan), std::invalid_argument); + BOOST_CHECK_THROW(w.value_double(nan), fc::assert_exception); } BOOST_CHECK(out.empty()); // nothing written, frame state untouched { fc::json_writer w(out); - BOOST_CHECK_THROW(w.value_double(pos_inf), std::invalid_argument); - BOOST_CHECK_THROW(w.value_double(neg_inf), std::invalid_argument); + BOOST_CHECK_THROW(w.value_double(pos_inf), fc::assert_exception); + BOOST_CHECK_THROW(w.value_double(neg_inf), fc::assert_exception); } // Finite values: to_json_stream(double) emits the quoted fixed-precision form // matching the variant path's emission shape. @@ -324,6 +324,33 @@ BOOST_AUTO_TEST_CASE(fc_variant_object_fallback) { fc::json::to_string(as_var, fc::json::yield_function_t())); } +// The variant walker bounds recursion the way fc::json::to_string's yield does: a tree +// nested deeper than fc::json_stream_max_depth throws instead of overflowing the stack. +BOOST_AUTO_TEST_CASE(fc_variant_walker_depth_guard) { + // json_stream_max_depth - 1 array wraps around a leaf emit cleanly (the leaf itself + // consumes the final depth level). + fc::variant nested{ std::string("leaf") }; + for (uint32_t i = 0; i < fc::json_stream_max_depth - 1; ++i) { + fc::variants wrap; + wrap.emplace_back(std::move(nested)); + nested = fc::variant(std::move(wrap)); + } + std::string out; + { + fc::json_writer w(out); + fc::to_json_stream(nested, w); + BOOST_REQUIRE(w.balanced()); + } + + // One more level pushes the leaf past the cap. + fc::variants wrap; + wrap.emplace_back(std::move(nested)); + fc::variant deep{ std::move(wrap) }; + std::string out2; + fc::json_writer w2(out2); + BOOST_CHECK_THROW(fc::to_json_stream(deep, w2), fc::assert_exception); +} + // Composition test: a reflected struct that has fc::time_point and fc::sha256 fields. BOOST_AUTO_TEST_CASE(reflector_with_fc_types) { // The reflector-based path must find the to_json_stream overloads for fc::time_point diff --git a/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp b/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp index 11afc1385f..62b93bf207 100644 --- a/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp +++ b/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp @@ -134,9 +134,9 @@ namespace sysio { /// Parses each captured `abi` byte-string into an `abi_serializer` and returns /// the resolver. CPU-bound; safe to run on any thread (no chainbase access). /// On a per-account basis, an exception during `to_abi` / `abi_serializer` is - /// swallowed and the account's slot is left empty -- callers fall back to the - /// hex representation for actions with un-decodable ABI, matching the prior - /// `make_resolver(throw_on_yield::no)` behaviour. + /// logged at debug level and the account's slot is left empty -- callers fall + /// back to the hex representation for actions with un-decodable ABI, matching + /// the prior `make_resolver(throw_on_yield::no)` behaviour. inline abi_resolver build_resolver_from_captured_abis(const captured_abis& cache, const fc::microseconds& max_time) { chain::abi_serializer_cache_t serializers; @@ -155,7 +155,14 @@ namespace sysio { } else { serializers.emplace(name, std::nullopt); } + } catch (const fc::exception& e) { + dlog("failed to build abi_serializer for {}: {}", name, e.to_detail_string()); + serializers.emplace(name, std::nullopt); + } catch (const std::exception& e) { + dlog("failed to build abi_serializer for {}: {}", name, e.what()); + serializers.emplace(name, std::nullopt); } catch (...) { + dlog("failed to build abi_serializer for {}: unknown exception", name); serializers.emplace(name, std::nullopt); } } diff --git a/plugins/chain_plugin/src/chain_plugin.cpp b/plugins/chain_plugin/src/chain_plugin.cpp index 008fecdb48..7101ced393 100644 --- a/plugins/chain_plugin/src/chain_plugin.cpp +++ b/plugins/chain_plugin/src/chain_plugin.cpp @@ -3257,7 +3257,7 @@ void read_only::convert_block_stream( const chain::signed_block_ptr& block, abi_ sink.begin_object(); chain::impl::abi_to_variant::emit_signed_block_body(sink, *block, resolver, ctx); sink.key("id"); - sink.template emit(block_id); + sink.emit(block_id); sink.key("block_num"); sink.value_uint64(block->block_num()); sink.key("ref_block_prefix"); diff --git a/plugins/http_plugin/include/sysio/http_plugin/bind_stream.hpp b/plugins/http_plugin/include/sysio/http_plugin/bind_stream.hpp index b74f81865d..7d0c7ec87e 100644 --- a/plugins/http_plugin/include/sysio/http_plugin/bind_stream.hpp +++ b/plugins/http_plugin/include/sysio/http_plugin/bind_stream.hpp @@ -285,13 +285,18 @@ api_entry_stream bind_stream(http_plugin& http, Handle handle, try { // ---- Parse params (if the method takes any) ---- - [[maybe_unused]] params_t params; - if constexpr (has_params) { - params = parse_params_rt(body, pt); - } else { - // No params expected -- still validate the body per `pt`. - (void)parse_params_rt(body, pt); - } + // Immediately-invoked initializer so `params` is constructed directly from the + // parse result (no default-construct-then-assign, and param structs need not be + // default-constructible). The no-params arm still validates the body per `pt` + // and yields the std::monostate placeholder. + [[maybe_unused]] params_t params = [&]() -> params_t { + if constexpr (has_params) { + return parse_params_rt(body, pt); + } else { + (void)parse_params_rt(body, pt); + return {}; + } + }(); // ---- Dispatch on Kind -------------------------------------- if constexpr (Kind == dispatch::sync) { diff --git a/plugins/trace_api_plugin/src/abi_data_handler.cpp b/plugins/trace_api_plugin/src/abi_data_handler.cpp index 13ac7d0631..f420edb39c 100644 --- a/plugins/trace_api_plugin/src/abi_data_handler.cpp +++ b/plugins/trace_api_plugin/src/abi_data_handler.cpp @@ -12,7 +12,7 @@ namespace sysio::trace_api { auto make_abi_yield() { return [](size_t recursion_depth) { SYS_ASSERT( recursion_depth < chain::abi_serializer::max_recursion_depth, chain::abi_recursion_depth_exception, - "exceeded max_recursion_depth {} ", chain::abi_serializer::max_recursion_depth ); + "exceeded max_recursion_depth {}", chain::abi_serializer::max_recursion_depth ); }; } } From 493c9a412a7aa94b8992328dc0cb170a3e34e736 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Tue, 14 Jul 2026 08:41:21 -0500 Subject: [PATCH 61/71] fix macOS build and a flaky streaming get_table_rows test The streaming http path used std::move_only_function directly for its url_response_stream_callback / stream_emitter aliases, which AppleClang's libc++ does not provide -- the macOS CI build failed to compile http_plugin. chain::types.hpp already carried a narrow void(T&&)-only polyfill for next_function; promote it to a general fc::move_only_function (fc/move_only_function.hpp) that selects std::move_only_function when the standard library has it and the type-erased fallback otherwise, and route the http_plugin aliases, post_http_thread_pool, and the trace_api benchmark through it. Direct unit coverage for the polyfill implementation so it is exercised on every platform, not only where the alias resolves to it. stream_decode_failure_falls_back_to_hex_valid_json constructed a std::string from .begin()/.end() of two SEPARATE get_table_test_abi() temporaries -- the iterators point into different std::string objects, so end - begin is a garbage distance and basic_string throws length_error depending on heap layout. This is why the ubuntu/gcc/asan/ubsan jobs failed intermittently on that test while the production streaming code was fine. Read the abi once into a named string. verified: the gcc CI container image builds plugin_test clean and the fixed test passes 80/80 where it previously failed ~1 in 5 --- benchmark/trace_api_json.cpp | 2 +- libraries/chain/include/sysio/chain/types.hpp | 68 ++------------- .../libfc/include/fc/move_only_function.hpp | 82 ++++++++++++++++++ libraries/libfc/test/CMakeLists.txt | 1 + .../libfc/test/test_move_only_function.cpp | 83 +++++++++++++++++++ .../include/sysio/http_plugin/http_plugin.hpp | 7 +- plugins/http_plugin/src/http_plugin.cpp | 2 +- tests/test_get_table_rows_page.cpp | 5 +- 8 files changed, 180 insertions(+), 70 deletions(-) create mode 100644 libraries/libfc/include/fc/move_only_function.hpp create mode 100644 libraries/libfc/test/test_move_only_function.cpp diff --git a/benchmark/trace_api_json.cpp b/benchmark/trace_api_json.cpp index 068078d323..5ed9c1c2ba 100644 --- a/benchmark/trace_api_json.cpp +++ b/benchmark/trace_api_json.cpp @@ -192,7 +192,7 @@ void trace_api_json_benchmarking() { }); benchmarking("struct-pass (mof): " + label_prefix, [&]() { data_log_entry e = std::move(fresh_b[fresh_idx_b++ % pool_size]); - std::move_only_function body = + fc::move_only_function body = [e = std::move(e)](fc::json_writer& w) mutable { asm volatile("" : : "g"(&e) : "memory"); }; diff --git a/libraries/chain/include/sysio/chain/types.hpp b/libraries/chain/include/sysio/chain/types.hpp index 9d928115dc..7aefd30e18 100644 --- a/libraries/chain/include/sysio/chain/types.hpp +++ b/libraries/chain/include/sysio/chain/types.hpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -517,7 +518,7 @@ namespace sysio::chain { // The third option is a function which can be executed in a multithreaded context (likely on the // http_plugin thread pool) and which completes the API processing and returns the result T. // - // The user-provided callable is held inside a std::move_only_function so consume-on-call captures + // The user-provided callable is held inside an fc::move_only_function so consume-on-call captures // (e.g. [cb = std::move(cb)] mutable {...}) work correctly. The outer wrapper is copyable via a // shared_ptr indirection - required because the callback travels through paths that copy // (boost::signals2 slot dispatch, boost::multi_index value storage, lambdas captured by [=], @@ -541,68 +542,11 @@ namespace sysio::chain { using next_function_variant = std::variant()>>; namespace detail { - /** - * @brief Move-only callable storage for standard libraries without std::move_only_function. - * - * libc++ versions shipped with current AppleClang do not always provide the C++23 - * std::move_only_function API. This wrapper preserves the same move-only capture support - * needed by next_function without requiring that library feature. - * - * This is intentionally a narrow polyfill for next_function's current - * `void(T&&)` storage shape, not a general std::move_only_function - * replacement. If next_function starts storing another callable - * signature, extend this wrapper alongside that change. - */ + // Portable move-only callable storage; fc::move_only_function selects + // std::move_only_function where the standard library provides it and a + // general type-erased polyfill otherwise (AppleClang's libc++ lacks it). template - class move_only_function; - - template - class move_only_function { - public: - move_only_function() = default; - move_only_function(std::nullptr_t) noexcept {} - - template - requires (!std::is_same_v, move_only_function> && - std::is_invocable_r_v&, Arg&&>) - move_only_function(F&& f) - : _callable(std::make_unique>>(std::forward(f))) {} - - move_only_function(move_only_function&&) noexcept = default; - move_only_function& operator=(move_only_function&&) noexcept = default; - move_only_function(const move_only_function&) = delete; - move_only_function& operator=(const move_only_function&) = delete; - - void operator()(Arg&& arg) { _callable->invoke(std::forward(arg)); } - explicit operator bool() const noexcept { return static_cast(_callable); } - - private: - struct callable_base { - virtual ~callable_base() = default; - virtual void invoke(Arg&& arg) = 0; - }; - - template - struct callable final : callable_base { - template - explicit callable(Fn&& fn) - : f(std::forward(fn)) {} - - void invoke(Arg&& arg) override { f(std::forward(arg)); } - - F f; - }; - - std::unique_ptr _callable; - }; - - template - using next_move_only_function = -#if defined(__cpp_lib_move_only_function) && __cpp_lib_move_only_function >= 202110L - std::move_only_function; -#else - move_only_function; -#endif + using next_move_only_function = fc::move_only_function; } template diff --git a/libraries/libfc/include/fc/move_only_function.hpp b/libraries/libfc/include/fc/move_only_function.hpp new file mode 100644 index 0000000000..288d365a34 --- /dev/null +++ b/libraries/libfc/include/fc/move_only_function.hpp @@ -0,0 +1,82 @@ +#pragma once + +/** + * @file fc/move_only_function.hpp + * @brief `fc::move_only_function`: std::move_only_function where the standard + * library provides it, otherwise a type-erased polyfill. + * + * libc++ versions shipped with current AppleClang do not provide the C++23 + * std::move_only_function API, so portable code uses this alias instead of naming the + * std template directly. + * + * The polyfill supports the subset the tree relies on: construction from any callable + * invocable with the signature (including move-only captures), move-only semantics, + * `explicit operator bool`, and a non-const `operator()`. cv/ref/noexcept-qualified + * signatures are not implemented. + */ + +#include + +#include +#include +#include +#include + +namespace fc { + +namespace detail { + + template + class move_only_function_impl; + + template + class move_only_function_impl { + public: + move_only_function_impl() = default; + move_only_function_impl(std::nullptr_t) noexcept {} + + template + requires (!std::is_same_v, move_only_function_impl> && + std::is_invocable_r_v&, Args...>) + move_only_function_impl(F&& f) + : _callable(std::make_unique>>(std::forward(f))) {} + + move_only_function_impl(move_only_function_impl&&) noexcept = default; + move_only_function_impl& operator=(move_only_function_impl&&) noexcept = default; + move_only_function_impl(const move_only_function_impl&) = delete; + move_only_function_impl& operator=(const move_only_function_impl&) = delete; + + R operator()(Args... args) { return _callable->invoke(std::forward(args)...); } + explicit operator bool() const noexcept { return static_cast(_callable); } + + private: + struct callable_base { + virtual ~callable_base() = default; + virtual R invoke(Args... args) = 0; + }; + + template + struct callable final : callable_base { + template + explicit callable(Fn&& fn) + : f(std::forward(fn)) {} + + R invoke(Args... args) override { return f(std::forward(args)...); } + + F f; + }; + + std::unique_ptr _callable; + }; + +} // namespace detail + +template +using move_only_function = +#if defined(__cpp_lib_move_only_function) && __cpp_lib_move_only_function >= 202110L + std::move_only_function; +#else + detail::move_only_function_impl; +#endif + +} // namespace fc diff --git a/libraries/libfc/test/CMakeLists.txt b/libraries/libfc/test/CMakeLists.txt index 605dff83ee..5679ffecd1 100644 --- a/libraries/libfc/test/CMakeLists.txt +++ b/libraries/libfc/test/CMakeLists.txt @@ -12,6 +12,7 @@ add_executable( test_fc io/test_cfile.cpp io/test_json.cpp io/test_json_stream.cpp + test_move_only_function.cpp io/test_secure_file.cpp log/test_json_formatter.cpp io/test_json_variant.cpp diff --git a/libraries/libfc/test/test_move_only_function.cpp b/libraries/libfc/test/test_move_only_function.cpp new file mode 100644 index 0000000000..f0c4ae18d3 --- /dev/null +++ b/libraries/libfc/test/test_move_only_function.cpp @@ -0,0 +1,83 @@ +#include + +#include + +#include +#include + +// On libstdc++/newer libc++ the fc::move_only_function alias resolves to +// std::move_only_function, so the polyfill branch never compiles there. These cases +// instantiate fc::detail::move_only_function_impl directly so the polyfill gets +// compile + behavior coverage on every platform, not just AppleClang. +BOOST_AUTO_TEST_SUITE(move_only_function_tests) + +BOOST_AUTO_TEST_CASE(polyfill_void_nullary) { + int calls = 0; + fc::detail::move_only_function_impl f{ [&] { ++calls; } }; + BOOST_REQUIRE(static_cast(f)); + f(); + BOOST_CHECK_EQUAL(calls, 1); + + fc::detail::move_only_function_impl empty; + BOOST_CHECK(!static_cast(empty)); + fc::detail::move_only_function_impl null_constructed{ nullptr }; + BOOST_CHECK(!static_cast(null_constructed)); +} + +BOOST_AUTO_TEST_CASE(polyfill_returns_value_and_forwards_args) { + fc::detail::move_only_function_impl add{ [](int a, int b) { return a + b; } }; + BOOST_CHECK_EQUAL(add(2, 40), 42); + + // Move-only by-value argument must forward through the type-erasure boundary. + fc::detail::move_only_function_impl)> take{ + [](std::unique_ptr p) { return *p; } }; + BOOST_CHECK_EQUAL(take(std::make_unique(7)), 7); + + // Rvalue-reference parameter -- the shape chain::next_function stores. + fc::detail::move_only_function_impl&&)> sink{ + [](std::unique_ptr&& p) { auto consumed = std::move(p); BOOST_CHECK_EQUAL(*consumed, 9); } }; + sink(std::make_unique(9)); +} + +BOOST_AUTO_TEST_CASE(polyfill_move_only_capture_and_move_semantics) { + auto payload = std::make_unique(11); + fc::detail::move_only_function_impl f{ [p = std::move(payload)] { return *p; } }; + + // Move construction and move assignment transfer the callable. + fc::detail::move_only_function_impl g{ std::move(f) }; + BOOST_CHECK(!static_cast(f)); + BOOST_REQUIRE(static_cast(g)); + BOOST_CHECK_EQUAL(g(), 11); + + fc::detail::move_only_function_impl h; + h = std::move(g); + BOOST_REQUIRE(static_cast(h)); + BOOST_CHECK_EQUAL(h(), 11); +} + +BOOST_AUTO_TEST_CASE(polyfill_nested_shape) { + // Mirrors url_response_stream_callback: a move-only function taking a move-only + // function by value. + using inner_t = fc::detail::move_only_function_impl; + using outer_t = fc::detail::move_only_function_impl; + + int observed = 0; + outer_t outer{ [&](int code, inner_t emit) { + int v = code; + emit(v); + observed = v; + } }; + outer(200, inner_t{ [](int& v) { v += 22; } }); + BOOST_CHECK_EQUAL(observed, 222); +} + +BOOST_AUTO_TEST_CASE(alias_basic_usage) { + // Whichever implementation the alias selects on this platform, the common subset + // (construct from move-only capture, bool conversion, single invocation) must work. + auto payload = std::make_unique(5); + fc::move_only_function f{ [p = std::move(payload)] { return *p; } }; + BOOST_REQUIRE(static_cast(f)); + BOOST_CHECK_EQUAL(f(), 5); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/plugins/http_plugin/include/sysio/http_plugin/http_plugin.hpp b/plugins/http_plugin/include/sysio/http_plugin/http_plugin.hpp index fd96d2edb5..1db3975064 100644 --- a/plugins/http_plugin/include/sysio/http_plugin/http_plugin.hpp +++ b/plugins/http_plugin/include/sysio/http_plugin/http_plugin.hpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -56,8 +57,8 @@ namespace sysio { * have to satisfy CopyConstructible; pass the typed struct directly without * round-tripping through fc::variant. */ - using stream_emitter = std::move_only_function; - using url_response_stream_callback = std::move_only_function; + using stream_emitter = fc::move_only_function; + using url_response_stream_callback = fc::move_only_function; using url_handler_stream = std::function; /** @@ -199,7 +200,7 @@ namespace sysio { // Accepts a move-only callable so streaming-cb macros can capture // url_response_stream_callback (move_only_function) into the closure. // std::function values implicitly convert and so are still accepted. - void post_http_thread_pool(std::move_only_function f); + void post_http_thread_pool(fc::move_only_function f); bool is_on_loopback(api_category category) const; diff --git a/plugins/http_plugin/src/http_plugin.cpp b/plugins/http_plugin/src/http_plugin.cpp index 74da7bcd8e..ddcdfb8937 100644 --- a/plugins/http_plugin/src/http_plugin.cpp +++ b/plugins/http_plugin/src/http_plugin.cpp @@ -698,7 +698,7 @@ namespace sysio { SYS_ASSERT(p.second, chain::plugin_config_exception, "http url {} is not unique", p.first->first); } - void http_plugin::post_http_thread_pool(std::move_only_function f) { + void http_plugin::post_http_thread_pool(fc::move_only_function f) { if( f ) boost::asio::post( my->plugin_state->thread_pool.get_executor(), std::move(f) ); } diff --git a/tests/test_get_table_rows_page.cpp b/tests/test_get_table_rows_page.cpp index 1aad01e834..c97472f5c3 100644 --- a/tests/test_get_table_rows_page.cpp +++ b/tests/test_get_table_rows_page.cpp @@ -395,9 +395,8 @@ BOOST_FIXTURE_TEST_CASE(stream_decode_failure_falls_back_to_hex_valid_json, vali deploy_contract(*this); populate_numobjs(*this, 3); - abi_def abi = fc::json::from_string( - std::string(test_contracts::get_table_test_abi().begin(), - test_contracts::get_table_test_abi().end())).as(); + const std::string abi_json = test_contracts::get_table_test_abi(); + abi_def abi = fc::json::from_string(abi_json).as(); for (auto& st : abi.structs) { if (st.name == "numobj") { st.fields.push_back({"extra_field", "uint64"}); From 166c76741e893efb9c3aabd810a8bf38577f675d Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Tue, 14 Jul 2026 09:11:07 -0500 Subject: [PATCH 62/71] add platform-specific integer to_json_stream overloads for macOS int64_t/uint64_t are aliases of different concrete types per platform: long on 64-bit Linux, long long on macOS. The streaming overload set covered the fixed-width aliases but not the remaining standard 64-bit integer type, so a reflected size_t field (get_unapplied_transactions_result::unapplied_size) resolved to unsigned long on macOS -- a type with no to_json_stream overload -- and fell through to the reflector primary, failing the macOS build. Mirror the platform guard fc::variant already uses (fc/variant.hpp): on Apple add a size_t overload, on non-MSVC non-Apple add long long / unsigned long long, each delegating to the int64_t/uint64_t emitter. The variant path was already covered this way, which is why only the streaming path broke. Only the macOS build exercises the size_t overload; the reflected_struct_platform_ints test streams size_t/long long/unsigned long long and asserts parity with the variant path so whichever type is the platform's uncovered one is checked on every build. --- .../libfc/include/fc/reflect/json_stream.hpp | 10 ++++++++ libraries/libfc/test/io/test_json_stream.cpp | 24 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/libraries/libfc/include/fc/reflect/json_stream.hpp b/libraries/libfc/include/fc/reflect/json_stream.hpp index 7b3c228df2..c2325870c5 100644 --- a/libraries/libfc/include/fc/reflect/json_stream.hpp +++ b/libraries/libfc/include/fc/reflect/json_stream.hpp @@ -67,6 +67,16 @@ inline void to_json_stream(uint64_t n, json_writer& w) { w.value_uint64(n); } } +// int64_t / uint64_t are aliases of different concrete types per platform (long on 64-bit Linux, +// long long on macOS), so one of the standard 64-bit integer types is left without an overload +// above. Mirror to_variant's platform guard (fc/variant.hpp) so reflected fields of that type +// (e.g. a size_t member on macOS) stream instead of falling through to the reflector primary. +#ifdef __APPLE__ +inline void to_json_stream(size_t n, json_writer& w) { to_json_stream(static_cast(n), w); } +#elif !defined(_MSC_VER) +inline void to_json_stream(long long int n, json_writer& w) { to_json_stream(static_cast(n), w); } +inline void to_json_stream(unsigned long long int n, json_writer& w) { to_json_stream(static_cast(n), w); } +#endif // double / float emit as a JSON-quoted fixed-precision string to match fc::variant's emission shape // (variant.cpp s_fc_to_string + json.cpp double_type case). Reflector-driven struct fields (e.g. // get_producers_result.total_producer_vote_weight) depend on this shape; emitting a bare JSON number diff --git a/libraries/libfc/test/io/test_json_stream.cpp b/libraries/libfc/test/io/test_json_stream.cpp index bd3a3b955e..cd1a31cda2 100644 --- a/libraries/libfc/test/io/test_json_stream.cpp +++ b/libraries/libfc/test/io/test_json_stream.cpp @@ -64,6 +64,18 @@ struct block_like_t { std::string producer; }; +// int64_t/uint64_t alias different concrete types per platform (long on 64-bit Linux, long long +// on macOS), so exactly one of {size_t, long long, unsigned long long} is NOT one of the aliased +// fixed-width overloads on any given platform. A struct carrying all three exercises the +// platform-specific integer to_json_stream overload wherever the build runs (size_t on macOS; +// long long / unsigned long long on Linux) -- pins the fix for the get_unapplied_transactions_result +// size_t field that failed the macOS build. +struct platform_ints_t { + size_t sz = 0; + long long ll = 0; + unsigned long long ull = 0; +}; + } // namespace FC_REFLECT(point_t, (x)(y)(label)) @@ -72,6 +84,7 @@ FC_REFLECT_ENUM(color_t, (red)(green)(blue)) FC_REFLECT(with_optional_t, (id)(note)) FC_REFLECT(with_map_t, (counts)) FC_REFLECT(block_like_t, (timestamp)(digest)(producer)) +FC_REFLECT(platform_ints_t, (sz)(ll)(ull)) BOOST_AUTO_TEST_SUITE(json_stream_test) @@ -194,6 +207,17 @@ BOOST_AUTO_TEST_CASE(reflected_struct) { BOOST_CHECK_EQUAL(fc::to_json_string(p), "{\"x\":3,\"y\":-4,\"label\":\"origin\"}"); } +// The size_t / long long / unsigned long long fields must stream identically to the variant +// path, including the >2^32 quoting rule. Whichever of the three is the platform's "extra" +// 64-bit integer type hits the guarded overload in reflect/json_stream.hpp. +BOOST_AUTO_TEST_CASE(reflected_struct_platform_ints) { + platform_ints_t v{ .sz = 5, .ll = -8'000'000'000LL, .ull = 20'000'000'000ULL }; + fc::variant as_var; + fc::to_variant(v, as_var); + BOOST_CHECK_EQUAL(fc::to_json_string(v), + fc::json::to_string(as_var, fc::json::yield_function_t())); +} + BOOST_AUTO_TEST_CASE(nested_struct) { nested_t n{ .name = "n", From f051c025f596b40bc924a1f4c9971c3583f8ac22 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Tue, 14 Jul 2026 10:43:12 -0500 Subject: [PATCH 63/71] fix value_double_locale_independent for macOS libc++ The assertion body only runs when a comma-radix locale (de_DE/fr_FR) is present, which the CI Linux builder image lacks, so it had never executed there -- macOS, where those locales ship, ran it for the first time and it failed on two stale assumptions: - it expected the shortest-form spelling (to_json_string(1.5) == "1.5"), but doubles are emitted as a quoted fixed-precision string (digits10+2) to match fc::variant; and the min()/max() round-trip cannot survive that fixed format (denormal-range values render as all-zero fractional digits). - it parsed back with std::from_chars, which AppleClang's libc++ does not implement. Rewrite it to assert the property the test actually exists for: under a comma locale the emitted number carries no ',' radix and is byte-identical to the fc::variant path (the reference emitter). Neither production path is affected -- from_chars was test-only and to_chars(fixed) works on macOS. Verified by running the assertion body under a locally generated de_DE.UTF-8 (localedef + LOCPATH, no root): 12/12 assertions pass with the comma locale active. --- libraries/libfc/test/io/test_json_stream.cpp | 27 ++++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/libraries/libfc/test/io/test_json_stream.cpp b/libraries/libfc/test/io/test_json_stream.cpp index cd1a31cda2..d1a945d72b 100644 --- a/libraries/libfc/test/io/test_json_stream.cpp +++ b/libraries/libfc/test/io/test_json_stream.cpp @@ -143,21 +143,20 @@ BOOST_AUTO_TEST_CASE(value_double_locale_independent) { BOOST_TEST_MESSAGE("no comma-radix locale available; skipping locale-independence assertion"); return; } - BOOST_CHECK_EQUAL(fc::to_json_string(1.5), "1.5"); - BOOST_CHECK_EQUAL(fc::to_json_string(-2.25), "-2.25"); - // Round-trip: any finite double must parse back to its bit-exact value. - // strtod_l with the C locale avoids re-introducing the comma-radix issue. - auto round_trip = [](double d) { - const std::string s = fc::to_json_string(d); - double parsed = 0.0; - auto [_, ec] = std::from_chars(s.data(), s.data() + s.size(), parsed); - BOOST_REQUIRE(ec == std::errc{}); - BOOST_CHECK_EQUAL(parsed, d); + + // Under a comma-radix locale the emitted number must (a) never contain a ',' radix + // and (b) match the variant path byte-for-byte -- doubles emit as a quoted + // fixed-precision string (digits10+2), so the check is against the reference + // emitter, not a hard-coded shortest-form spelling. std::from_chars is + // deliberately not used: AppleClang's libc++ does not implement it. + auto via_variant = [](double d) { + return fc::json::to_string(fc::variant(d), fc::json::yield_function_t()); }; - round_trip(0.1); - round_trip(1.0 / 3.0); - round_trip(std::numeric_limits::min()); - round_trip(std::numeric_limits::max()); + for (double d : {1.5, -2.25, 0.1, 1.0 / 3.0, 1234.5, -9876.0}) { + const std::string s = fc::to_json_string(d); + BOOST_CHECK(s.find(',') == std::string::npos); // no comma radix leaked from LC_NUMERIC + BOOST_CHECK_EQUAL(s, via_variant(d)); // period radix + exact parity with fc::variant + } } BOOST_AUTO_TEST_CASE(value_double_rejects_non_finite) { From 6f5acf33d62e7e7082e7f5bb70528fc3065505cd Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Mon, 20 Jul 2026 13:38:31 -0500 Subject: [PATCH 64/71] http: charge streaming response emission against bytes_in_flight incrementally - json_writer: optional growth guard consulted at token boundaries once per stride (64KiB default); re-arms across a guard throw so catch-and-continue serializers cannot absorb a budget abort - make_http_stream_response_handler: settle the response buffer into bytes_in_flight at guard checkpoints and abort over-budget emissions with the busy response instead of materializing them in full - get_finalizer_info: active/pending policy fields keep their key and emit explicit JSON null when absent (new fc::nullable), restoring the pre-typed-result schema - fc::microseconds and the abi stream_sink route 64-bit emission through the guarded to_json_stream overloads so the integer quoting threshold matches the variant path --- libraries/chain/abi_serializer.cpp | 11 +- libraries/libfc/include/fc/io/json_stream.hpp | 54 ++++++- libraries/libfc/include/fc/nullable.hpp | 49 +++++++ libraries/libfc/include/fc/variant.hpp | 29 ++++ libraries/libfc/src/time.cpp | 6 +- libraries/libfc/test/io/test_json_stream.cpp | 130 +++++++++++++++++ .../sysio/chain_plugin/chain_plugin.hpp | 10 +- plugins/chain_plugin/src/chain_plugin.cpp | 2 +- .../include/sysio/http_plugin/common.hpp | 55 ++++++-- plugins/http_plugin/test/unit_tests.cpp | 133 ++++++++++++++++++ tests/test_chain_plugin.cpp | 25 ++++ unittests/abi_tests.cpp | 67 +++++++++ 12 files changed, 555 insertions(+), 16 deletions(-) create mode 100644 libraries/libfc/include/fc/nullable.hpp diff --git a/libraries/chain/abi_serializer.cpp b/libraries/chain/abi_serializer.cpp index cd0b232057..d7c2e776c3 100644 --- a/libraries/chain/abi_serializer.cpp +++ b/libraries/chain/abi_serializer.cpp @@ -1432,8 +1432,15 @@ void stream_sink::end_array() { void stream_sink::key(std::string_view k) { w_.key(k); } void stream_sink::value_string(std::string_view s) { w_.value_string(s); on_value_emitted(); } -void stream_sink::value_int64(int64_t n) { w_.value_int64(n); on_value_emitted(); } -void stream_sink::value_uint64(uint64_t n) { w_.value_uint64(n); on_value_emitted(); } +// 64-bit integers route through the guarded fc::to_json_stream overloads (not the writer's +// raw value_* emitters) so magnitudes past json_integer_quote_magnitude emit quoted, exactly +// as variant_sink::value_int64's fc::variant(n) renders through fc::json::to_string. ABI +// built-in int64/uint64 fields already stream through the guarded generic in +// get_built_in_stream_unpacks; this pins the same quoting for the sink interface itself, so +// direct sink callers (the bytes size field today, any future caller) cannot diverge +// between the two sinks. +void stream_sink::value_int64(int64_t n) { fc::to_json_stream(n, w_); on_value_emitted(); } +void stream_sink::value_uint64(uint64_t n) { fc::to_json_stream(n, w_); on_value_emitted(); } void stream_sink::value_bool(bool b) { w_.value_bool(b); on_value_emitted(); } void stream_sink::value_null() { w_.value_null(); on_value_emitted(); } diff --git a/libraries/libfc/include/fc/io/json_stream.hpp b/libraries/libfc/include/fc/io/json_stream.hpp index 6fcf28684b..4bfd1c3948 100644 --- a/libraries/libfc/include/fc/io/json_stream.hpp +++ b/libraries/libfc/include/fc/io/json_stream.hpp @@ -36,6 +36,13 @@ inline constexpr int64_t json_integer_quote_magnitude = 0xffffffff; /// fc::json::to_string's yield would. inline constexpr uint32_t json_stream_max_depth = 200; +/// Default byte-growth stride between growth-guard invocations (see json_writer's guarded +/// constructor). Small enough that a caller enforcing a memory budget observes emission +/// growth promptly; large enough that the guard is a negligible fraction of emission cost +/// (one invocation per ~64 KiB appended). Responses smaller than one stride never invoke +/// the guard mid-emission. +inline constexpr size_t json_writer_default_guard_stride = 64 * 1024; + /** * Streaming JSON writer that emits tokens directly into an output std::string. * @@ -57,9 +64,32 @@ inline constexpr uint32_t json_stream_max_depth = 200; */ class json_writer { public: - explicit json_writer(std::string& out) + /// Growth-guard callback: invoked with the output buffer's current size (bytes) each time + /// the buffer has grown one stride past the previous invocation. The guard may throw to + /// abort emission -- eg an HTTP memory-budget enforcer rejecting a response mid-serialize. + /// The guard must not emit through the writer it guards. + using growth_guard_t = std::function; + + /// @param out buffer tokens are appended to; also the guard's measured quantity. + /// @param growth_guard optional guard consulted as the buffer grows (empty = never). + /// @param guard_stride minimum byte growth between guard invocations; a stride of 0 + /// re-checks on every token (test hook, not for production use). + /// + /// Guard-throw semantics: if the guard throws, the writer re-arms so the NEXT token + /// emitted re-invokes the guard immediately (regardless of stride). Mid-emission + /// rollback handlers (eg abi_serializer's catch-rewind-hex-fallback path) legitimately + /// swallow exceptions from nested serializers; re-arming guarantees a guard abort + /// re-raises out of every such handler instead of being silently absorbed -- unless the + /// guard's condition has cleared, in which case emission resumes legitimately. + explicit json_writer(std::string& out, growth_guard_t growth_guard = {}, + size_t guard_stride = json_writer_default_guard_stride) : out_(out) + , guard_(std::move(growth_guard)) + , guard_stride_(guard_stride) { + if (guard_) { + next_guard_check_ = out_.size() + guard_stride_; + } // Enough room for a reasonable response without reallocation; callers that know // the expected size can reserve() themselves before constructing the writer. // Skip if the caller already has slack so we don't risk a spec-permitted shrink. @@ -94,6 +124,7 @@ class json_writer { } void key(std::string_view k) { + guard_check(); assert(!stack_.empty() && stack_.back().ctx == context::object); assert(!awaiting_value_); // two key() calls in a row would produce "a":,"b": (invalid JSON) if (stack_.back().has_item) { @@ -252,7 +283,25 @@ class json_writer { bool has_item = false; // true once one element or key:value has been emitted in this frame }; + /// Consult the growth guard if the buffer has crossed the next stride boundary (or on + /// every token once a prior guard invocation threw -- see the constructor contract). + /// Called from the head of key() and value_prefix(), which between them front every + /// token-emitting entry point except end_object/end_array (1 byte each; the final + /// buffer size is the caller's to observe after emission completes). + void guard_check() { + if (out_.size() < next_guard_check_) { + return; + } + // Re-arm across a guard throw: 0 makes every subsequent token re-enter here while + // the guard keeps throwing, so a catch-and-continue serializer upstream cannot + // absorb the abort. Restored to a real stride only when the guard returns cleanly. + next_guard_check_ = 0; + guard_(out_.size()); + next_guard_check_ = out_.size() + guard_stride_; + } + void value_prefix() { + guard_check(); if (awaiting_value_) { // Value right after a key() - no separator, first-item bookkeeping already set. awaiting_value_ = false; @@ -288,6 +337,9 @@ class json_writer { std::string& out_; std::vector stack_; // small; vector is fine and avoids extra deps + growth_guard_t guard_; // empty unless the guarded constructor form was used + size_t guard_stride_ = json_writer_default_guard_stride; + size_t next_guard_check_ = std::numeric_limits::max(); // SIZE_MAX = no guard bool awaiting_value_ = false; }; diff --git a/libraries/libfc/include/fc/nullable.hpp b/libraries/libfc/include/fc/nullable.hpp new file mode 100644 index 0000000000..16a402849e --- /dev/null +++ b/libraries/libfc/include/fc/nullable.hpp @@ -0,0 +1,49 @@ +#pragma once + +#include +#include + +namespace fc { + +/** + * Optional-valued serialization wrapper whose JSON key is ALWAYS present. + * + * Reflected std::optional members are OMITTED from serialized output when disengaged -- + * both the to_variant reflector (to_variant_visitor::add) and the streaming reflector + * (to_json_stream_visitor::emit) skip them. Some endpoint schemas instead pin + * "key always present, value null when absent"; eg get_finalizer_info's active/pending + * policy fields, whose original fc::variant representation emitted explicit nulls that + * clients dereference before testing. Wrapping such a member as fc::nullable keeps + * that shape while the member stays strongly typed: + * + * - engaged -> serializes exactly like a plain T member + * - disengaged -> serializes as an explicit JSON null under the member's key + * + * from_variant mirrors the emission: a JSON null deserializes to the disengaged state. + * + * The serializer overloads (to_variant / from_variant / to_json_stream) live in + * fc/variant.hpp alongside the std::optional trio so the two families' shapes stay in + * lock-step; this header carries only the vocabulary type. + */ +template +struct nullable { + std::optional value; + + nullable() = default; + nullable(const T& v) : value(v) {} + nullable(T&& v) : value(std::move(v)) {} + nullable& operator=(const T& v) { value = v; return *this; } + nullable& operator=(T&& v) { value = std::move(v); return *this; } + + bool has_value() const { return value.has_value(); } + explicit operator bool() const { return value.has_value(); } + const T& operator*() const { return *value; } + T& operator*() { return *value; } + const T* operator->() const { return &*value; } + T* operator->() { return &*value; } + void reset() { value.reset(); } + + friend bool operator==(const nullable& a, const nullable& b) { return a.value == b.value; } +}; + +} // namespace fc diff --git a/libraries/libfc/include/fc/variant.hpp b/libraries/libfc/include/fc/variant.hpp index 3c26c45486..852920f6fe 100644 --- a/libraries/libfc/include/fc/variant.hpp +++ b/libraries/libfc/include/fc/variant.hpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -507,6 +508,34 @@ namespace fc } } + /** @ingroup Serializable + * fc::nullable trio -- like std::optional except a disengaged value serializes + * as an explicit JSON null under its key instead of being omitted by the reflector + * visitors (see fc/nullable.hpp). Kept adjacent to the std::optional overloads so + * the two families' engaged-value shapes cannot drift. + */ + template + void to_variant( const nullable& n, variant& vo ) + { + if( n.value ) to_variant( *n.value, vo ); + else vo = variant(); + } + + template + void from_variant( const variant& var, nullable& n ) + { + if( var.is_null() ) { n.value.reset(); return; } + n.value = T(); + from_variant( var, *n.value ); + } + + template + void to_json_stream( const nullable& n, json_writer& w ) + { + if( n.value ) to_json_stream( *n.value, w ); + else w.value_null(); + } + template void to_json_stream( const std::unordered_set& var, json_writer& w ) { diff --git a/libraries/libfc/src/time.cpp b/libraries/libfc/src/time.cpp index c08f38ef73..3f46383ee3 100644 --- a/libraries/libfc/src/time.cpp +++ b/libraries/libfc/src/time.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -174,7 +175,10 @@ namespace fc { } void to_json_stream( const microseconds& us, json_writer& w ) { - w.value_int64( us.count() ); + // Delegate to the guarded int64_t overload (fc/reflect/json_stream.hpp) so counts with + // magnitude past json_integer_quote_magnitude emit as quoted strings, exactly as the + // to_variant -> fc::json::to_string path renders the int64 count. + to_json_stream( us.count(), w ); } void from_variant( const variant& input_variant, microseconds& output_microseconds ) { diff --git a/libraries/libfc/test/io/test_json_stream.cpp b/libraries/libfc/test/io/test_json_stream.cpp index d1a945d72b..bdc1a0bf17 100644 --- a/libraries/libfc/test/io/test_json_stream.cpp +++ b/libraries/libfc/test/io/test_json_stream.cpp @@ -54,6 +54,11 @@ struct with_optional_t { std::optional note; }; +struct with_nullable_t { + int32_t id = 0; + fc::nullable maybe_point; +}; + struct with_map_t { std::map counts; }; @@ -82,6 +87,7 @@ FC_REFLECT(point_t, (x)(y)(label)) FC_REFLECT(nested_t, (name)(values)(maybe)(where)) FC_REFLECT_ENUM(color_t, (red)(green)(blue)) FC_REFLECT(with_optional_t, (id)(note)) +FC_REFLECT(with_nullable_t, (id)(maybe_point)) FC_REFLECT(with_map_t, (counts)) FC_REFLECT(block_like_t, (timestamp)(digest)(producer)) FC_REFLECT(platform_ints_t, (sz)(ll)(ull)) @@ -248,6 +254,40 @@ BOOST_AUTO_TEST_CASE(enum_value) { BOOST_CHECK_EQUAL(fc::to_json_string(color_t::green), "\"green\""); } +BOOST_AUTO_TEST_CASE(nullable_field_emits_explicit_null) { + // fc::nullable pins "key always present": a disengaged value serializes as an explicit + // JSON null under its key on BOTH paths, where a reflected std::optional omits the key + // entirely (see optional_field_omitted below). Endpoint schemas like get_finalizer_info + // depend on this shape. + auto via_variant = [](const with_nullable_t& v) { + fc::variant var; + fc::to_variant(v, var); + return fc::json::to_string(var, fc::json::yield_function_t()); + }; + + const with_nullable_t absent{.id = 1, .maybe_point = {}}; + BOOST_CHECK_EQUAL(fc::to_json_string(absent), R"({"id":1,"maybe_point":null})"); + BOOST_CHECK_EQUAL(fc::to_json_string(absent), via_variant(absent)); + + const with_nullable_t present{.id = 2, .maybe_point = point_t{.x = 3, .y = 4, .label = "p"}}; + BOOST_CHECK_EQUAL(fc::to_json_string(present), R"({"id":2,"maybe_point":{"x":3,"y":4,"label":"p"}})"); + BOOST_CHECK_EQUAL(fc::to_json_string(present), via_variant(present)); + + // from_variant mirrors the emission: null resets an engaged value, an object engages it. + fc::variant var; + fc::to_variant(absent, var); + with_nullable_t back{.id = 9, .maybe_point = point_t{}}; + fc::from_variant(var, back); + BOOST_CHECK_EQUAL(back.id, 1); + BOOST_CHECK(!back.maybe_point.has_value()); + + fc::to_variant(present, var); + fc::from_variant(var, back); + BOOST_REQUIRE(back.maybe_point.has_value()); + BOOST_CHECK_EQUAL(back.maybe_point->x, 3); + BOOST_CHECK_EQUAL(back.maybe_point->label, "p"); +} + BOOST_AUTO_TEST_CASE(optional_field_omitted) { // Unset optional field on a reflected struct is omitted entirely. with_optional_t o{.id = 1, .note = std::nullopt}; @@ -296,6 +336,25 @@ BOOST_AUTO_TEST_CASE(fc_microseconds) { BOOST_CHECK_EQUAL(fc::to_json_string(fc::microseconds{0}), "0"); BOOST_CHECK_EQUAL(fc::to_json_string(fc::microseconds{1'234'567}), "1234567"); BOOST_CHECK_EQUAL(fc::to_json_string(fc::microseconds{-5}), "-5"); + + // Counts with magnitude past json_integer_quote_magnitude must emit quoted, exactly as + // the variant path renders the int64 count through fc::json::to_string. Boundary on + // both sides of +-0xffffffff (decimal literals: hex would be unsigned and break negation). + BOOST_CHECK_EQUAL(fc::to_json_string(fc::microseconds{4294967295}), "4294967295"); + BOOST_CHECK_EQUAL(fc::to_json_string(fc::microseconds{4294967296}), "\"4294967296\""); + BOOST_CHECK_EQUAL(fc::to_json_string(fc::microseconds{-4294967295}), "-4294967295"); + BOOST_CHECK_EQUAL(fc::to_json_string(fc::microseconds{-4294967296}), "\"-4294967296\""); + + auto via_variant = [](const fc::microseconds& us) { + fc::variant v; + fc::to_variant(us, v); + return fc::json::to_string(v, fc::json::yield_function_t()); + }; + for (int64_t count : {int64_t{0}, int64_t{4294967295}, int64_t{4294967296}, + int64_t{-4294967295}, int64_t{-4294967296}, + std::numeric_limits::min(), std::numeric_limits::max()}) { + BOOST_CHECK_EQUAL(fc::to_json_string(fc::microseconds{count}), via_variant(fc::microseconds{count})); + } } BOOST_AUTO_TEST_CASE(fc_time_point_roundtrip_vs_variant) { @@ -554,4 +613,75 @@ BOOST_AUTO_TEST_CASE(set_raw_splices_preformatted_fragment) { BOOST_CHECK_EQUAL(out, R"({"name":"alice","payload":{"a":1,"b":"two"},"done":true})"); } +// -- growth guard ------------------------------------------------------------------------- + +namespace { +/// Foreign (non-fc, non-std) exception type: models the http layer's budget abort +/// (sysio::detail::stream_response_budget_exceeded), which must fly through serializer +/// catch(fc::exception&) handlers untouched. +struct test_budget_abort {}; +} // namespace + +BOOST_AUTO_TEST_CASE(growth_guard_fires_each_stride) { + // The guard is consulted at token boundaries once the buffer has grown one stride past + // the previous invocation: observed sizes are strictly increasing by at least a stride. + std::string out; + std::vector observed; + { + fc::json_writer w(out, [&](size_t sz) { observed.push_back(sz); }, 16); + w.begin_array(); + for (int i = 0; i < 100; ++i) w.value_string("0123456789"); + w.end_array(); + } + BOOST_REQUIRE(!observed.empty()); + for (size_t i = 1; i < observed.size(); ++i) { + BOOST_CHECK_GE(observed[i], observed[i - 1] + 16); + } + + // Emissions smaller than one stride never invoke the guard: zero overhead for the + // typical small response. + std::string small; + size_t calls = 0; + { + fc::json_writer w(small, [&](size_t) { ++calls; }, 1u << 20); + w.value_string("tiny"); + } + BOOST_CHECK_EQUAL(calls, 0u); +} + +BOOST_AUTO_TEST_CASE(growth_guard_throw_aborts_emission) { + // A guard throw propagates out of the token call and stops the buffer growing: the + // emission loop below would append ~13KB unguarded, but aborts within a token or two + // of the guard's threshold. + std::string out; + fc::json_writer w(out, [&](size_t sz) { if (sz > 64) throw test_budget_abort{}; }, 8); + w.begin_array(); + auto emit_many = [&] { for (int i = 0; i < 1000; ++i) w.value_string("xxxxxxxxxx"); }; + BOOST_CHECK_THROW(emit_many(), test_budget_abort); + BOOST_CHECK_LT(out.size(), 200u); +} + +BOOST_AUTO_TEST_CASE(growth_guard_rearms_after_throw) { + // A catch-and-continue serializer (eg the abi_serializer hex-fallback path) absorbing + // the guard's throw must NOT disable the guard: the very next token re-invokes it + // regardless of stride, so the abort re-raises until it unwinds out of the emitter. + // Once the guard passes again (budget freed), emission resumes normally. + std::string out; + bool reject = true; + size_t calls = 0; + fc::json_writer w(out, [&](size_t) { ++calls; if (reject) throw test_budget_abort{}; }, 4); + w.begin_array(); + w.value_string("aaaaaaaaaa"); // grows past the stride; guard fires on the NEXT token + BOOST_CHECK_THROW(w.value_string("bbb"), test_budget_abort); // stride crossed -> guard throws, nothing appended + BOOST_CHECK_EQUAL(calls, 1u); + BOOST_CHECK_THROW(w.value_string("ccc"), test_budget_abort); // absorbed upstream? re-fires immediately + BOOST_CHECK_EQUAL(calls, 2u); + reject = false; // budget freed: same retry now passes + w.value_string("ddd"); + w.end_array(); + BOOST_CHECK_EQUAL(calls, 3u); + BOOST_CHECK(w.balanced()); + BOOST_CHECK_EQUAL(out, R"(["aaaaaaaaaa","ddd"])"); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp b/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp index 62b93bf207..8cf41a747c 100644 --- a/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp +++ b/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp @@ -24,6 +24,7 @@ #include #include +#include #include #include @@ -668,8 +669,13 @@ class read_only : public api_base { }; struct get_finalizer_info_result { - std::optional active_finalizer_policy; // current active policy - std::optional pending_finalizer_policy; // current pending policy. Empty if not existing + // fc::nullable (not std::optional) pins the endpoint schema: both keys are always + // present and serialize as an explicit JSON null when the policy does not exist, + // matching the shape the earlier fc::variant-typed fields emitted. A reflected + // std::optional would omit the key entirely when disengaged, breaking clients that + // access the key before testing it for null. + fc::nullable active_finalizer_policy; // current active policy. null if not existing + fc::nullable pending_finalizer_policy; // current pending policy. null if not existing // Last tracked vote information for each of the finalizers in // active_finalizer_policy and pending_finalizer_policy. diff --git a/plugins/chain_plugin/src/chain_plugin.cpp b/plugins/chain_plugin/src/chain_plugin.cpp index 7101ced393..bb0a1a4182 100644 --- a/plugins/chain_plugin/src/chain_plugin.cpp +++ b/plugins/chain_plugin/src/chain_plugin.cpp @@ -3060,7 +3060,7 @@ read_only::get_finalizer_info_result read_only::get_finalizer_info( const read_o // Populate a particular finalizer policy auto add_policy_to_result = [&](const finalizer_policy_ptr& from_policy, - std::optional& to_policy) { + fc::nullable& to_policy) { if (from_policy) { to_policy = *from_policy; for (const auto& f: from_policy->finalizers) { diff --git a/plugins/http_plugin/include/sysio/http_plugin/common.hpp b/plugins/http_plugin/include/sysio/http_plugin/common.hpp index 066ea93b66..819f0052bb 100644 --- a/plugins/http_plugin/include/sysio/http_plugin/common.hpp +++ b/plugins/http_plugin/include/sysio/http_plugin/common.hpp @@ -137,6 +137,21 @@ static size_t in_flight_sizeof(const std::optional& o) { return 0; } +/** +* Thrown by the streaming-response growth guard when the incrementally-charged response +* buffer pushes bytes_in_flight past the configured budget (--http-max-bytes-in-flight-mb). +* +* Deliberately NOT derived from fc::exception or std::exception: mid-emission rollback +* handlers (eg abi_serializer's catch-rewind-hex-fallback path) catch and absorb serializer +* exceptions to keep emitting, and absorbing a budget abort would defeat the cap. A foreign +* type dodges their typed catches, and json_writer re-fires the guard on the next token +* after a throw so even a catch(...) absorber re-raises this until it unwinds out of the +* emitter (see json_writer's guarded-constructor contract). +*/ +struct stream_response_budget_exceeded { + std::string error; ///< busy-response text produced by verify_max_bytes_in_flight +}; + }// namespace detail // key -> priority, url_handler @@ -249,11 +264,15 @@ inline auto make_http_response_handler(http_plugin_state& plugin_state, detail:: * the api thread (read_only / read_write queue) never pays the per-field * allocation cost the variant tree built. * -* Backpressure note: bytes_in_flight tracking happens after the emitter runs (we -* don't know the body size until the buffer is full), so this path differs from -* the variant path's pre-post estimate. Net effect is "verify_max_bytes_in_flight -* checks the actual produced body size before send_response," which is the same -* end-state check the variant path performs. +* Backpressure note: the response buffer is charged against bytes_in_flight +* incrementally WHILE the emitter runs -- json_writer's growth guard settles the +* buffer's size into the budget every json_writer_default_guard_stride bytes and +* throws stream_response_budget_exceeded (mapped to the busy response) once +* verify_max_bytes_in_flight rejects. Concurrent large streaming responses are +* therefore visible to --http-max-bytes-in-flight-mb during serialization, and +* over-budget emissions abort early instead of materializing in full; the variant +* path achieves the analogous mid-flight visibility by pre-charging its +* in_flight_sizeof estimate of the response variant tree. */ inline auto make_http_stream_response_handler(http_plugin_state& plugin_state, detail::abstract_conn_ptr session_ptr) { return url_response_stream_callback{ @@ -272,19 +291,37 @@ inline auto make_http_stream_response_handler(http_plugin_state& plugin_state, d // I/O onto the http thread pool. boost::asio::dispatch(plugin_state.thread_pool.get_executor(), [&plugin_state, session_ptr{std::move(session_ptr)}, code, emitter{std::move(emitter)}]() mutable { + // `charged` mirrors what this response has contributed to the shared + // bytes_in_flight budget; settle_to trues the contribution up against the + // buffer's actual size at guard checkpoints, after emission completes, and + // (via the scoped_exit) on every exit path -- including rewinds that SHRANK + // the buffer below an earlier checkpoint, hence the signed reconciliation. + size_t charged = 0; + auto settle_to = [&plugin_state, &charged](size_t actual) { + if (actual >= charged) + plugin_state.bytes_in_flight += actual - charged; + else + plugin_state.bytes_in_flight -= charged - actual; + charged = actual; + }; + auto on_exit = fc::make_scoped_exit([&settle_to]() { settle_to(0); }); try { std::string body; { - fc::json_writer w(body); + fc::json_writer w(body, [&settle_to, &session_ptr](size_t buffer_size) { + settle_to(buffer_size); + if (auto error_str = session_ptr->verify_max_bytes_in_flight(0); !error_str.empty()) + throw detail::stream_response_budget_exceeded{std::move(error_str)}; + }); emitter(w); } - plugin_state.bytes_in_flight += body.size(); - auto on_exit = fc::make_scoped_exit([&, sz=body.size()]() { plugin_state.bytes_in_flight -= sz; }); - + settle_to(body.size()); if (auto error_str = session_ptr->verify_max_bytes_in_flight(0); !error_str.empty()) session_ptr->send_busy_response(std::move(error_str)); else session_ptr->send_response(std::move(body), code); + } catch (detail::stream_response_budget_exceeded& e) { + session_ptr->send_busy_response(std::move(e.error)); } catch (...) { session_ptr->handle_exception(); } diff --git a/plugins/http_plugin/test/unit_tests.cpp b/plugins/http_plugin/test/unit_tests.cpp index 284cdc0577..6cb9768cef 100644 --- a/plugins/http_plugin/test/unit_tests.cpp +++ b/plugins/http_plugin/test/unit_tests.cpp @@ -47,6 +47,8 @@ constexpr uint32_t request_body_bytes_in_flight_index = 5; constexpr uint32_t category_uw_index = 6; constexpr uint32_t stream_status_codes_index = 7; constexpr uint32_t stream_request_body_index = 8; +constexpr uint32_t stream_response_bytes_index = 9; +constexpr uint32_t stream_over_budget_index = 10; // Keeps unsharded IPv6 probe coverage on the historical port 9999. constexpr uint32_t ipv6_probe_index = 2; @@ -1058,5 +1060,136 @@ BOOST_FIXTURE_TEST_CASE(request_body_bytes_in_flight_stream, http_plugin_test_fi wait_for_no_bytes_in_flight(); } +// Streaming responses must be charged against bytes_in_flight WHILE the emitter runs, not only +// after the body is fully materialized: json_writer's growth guard settles the buffer into the +// budget every stride. The emitter below samples the budget mid-emission; without incremental +// accounting the sampled value reads 0 (the response was invisible to the budget until complete). +BOOST_FIXTURE_TEST_CASE(stream_response_bytes_in_flight, http_plugin_test_fixture) { + const std::string endpoint = test_http_endpoint("127.0.0.1", stream_response_bytes_index); + const std::string server_address = "--http-server-address=" + endpoint; + const std::string port = test_http_port(stream_response_bytes_index); + + http_plugin* http_plugin = init({"--plugin=sysio::http_plugin", + server_address.c_str(), + "--http-max-bytes-in-flight-mb=64"}); + BOOST_REQUIRE(http_plugin); + + // 32 x 64KiB chunks ~= 2 MiB emitted; sample the budget halfway through emission. + constexpr size_t chunk_size = 64 * 1024; + constexpr size_t chunk_count = 32; + std::atomic observed_bytes_in_flight{0}; + http_plugin->add_api_stream({{std::string("/2megabyte_stream"), api_category::node, + [&](string&&, string&&, url_response_stream_callback&& cb) { + cb(200, [&](fc::json_writer& w) { + w.begin_array(); + for (size_t i = 0; i < chunk_count; ++i) { + w.value_string(std::string(chunk_size, 'x')); + if (i == chunk_count / 2) + observed_bytes_in_flight.store(http_plugin->bytes_in_flight()); + } + w.end_array(); + }); + }}}, appbase::exec_queue::read_write); + + boost::asio::io_context ctx; + boost::asio::ip::tcp::resolver resolver(ctx); + + auto get_response = [&]() { + boost::asio::ip::tcp::socket s(ctx, boost::asio::ip::tcp::v4()); + boost::asio::connect(s, resolver.resolve("127.0.0.1", port)); + boost::beast::http::request req(boost::beast::http::verb::get, "/2megabyte_stream", 11); + req.set(http::field::host, endpoint); + boost::beast::http::write(s, req); + + boost::beast::http::response resp; + boost::beast::flat_buffer buffer; + boost::beast::http::read(s, buffer, resp); + return resp; + }; + + auto resp = get_response(); + BOOST_REQUIRE(resp.result() == boost::beast::http::status::ok); + BOOST_CHECK_GT(resp.body().size(), chunk_size * chunk_count); + // Regression assertion: 0 without the growth guard's incremental settle. At the halfway + // sample ~1 MiB has been emitted; charging lags by at most one stride, so half that is a + // safely conservative floor. + BOOST_CHECK_GE(observed_bytes_in_flight.load(), chunk_size * chunk_count / 4); + + uint16_t max = std::numeric_limits::max(); + while (http_plugin->requests_in_flight() > 0 && --max) + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + BOOST_CHECK(max > 0); + BOOST_CHECK_EQUAL(http_plugin->bytes_in_flight(), 0u); +} + +// Over-budget streaming responses must abort DURING emission (503 busy) instead of +// materializing in full and only then being rejected: the growth guard throws once +// verify_max_bytes_in_flight rejects the incrementally-charged buffer. The chunk counter +// proves early abort; the follow-up request proves the budget released and the plugin +// still serves. +BOOST_FIXTURE_TEST_CASE(stream_response_over_budget_aborts_early, http_plugin_test_fixture) { + const std::string endpoint = test_http_endpoint("127.0.0.1", stream_over_budget_index); + const std::string server_address = "--http-server-address=" + endpoint; + const std::string port = test_http_port(stream_over_budget_index); + + http_plugin* http_plugin = init({"--plugin=sysio::http_plugin", + server_address.c_str(), + "--http-max-bytes-in-flight-mb=1"}); + BOOST_REQUIRE(http_plugin); + + // Attempts 32 MiB against a 1 MiB budget; the guard must stop it within a stride or two + // past the budget. + constexpr size_t chunk_size = 64 * 1024; + constexpr size_t chunk_count = 512; + std::atomic chunks_emitted{0}; + http_plugin->add_api_stream({{std::string("/32megabyte_stream"), api_category::node, + [&](string&&, string&&, url_response_stream_callback&& cb) { + cb(200, [&](fc::json_writer& w) { + w.begin_array(); + for (size_t i = 0; i < chunk_count; ++i) { + w.value_string(std::string(chunk_size, 'x')); + chunks_emitted.fetch_add(1); + } + w.end_array(); + }); + }, + }, + {std::string("/small_stream"), api_category::node, + [&](string&&, string&&, url_response_stream_callback&& cb) { + cb(200, [](fc::json_writer& w) { w.value_string("ok"); }); + }}}, appbase::exec_queue::read_write); + + boost::asio::io_context ctx; + boost::asio::ip::tcp::resolver resolver(ctx); + + auto get_status = [&](const char* path) { + boost::asio::ip::tcp::socket s(ctx, boost::asio::ip::tcp::v4()); + boost::asio::connect(s, resolver.resolve("127.0.0.1", port)); + boost::beast::http::request req(boost::beast::http::verb::get, path, 11); + req.set(http::field::host, endpoint); + boost::beast::http::write(s, req); + + boost::beast::http::response resp; + boost::beast::flat_buffer buffer; + boost::beast::http::read(s, buffer, resp); + return resp.result(); + }; + + BOOST_CHECK(get_status("/32megabyte_stream") == boost::beast::http::status::service_unavailable); + // Early abort: the guard threw within a couple of strides past the 1 MiB budget, far + // short of the 512 chunks the emitter would produce unguarded. + BOOST_CHECK_LT(chunks_emitted.load(), 64u); + BOOST_CHECK_GT(chunks_emitted.load(), 0u); + + uint16_t max = std::numeric_limits::max(); + while (http_plugin->requests_in_flight() > 0 && --max) + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + BOOST_CHECK(max > 0); + BOOST_CHECK_EQUAL(http_plugin->bytes_in_flight(), 0u); + + // The budget drained: a small streaming response is admitted and served normally. + BOOST_CHECK(get_status("/small_stream") == boost::beast::http::status::ok); +} + //A warning for future tests: destruction of http_plugin_test_fixture sometimes does not destroy http_plugin's listeners. Tests // added in the future should avoid reusing ports of other tests in http_plugin_unit_tests. diff --git a/tests/test_chain_plugin.cpp b/tests/test_chain_plugin.cpp index caf211b929..29fab3b666 100644 --- a/tests/test_chain_plugin.cpp +++ b/tests/test_chain_plugin.cpp @@ -197,4 +197,29 @@ BOOST_FIXTURE_TEST_CASE(get_block_streaming_vs_variant_byte_identical, chain_plu BOOST_CHECK_EQUAL(variant_path, stream_path); } FC_LOG_AND_RETHROW() } +// get_finalizer_info absent-policy schema: both policy keys are ALWAYS present and emit an +// explicit JSON null when the policy does not exist (fc::nullable fields). A fresh chain +// has an active policy (Savanna activates at genesis) and no pending policy, so this pins +// the disengaged shape end-to-end plus byte parity between the variant and streaming paths. +// A reflected std::optional would omit the pending key entirely -- the regression this +// guards against. +BOOST_FIXTURE_TEST_CASE(get_finalizer_info_absent_policy_emits_null, chain_plugin_tester) { try { + produce_blocks(2); + + std::optional _tracked_votes; + read_only ro(*control, {}, {}, _tracked_votes, + fc::microseconds::maximum(), fc::microseconds::maximum(), {}); + + const read_only::get_finalizer_info_result result = ro.get_finalizer_info({}, fc::time_point::maximum()); + BOOST_REQUIRE(result.active_finalizer_policy.has_value()); + BOOST_REQUIRE(!result.pending_finalizer_policy.has_value()); + + const std::string variant_path = fc::json::to_string(fc::variant(result), fc::time_point::maximum()); + const std::string stream_path = fc::to_json_string(result); + + BOOST_CHECK_EQUAL(variant_path, stream_path); + BOOST_CHECK(stream_path.find("\"pending_finalizer_policy\":null") != std::string::npos); + BOOST_CHECK(stream_path.find("\"active_finalizer_policy\":{") != std::string::npos); +} FC_LOG_AND_RETHROW() } + BOOST_AUTO_TEST_SUITE_END() diff --git a/unittests/abi_tests.cpp b/unittests/abi_tests.cpp index 3d6efbd3a5..2f25767a2b 100644 --- a/unittests/abi_tests.cpp +++ b/unittests/abi_tests.cpp @@ -752,6 +752,73 @@ BOOST_AUTO_TEST_CASE(uint_types) } FC_LOG_AND_RETHROW() } +// int64/uint64 quoting boundaries through the ABI paths: values with magnitude past +// fc::json_integer_quote_magnitude (0xffffffff) emit as quoted JSON strings, values at the +// boundary stay bare -- and binary_to_json_stream must agree byte-for-byte with +// binary_to_variant + json::to_string (verify_byte_round_trip_conversion runs +// verify_stream_matches_variant on every case). +BOOST_AUTO_TEST_CASE(large_int_quoting_boundaries) +{ try { + + const char* test_abi = R"=====( + { + "version": "sysio::abi/1.0", + "types": [], + "structs": [{ + "name": "int_boundaries", + "base": "", + "fields": [{ + "name": "i64_bare_max", + "type": "int64" + },{ + "name": "i64_quoted", + "type": "int64" + },{ + "name": "i64_neg_bare", + "type": "int64" + },{ + "name": "i64_neg_quoted", + "type": "int64" + },{ + "name": "u64_bare", + "type": "uint64" + },{ + "name": "u64_quoted", + "type": "uint64" + }] + }], + "actions": [], + "tables": [], + "ricardian_clauses": [] + } + )====="; + + abi_serializer abis(fc::json::from_string(test_abi).as(), yield_fn()); + + const char* test_data = R"=====( + { + "i64_bare_max" : 4294967295, + "i64_quoted" : "4294967296", + "i64_neg_bare" : -4294967295, + "i64_neg_quoted" : "-4294967296", + "u64_bare" : 4294967295, + "u64_quoted" : "18446744073709551615" + } + )====="; + + auto var2 = verify_byte_round_trip_conversion(abis, "int_boundaries", fc::json::from_string(test_data)); + + // Pin the emitted shapes: at the quote magnitude stays bare, one past it emits quoted. + const std::string json = fc::json::to_string(var2, get_deadline()); + BOOST_CHECK(json.find("\"i64_bare_max\":4294967295") != std::string::npos); + BOOST_CHECK(json.find("\"i64_quoted\":\"4294967296\"") != std::string::npos); + BOOST_CHECK(json.find("\"i64_neg_bare\":-4294967295") != std::string::npos); + BOOST_CHECK(json.find("\"i64_neg_quoted\":\"-4294967296\"") != std::string::npos); + BOOST_CHECK(json.find("\"u64_bare\":4294967295") != std::string::npos); + BOOST_CHECK(json.find("\"u64_quoted\":\"18446744073709551615\"") != std::string::npos); + +} FC_LOG_AND_RETHROW() } + // Shared fixture for `general` and the streaming-parity test. Top-level fields // cover every built-in plus inherited base structs, large-int quoting boundaries, From b53438bb44f3e08842328e785fba54b57d9ef2d9 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Mon, 20 Jul 2026 14:32:19 -0500 Subject: [PATCH 65/71] fc: enforce the response budget within single JSON tokens - value_string/key route the escape loop's periodic yield into the growth guard, measuring the output buffer so escape expansion counts - value_hex/raw_value emit in 64KiB chunks with a guard check between chunks when guarded; unguarded writers keep the single tight loop - one oversized token (eg a large action console string, hex blob, or spliced fragment) can no longer materialize past the budget behind a single token-boundary check --- libraries/libfc/include/fc/io/json_stream.hpp | 61 ++++++++++++++++--- libraries/libfc/test/io/test_json_stream.cpp | 43 +++++++++++++ plugins/http_plugin/test/unit_tests.cpp | 51 ++++++++++++++++ 3 files changed, 148 insertions(+), 7 deletions(-) diff --git a/libraries/libfc/include/fc/io/json_stream.hpp b/libraries/libfc/include/fc/io/json_stream.hpp index 4bfd1c3948..6368af4345 100644 --- a/libraries/libfc/include/fc/io/json_stream.hpp +++ b/libraries/libfc/include/fc/io/json_stream.hpp @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -43,6 +44,12 @@ inline constexpr uint32_t json_stream_max_depth = 200; /// the guard mid-emission. inline constexpr size_t json_writer_default_guard_stride = 64 * 1024; +/// Output-byte chunk size for guarded emission of single large tokens (value_hex / +/// raw_value). A guarded writer breaks the append loop at this granularity and consults +/// the guard between chunks, so one oversized blob cannot materialize past the budget +/// behind a single token-boundary check. Unguarded writers keep the single tight loop. +inline constexpr size_t json_writer_guarded_chunk_bytes = 64 * 1024; + /** * Streaming JSON writer that emits tokens directly into an output std::string. * @@ -81,6 +88,15 @@ class json_writer { /// swallow exceptions from nested serializers; re-arming guarantees a guard abort /// re-raises out of every such handler instead of being silently absorbed -- unless the /// guard's condition has cleared, in which case emission resumes legitimately. + /// + /// Intra-token checks: guarded writers also consult the guard WHILE a single large + /// token streams -- every escape_string_yield_check_count input chars of a string value + /// or key (measuring the output buffer, so escape expansion counts) and every + /// json_writer_guarded_chunk_bytes appended by value_hex / raw_value -- so one + /// oversized token cannot materialize past the budget behind a single token-boundary + /// check. A guard throw mid-token leaves a partial token in the buffer; the writer's + /// re-arm plus the caller's checkpoint()/rewind() (or wholesale abandonment of the + /// buffer, as the http handler does) keep that safe. explicit json_writer(std::string& out, growth_guard_t growth_guard = {}, size_t guard_stride = json_writer_default_guard_stride) : out_(out) @@ -89,6 +105,10 @@ class json_writer { { if (guard_) { next_guard_check_ = out_.size() + guard_stride_; + // Consulted by escape_string every escape_string_yield_check_count input chars; + // routes the escape loop's periodic yield into the same stride-gated guard check + // the token boundaries use. + escape_yield_ = [this](size_t) { guard_check(); }; } // Enough room for a reasonable response without reallocation; callers that know // the expected size can reserve() themselves before constructing the writer. @@ -135,7 +155,7 @@ class json_writer { out_.push_back('"'); // key strings must also be escaped (eg field names containing special chars are rare, // but correctness matters on untrusted input echoed into error messages). - fc::escape_string(k, out_, json_yield_function_t(), true); + fc::escape_string(k, out_, escape_yield_, true); out_.push_back('"'); out_.push_back(':'); // A key mandates a value follows; suppress the comma from value_prefix for that value. @@ -145,7 +165,9 @@ class json_writer { void value_string(std::string_view s) { value_prefix(); out_.push_back('"'); - fc::escape_string(s, out_, json_yield_function_t(), true); + // escape_yield_ runs the growth guard periodically inside the escape loop, so a + // single oversized string value is budget-checked while it streams. + fc::escape_string(s, out_, escape_yield_, true); out_.push_back('"'); } @@ -195,9 +217,20 @@ class json_writer { out_.push_back('"'); static constexpr char digits[] = "0123456789abcdef"; const auto* p = reinterpret_cast(data); - for (size_t i = 0; i < size; ++i) { - out_.push_back(digits[p[i] >> 4]); - out_.push_back(digits[p[i] & 0x0f]); + // Guarded writers emit in bounded chunks and consult the guard between them, so a + // single large blob is budget-checked while it streams; unguarded writers keep the + // one tight loop (chunk_end = size on the first pass). + constexpr size_t src_chunk = json_writer_guarded_chunk_bytes / 2; // 2 hex chars per source byte + size_t i = 0; + while (i < size) { + const size_t chunk_end = guard_ ? std::min(size, i + src_chunk) : size; + for (; i < chunk_end; ++i) { + out_.push_back(digits[p[i] >> 4]); + out_.push_back(digits[p[i] & 0x0f]); + } + if (guard_) { + guard_check(); + } } out_.push_back('"'); } @@ -207,7 +240,18 @@ class json_writer { /// without an additional parse/re-emit cycle. void raw_value(std::string_view raw) { value_prefix(); - out_.append(raw.data(), raw.size()); + if (!guard_) { + out_.append(raw.data(), raw.size()); + return; + } + // Guarded writers splice in bounded chunks and consult the guard between them, so a + // single large preformatted fragment is budget-checked while it streams. + for (size_t off = 0; off < raw.size();) { + const size_t n = std::min(raw.size() - off, json_writer_guarded_chunk_bytes); + out_.append(raw.data() + off, n); + off += n; + guard_check(); + } } /// Fused key + value emitter; the streaming-JSON counterpart to @@ -287,7 +331,9 @@ class json_writer { /// every token once a prior guard invocation threw -- see the constructor contract). /// Called from the head of key() and value_prefix(), which between them front every /// token-emitting entry point except end_object/end_array (1 byte each; the final - /// buffer size is the caller's to observe after emission completes). + /// buffer size is the caller's to observe after emission completes) -- and, for guarded + /// writers, WITHIN large tokens: from escape_yield_ inside the string-escape loop and + /// between chunks in value_hex / raw_value, so no single token outruns the budget. void guard_check() { if (out_.size() < next_guard_check_) { return; @@ -338,6 +384,7 @@ class json_writer { std::string& out_; std::vector stack_; // small; vector is fine and avoids extra deps growth_guard_t guard_; // empty unless the guarded constructor form was used + json_yield_function_t escape_yield_; // guarded: runs guard_check inside escape_string loops size_t guard_stride_ = json_writer_default_guard_stride; size_t next_guard_check_ = std::numeric_limits::max(); // SIZE_MAX = no guard bool awaiting_value_ = false; diff --git a/libraries/libfc/test/io/test_json_stream.cpp b/libraries/libfc/test/io/test_json_stream.cpp index bdc1a0bf17..0c00c0278b 100644 --- a/libraries/libfc/test/io/test_json_stream.cpp +++ b/libraries/libfc/test/io/test_json_stream.cpp @@ -661,6 +661,49 @@ BOOST_AUTO_TEST_CASE(growth_guard_throw_aborts_emission) { BOOST_CHECK_LT(out.size(), 200u); } +BOOST_AUTO_TEST_CASE(growth_guard_intra_token_abort) { + // A single oversized token must be budget-checked WHILE it streams: value_string (via + // the escape loop's periodic yield), value_hex, and raw_value all abort within a chunk + // of the guard's threshold instead of materializing the full token behind one + // token-boundary pre-check. + constexpr size_t threshold = 64 * 1024; + // Abort bound: threshold + one guarded-emission chunk + escape/stride slack -- far + // below the full token size. + constexpr size_t abort_bound = threshold + fc::json_writer_guarded_chunk_bytes + 8 * 1024; + const std::string big(1u << 20, 'x'); // 1 MiB payload; hex form is 2 MiB + + { + std::string out; + fc::json_writer w(out, [&](size_t sz) { if (sz > threshold) throw test_budget_abort{}; }, 1024); + BOOST_CHECK_THROW(w.value_string(big), test_budget_abort); + BOOST_CHECK_LT(out.size(), abort_bound); + } + { + std::string out; + fc::json_writer w(out, [&](size_t sz) { if (sz > threshold) throw test_budget_abort{}; }, 1024); + BOOST_CHECK_THROW(w.value_hex(big.data(), big.size()), test_budget_abort); + BOOST_CHECK_LT(out.size(), abort_bound); + } + { + std::string out; + fc::json_writer w(out, [&](size_t sz) { if (sz > threshold) throw test_budget_abort{}; }, 1024); + BOOST_CHECK_THROW(w.raw_value(big), test_budget_abort); + BOOST_CHECK_LT(out.size(), abort_bound); + } +} + +BOOST_AUTO_TEST_CASE(growth_guard_measures_escape_expansion) { + // 32 KiB of control characters escape to ~192 KiB of output (6 bytes per input byte). + // A guard threshold above the input size but below the expanded size must still fire, + // proving the guard measures emitted bytes, not input consumed. + const std::string ctrl(32 * 1024, '\x01'); + std::string out; + fc::json_writer w(out, [&](size_t sz) { if (sz > 100 * 1024) throw test_budget_abort{}; }, 1024); + BOOST_CHECK_THROW(w.value_string(ctrl), test_budget_abort); + BOOST_CHECK_GT(out.size(), 100 * 1024); // the expanded output crossed a threshold the raw input never reaches + BOOST_CHECK_LT(out.size(), 128 * 1024); +} + BOOST_AUTO_TEST_CASE(growth_guard_rearms_after_throw) { // A catch-and-continue serializer (eg the abi_serializer hex-fallback path) absorbing // the guard's throw must NOT disable the guard: the very next token re-invokes it diff --git a/plugins/http_plugin/test/unit_tests.cpp b/plugins/http_plugin/test/unit_tests.cpp index 6cb9768cef..f5e4fbfc58 100644 --- a/plugins/http_plugin/test/unit_tests.cpp +++ b/plugins/http_plugin/test/unit_tests.cpp @@ -49,6 +49,7 @@ constexpr uint32_t stream_status_codes_index = 7; constexpr uint32_t stream_request_body_index = 8; constexpr uint32_t stream_response_bytes_index = 9; constexpr uint32_t stream_over_budget_index = 10; +constexpr uint32_t stream_single_token_index = 11; // Keeps unsharded IPv6 probe coverage on the historical port 9999. constexpr uint32_t ipv6_probe_index = 2; @@ -1191,5 +1192,55 @@ BOOST_FIXTURE_TEST_CASE(stream_response_over_budget_aborts_early, http_plugin_te BOOST_CHECK(get_status("/small_stream") == boost::beast::http::status::ok); } +// One oversized token -- a single 32 MiB string value -- must abort mid-token against the +// budget: the growth guard fires inside the escape loop, so the emitter's post-token +// statement never runs. Distinguishes intra-token enforcement from token-boundary-only +// checks, which would materialize the full token first and only then reject (same 503, but +// token_completed would read true). +BOOST_FIXTURE_TEST_CASE(stream_response_single_token_over_budget, http_plugin_test_fixture) { + const std::string endpoint = test_http_endpoint("127.0.0.1", stream_single_token_index); + const std::string server_address = "--http-server-address=" + endpoint; + const std::string port = test_http_port(stream_single_token_index); + + http_plugin* http_plugin = init({"--plugin=sysio::http_plugin", + server_address.c_str(), + "--http-max-bytes-in-flight-mb=1"}); + BOOST_REQUIRE(http_plugin); + + std::atomic token_completed{false}; + http_plugin->add_api_stream({{std::string("/one_giant_token"), api_category::node, + [&](string&&, string&&, url_response_stream_callback&& cb) { + cb(200, [&](fc::json_writer& w) { + w.value_string(std::string(32u * 1024 * 1024, 'x')); + token_completed.store(true); + }); + }}}, appbase::exec_queue::read_write); + + boost::asio::io_context ctx; + boost::asio::ip::tcp::resolver resolver(ctx); + + auto get_status = [&](const char* path) { + boost::asio::ip::tcp::socket s(ctx, boost::asio::ip::tcp::v4()); + boost::asio::connect(s, resolver.resolve("127.0.0.1", port)); + boost::beast::http::request req(boost::beast::http::verb::get, path, 11); + req.set(http::field::host, endpoint); + boost::beast::http::write(s, req); + + boost::beast::http::response resp; + boost::beast::flat_buffer buffer; + boost::beast::http::read(s, buffer, resp); + return resp.result(); + }; + + BOOST_CHECK(get_status("/one_giant_token") == boost::beast::http::status::service_unavailable); + BOOST_CHECK(!token_completed.load()); + + uint16_t max = std::numeric_limits::max(); + while (http_plugin->requests_in_flight() > 0 && --max) + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + BOOST_CHECK(max > 0); + BOOST_CHECK_EQUAL(http_plugin->bytes_in_flight(), 0u); +} + //A warning for future tests: destruction of http_plugin_test_fixture sometimes does not destroy http_plugin's listeners. Tests // added in the future should avoid reusing ports of other tests in http_plugin_unit_tests. From 9d3b330a4b7973f1329463c69b8472b435510695 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Mon, 20 Jul 2026 14:52:58 -0500 Subject: [PATCH 66/71] fc: derive guarded emission chunking from the writer's stride value_hex/raw_value chunked at a fixed 64KiB regardless of the configured guard_stride, silently capping intra-token observation granularity for those token kinds. Derive the chunk from guard_stride_ instead -- matching value_string, whose escape-yield cadence is already stride-gated -- and drop the redundant json_writer_guarded_chunk_bytes constant. --- libraries/libfc/include/fc/io/json_stream.hpp | 35 +++++++++---------- libraries/libfc/test/io/test_json_stream.cpp | 7 ++-- 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/libraries/libfc/include/fc/io/json_stream.hpp b/libraries/libfc/include/fc/io/json_stream.hpp index 6368af4345..de8f883bbe 100644 --- a/libraries/libfc/include/fc/io/json_stream.hpp +++ b/libraries/libfc/include/fc/io/json_stream.hpp @@ -44,12 +44,6 @@ inline constexpr uint32_t json_stream_max_depth = 200; /// the guard mid-emission. inline constexpr size_t json_writer_default_guard_stride = 64 * 1024; -/// Output-byte chunk size for guarded emission of single large tokens (value_hex / -/// raw_value). A guarded writer breaks the append loop at this granularity and consults -/// the guard between chunks, so one oversized blob cannot materialize past the budget -/// behind a single token-boundary check. Unguarded writers keep the single tight loop. -inline constexpr size_t json_writer_guarded_chunk_bytes = 64 * 1024; - /** * Streaming JSON writer that emits tokens directly into an output std::string. * @@ -91,12 +85,12 @@ class json_writer { /// /// Intra-token checks: guarded writers also consult the guard WHILE a single large /// token streams -- every escape_string_yield_check_count input chars of a string value - /// or key (measuring the output buffer, so escape expansion counts) and every - /// json_writer_guarded_chunk_bytes appended by value_hex / raw_value -- so one - /// oversized token cannot materialize past the budget behind a single token-boundary - /// check. A guard throw mid-token leaves a partial token in the buffer; the writer's - /// re-arm plus the caller's checkpoint()/rewind() (or wholesale abandonment of the - /// buffer, as the http handler does) keep that safe. + /// or key (measuring the output buffer, so escape expansion counts) and every stride of + /// bytes appended by value_hex / raw_value (their append loops chunk at guard_stride) -- + /// so one oversized token cannot materialize past the budget behind a single + /// token-boundary check. A guard throw mid-token leaves a partial token in the buffer; + /// the writer's re-arm plus the caller's checkpoint()/rewind() (or wholesale abandonment + /// of the buffer, as the http handler does) keep that safe. explicit json_writer(std::string& out, growth_guard_t growth_guard = {}, size_t guard_stride = json_writer_default_guard_stride) : out_(out) @@ -217,10 +211,11 @@ class json_writer { out_.push_back('"'); static constexpr char digits[] = "0123456789abcdef"; const auto* p = reinterpret_cast(data); - // Guarded writers emit in bounded chunks and consult the guard between them, so a - // single large blob is budget-checked while it streams; unguarded writers keep the - // one tight loop (chunk_end = size on the first pass). - constexpr size_t src_chunk = json_writer_guarded_chunk_bytes / 2; // 2 hex chars per source byte + // Guarded writers emit in stride-sized chunks and consult the guard between them, so + // a single large blob is budget-checked at the writer's configured granularity while + // it streams; unguarded writers keep the one tight loop (chunk_end = size on the + // first pass). + const size_t src_chunk = std::max(1, guard_stride_ / 2); // 2 hex chars per source byte size_t i = 0; while (i < size) { const size_t chunk_end = guard_ ? std::min(size, i + src_chunk) : size; @@ -244,10 +239,12 @@ class json_writer { out_.append(raw.data(), raw.size()); return; } - // Guarded writers splice in bounded chunks and consult the guard between them, so a - // single large preformatted fragment is budget-checked while it streams. + // Guarded writers splice in stride-sized chunks and consult the guard between them, + // so a single large preformatted fragment is budget-checked at the writer's + // configured granularity while it streams. + const size_t chunk = std::max(1, guard_stride_); for (size_t off = 0; off < raw.size();) { - const size_t n = std::min(raw.size() - off, json_writer_guarded_chunk_bytes); + const size_t n = std::min(raw.size() - off, chunk); out_.append(raw.data() + off, n); off += n; guard_check(); diff --git a/libraries/libfc/test/io/test_json_stream.cpp b/libraries/libfc/test/io/test_json_stream.cpp index 0c00c0278b..251d445952 100644 --- a/libraries/libfc/test/io/test_json_stream.cpp +++ b/libraries/libfc/test/io/test_json_stream.cpp @@ -667,9 +667,10 @@ BOOST_AUTO_TEST_CASE(growth_guard_intra_token_abort) { // of the guard's threshold instead of materializing the full token behind one // token-boundary pre-check. constexpr size_t threshold = 64 * 1024; - // Abort bound: threshold + one guarded-emission chunk + escape/stride slack -- far - // below the full token size. - constexpr size_t abort_bound = threshold + fc::json_writer_guarded_chunk_bytes + 8 * 1024; + // Abort bound: threshold + one stride of guarded emission (hex/raw chunk at the + // writer's stride; the escape loop yields every 128 input chars) + slack -- far below + // the full token size. + constexpr size_t abort_bound = threshold + 8 * 1024; const std::string big(1u << 20, 'x'); // 1 MiB payload; hex form is 2 MiB { From db986345ff0e50e93ccc72f8902b7b77acf16d34 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Thu, 23 Jul 2026 12:12:00 -0500 Subject: [PATCH 67/71] fc: approve a string token's size with the guard before escape_string reserves The guarded json_writer let escape_string reserve the full input length up front, so a single over-budget string allocated its entire size (32 MiB under a 1 MiB cap) before any check could run; the growth guard bounded emission but never allocation. Escaped output is at least one byte per input byte, so value_string and key now consult the guard with the buffer's prospective size -- current size plus the token's input length -- before escape_string runs. An over-budget token aborts with zero allocation and zero buffer mutation, an approved token keeps the single full-size reserve, and escape expansion beyond the approval stays covered by the intra-loop checks. The guard contract documents that prospective sizes may be passed; the http settle_to reconciliation already handles them, charging the bytes before they are allocated. The allocation regression test drives a 32 MiB string and key at a 64 KiB threshold and asserts capacity stays at the writer's initial slack, with unguarded and approved-path controls pinning the legacy full reserve. --- libraries/libfc/include/fc/io/json_stream.hpp | 65 ++++++++++++---- libraries/libfc/test/io/test_json_stream.cpp | 78 +++++++++++++++---- 2 files changed, 112 insertions(+), 31 deletions(-) diff --git a/libraries/libfc/include/fc/io/json_stream.hpp b/libraries/libfc/include/fc/io/json_stream.hpp index de8f883bbe..8abe665e0a 100644 --- a/libraries/libfc/include/fc/io/json_stream.hpp +++ b/libraries/libfc/include/fc/io/json_stream.hpp @@ -65,10 +65,14 @@ inline constexpr size_t json_writer_default_guard_stride = 64 * 1024; */ class json_writer { public: - /// Growth-guard callback: invoked with the output buffer's current size (bytes) each time - /// the buffer has grown one stride past the previous invocation. The guard may throw to - /// abort emission -- eg an HTTP memory-budget enforcer rejecting a response mid-serialize. - /// The guard must not emit through the writer it guards. + /// Growth-guard callback: invoked with the output buffer's size (bytes) each time the + /// buffer has grown one stride past the previous invocation. Before a string value or + /// key is escaped into the buffer, the guard instead receives the PROSPECTIVE size -- + /// current size plus the token's input length, a lower bound on the token's peak -- so + /// it can reject (and a budget enforcer charge) the bytes before any capacity is + /// reserved for them; actual growth reconciles at the next invocation. The guard may + /// throw to abort emission -- eg an HTTP memory-budget enforcer rejecting a response + /// mid-serialize. The guard must not emit through the writer it guards. using growth_guard_t = std::function; /// @param out buffer tokens are appended to; also the guard's measured quantity. @@ -88,9 +92,12 @@ class json_writer { /// or key (measuring the output buffer, so escape expansion counts) and every stride of /// bytes appended by value_hex / raw_value (their append loops chunk at guard_stride) -- /// so one oversized token cannot materialize past the budget behind a single - /// token-boundary check. A guard throw mid-token leaves a partial token in the buffer; - /// the writer's re-arm plus the caller's checkpoint()/rewind() (or wholesale abandonment - /// of the buffer, as the http handler does) keep that safe. + /// token-boundary check. String values and keys additionally pre-check their known + /// input length with the guard BEFORE escape_string reserves capacity for it, so an + /// over-budget string aborts with zero allocation (see guard_check_pending). A guard + /// throw mid-token leaves a partial token in the buffer; the writer's re-arm plus the + /// caller's checkpoint()/rewind() (or wholesale abandonment of the buffer, as the http + /// handler does) keep that safe. explicit json_writer(std::string& out, growth_guard_t growth_guard = {}, size_t guard_stride = json_writer_default_guard_stride) : out_(out) @@ -138,7 +145,9 @@ class json_writer { } void key(std::string_view k) { - guard_check(); + // Subsumes the plain token-boundary guard_check() and additionally approves the + // key's input length before escape_string reserves capacity for it. + guard_check_pending(k.size()); assert(!stack_.empty() && stack_.back().ctx == context::object); assert(!awaiting_value_); // two key() calls in a row would produce "a":,"b": (invalid JSON) if (stack_.back().has_item) { @@ -157,10 +166,14 @@ class json_writer { } void value_string(std::string_view s) { + // Approve the string's input length with the guard BEFORE escape_string reserves + // capacity for it: an over-budget value aborts here with zero allocation. Runs + // ahead of value_prefix so a rejection mutates nothing (no dangling separator for + // absorb-and-continue callers); the escape loop's periodic escape_yield_ then + // covers expansion beyond the approval. + guard_check_pending(s.size()); value_prefix(); out_.push_back('"'); - // escape_yield_ runs the growth guard periodically inside the escape loop, so a - // single oversized string value is budget-checked while it streams. fc::escape_string(s, out_, escape_yield_, true); out_.push_back('"'); } @@ -326,11 +339,12 @@ class json_writer { /// Consult the growth guard if the buffer has crossed the next stride boundary (or on /// every token once a prior guard invocation threw -- see the constructor contract). - /// Called from the head of key() and value_prefix(), which between them front every - /// token-emitting entry point except end_object/end_array (1 byte each; the final - /// buffer size is the caller's to observe after emission completes) -- and, for guarded - /// writers, WITHIN large tokens: from escape_yield_ inside the string-escape loop and - /// between chunks in value_hex / raw_value, so no single token outruns the budget. + /// Called from value_prefix(), which -- together with the guard_check_pending calls + /// fronting key() and value_string() -- fronts every token-emitting entry point except + /// end_object/end_array (1 byte each; the final buffer size is the caller's to observe + /// after emission completes). For guarded writers it also runs WITHIN large tokens: + /// from escape_yield_ inside the string-escape loop and between chunks in value_hex / + /// raw_value, so no single token outruns the budget. void guard_check() { if (out_.size() < next_guard_check_) { return; @@ -343,6 +357,27 @@ class json_writer { next_guard_check_ = out_.size() + guard_stride_; } + /// Consult the growth guard with the buffer's PROSPECTIVE size before `pending` bytes + /// are appended. Escaped string output is at least one byte per input byte (pre + /// UTF-8-prune), so a string token's input length is a guaranteed lower bound on the + /// buffer growth it is about to cause; checking it first lets the guard reject -- and + /// a budget enforcer charge -- those bytes before escape_string reserves capacity for + /// them, so an over-budget string aborts with zero allocation instead of after a + /// full-size reserve. With pending == 0 this is exactly guard_check(), including the + /// re-arm-across-throw contract. On clean approval the next check is deferred until + /// the buffer grows a stride PAST the approved size: approved bytes do not re-fire the + /// guard, while escape expansion beyond them still does (via escape_yield_). A + /// prune-shrunk token that never reaches the approved size simply reconciles downward + /// at the guard's next invocation. + void guard_check_pending(size_t pending) { + if (out_.size() + pending < next_guard_check_) { + return; + } + next_guard_check_ = 0; + guard_(out_.size() + pending); + next_guard_check_ = out_.size() + pending + guard_stride_; + } + void value_prefix() { guard_check(); if (awaiting_value_) { diff --git a/libraries/libfc/test/io/test_json_stream.cpp b/libraries/libfc/test/io/test_json_stream.cpp index 251d445952..4babe3776c 100644 --- a/libraries/libfc/test/io/test_json_stream.cpp +++ b/libraries/libfc/test/io/test_json_stream.cpp @@ -662,14 +662,14 @@ BOOST_AUTO_TEST_CASE(growth_guard_throw_aborts_emission) { } BOOST_AUTO_TEST_CASE(growth_guard_intra_token_abort) { - // A single oversized token must be budget-checked WHILE it streams: value_string (via - // the escape loop's periodic yield), value_hex, and raw_value all abort within a chunk - // of the guard's threshold instead of materializing the full token behind one - // token-boundary pre-check. + // A single oversized token must not materialize behind one token-boundary check. + // value_string pre-checks its known input length with the guard and aborts BEFORE + // emitting (or allocating) anything; value_hex and raw_value abort within a chunk of + // the guard's threshold while they stream. (Mid-loop coverage for string escaping -- + // where the output size is NOT known up front -- is growth_guard_measures_escape_expansion.) constexpr size_t threshold = 64 * 1024; - // Abort bound: threshold + one stride of guarded emission (hex/raw chunk at the - // writer's stride; the escape loop yields every 128 input chars) + slack -- far below - // the full token size. + // Abort bound for the chunked emitters: threshold + one stride of guarded emission + + // slack -- far below the full token size. constexpr size_t abort_bound = threshold + 8 * 1024; const std::string big(1u << 20, 'x'); // 1 MiB payload; hex form is 2 MiB @@ -677,7 +677,7 @@ BOOST_AUTO_TEST_CASE(growth_guard_intra_token_abort) { std::string out; fc::json_writer w(out, [&](size_t sz) { if (sz > threshold) throw test_budget_abort{}; }, 1024); BOOST_CHECK_THROW(w.value_string(big), test_budget_abort); - BOOST_CHECK_LT(out.size(), abort_bound); + BOOST_CHECK_EQUAL(out.size(), 0u); // rejected by the size pre-check: nothing emitted at all } { std::string out; @@ -693,37 +693,83 @@ BOOST_AUTO_TEST_CASE(growth_guard_intra_token_abort) { } } +BOOST_AUTO_TEST_CASE(growth_guard_bounds_allocation) { + // A rejected token must not ALLOCATE its full size either: value_string / key pre-check + // the known input length with the guard BEFORE escape_string reserves capacity for it, + // so a 32 MiB string under a 64 KiB budget aborts with the buffer's capacity still at + // its initial slack -- not a ~32 MiB reservation that out.size() never reflects. + constexpr size_t threshold = 64 * 1024; + const std::string big(32u << 20, 'x'); // 32 MiB + { + std::string out; + fc::json_writer w(out, [&](size_t sz) { if (sz > threshold) throw test_budget_abort{}; }, 1024); + BOOST_CHECK_THROW(w.value_string(big), test_budget_abort); + BOOST_CHECK_LT(out.capacity(), threshold); // ~32 MiB before the size pre-check existed + BOOST_CHECK_EQUAL(out.size(), 0u); + } + { // oversized keys take the same pre-check path + std::string out; + fc::json_writer w(out, [&](size_t sz) { if (sz > threshold) throw test_budget_abort{}; }, 1024); + w.begin_object(); + BOOST_CHECK_THROW(w.key(big), test_budget_abort); + BOOST_CHECK_LT(out.capacity(), threshold); + } + { // an approved string keeps the single full-size reserve: no growth tax on the legit path + std::string out; + fc::json_writer w(out, [](size_t) { /* generous budget: never rejects */ }); + w.value_string(big); + BOOST_CHECK_EQUAL(out.size(), big.size() + 2); + BOOST_CHECK_GE(out.capacity(), big.size()); + } + { // unguarded writers are untouched: same output, same full reserve + std::string out; + fc::json_writer w(out); + w.value_string(big); + BOOST_CHECK_EQUAL(out.size(), big.size() + 2); + BOOST_CHECK_GE(out.capacity(), big.size()); + } +} + BOOST_AUTO_TEST_CASE(growth_guard_measures_escape_expansion) { // 32 KiB of control characters escape to ~192 KiB of output (6 bytes per input byte). - // A guard threshold above the input size but below the expanded size must still fire, - // proving the guard measures emitted bytes, not input consumed. + // The size pre-check approves the 32 KiB input (it is below the threshold), so this + // exercises the mid-loop machinery: a guard threshold above the input size but below + // the expanded size must still fire, proving the guard measures emitted bytes, not + // input consumed. const std::string ctrl(32 * 1024, '\x01'); std::string out; fc::json_writer w(out, [&](size_t sz) { if (sz > 100 * 1024) throw test_budget_abort{}; }, 1024); BOOST_CHECK_THROW(w.value_string(ctrl), test_budget_abort); BOOST_CHECK_GT(out.size(), 100 * 1024); // the expanded output crossed a threshold the raw input never reaches BOOST_CHECK_LT(out.size(), 128 * 1024); + // Allocation tracks the abort point, not some larger pre-computed estimate: capacity is + // the escape reserve for the approved input plus geometric growth to the abort. + BOOST_CHECK_LT(out.capacity(), 256 * 1024); } BOOST_AUTO_TEST_CASE(growth_guard_rearms_after_throw) { // A catch-and-continue serializer (eg the abi_serializer hex-fallback path) absorbing // the guard's throw must NOT disable the guard: the very next token re-invokes it // regardless of stride, so the abort re-raises until it unwinds out of the emitter. - // Once the guard passes again (budget freed), emission resumes normally. + // Once the guard passes again (budget freed), emission resumes normally. A rejected + // value_string mutates nothing (the size pre-check runs ahead of the separator), so + // absorb-and-continue leaves the buffer well-formed. std::string out; - bool reject = true; + bool reject = false; size_t calls = 0; fc::json_writer w(out, [&](size_t) { ++calls; if (reject) throw test_budget_abort{}; }, 4); w.begin_array(); - w.value_string("aaaaaaaaaa"); // grows past the stride; guard fires on the NEXT token - BOOST_CHECK_THROW(w.value_string("bbb"), test_budget_abort); // stride crossed -> guard throws, nothing appended + w.value_string("aaaaaaaaaa"); // pre-check crosses the stride; guard approves BOOST_CHECK_EQUAL(calls, 1u); - BOOST_CHECK_THROW(w.value_string("ccc"), test_budget_abort); // absorbed upstream? re-fires immediately + reject = true; // budget exhausted + BOOST_CHECK_THROW(w.value_string("bbb"), test_budget_abort); // pre-check throws, nothing appended BOOST_CHECK_EQUAL(calls, 2u); + BOOST_CHECK_THROW(w.value_string("ccc"), test_budget_abort); // absorbed upstream? re-fires immediately + BOOST_CHECK_EQUAL(calls, 3u); reject = false; // budget freed: same retry now passes w.value_string("ddd"); w.end_array(); - BOOST_CHECK_EQUAL(calls, 3u); + BOOST_CHECK_EQUAL(calls, 4u); BOOST_CHECK(w.balanced()); BOOST_CHECK_EQUAL(out, R"(["aaaaaaaaaa","ddd"])"); } From a3c456287aee85af746d4bcbaf494e763ba76ba8 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Thu, 23 Jul 2026 12:12:12 -0500 Subject: [PATCH 68/71] http: reserve queued streaming payloads against bytes_in_flight A typed result captured into a stream_emitter rode the dispatch queue to the http pool with nothing charged to bytes_in_flight until emission began, so queued results accumulated invisibly to --http-max-bytes-in-flight-mb -- unlike the variant path, which charges its in_flight_sizeof estimate before dispatch. bind_stream now reserves the result's packed-size estimate (http_plugin::reserve_bytes_in_flight) where the result is captured, and the RAII reservation rides the emitter closure until the captured payload is destroyed after emission. The dispatched handler re-checks the budget before running the emitter and drains over-budget work with the busy response, mirroring the variant path's pre-serialization verify. Trace-bearing results (streamed_processed_trace is emitted, never wire-packed) report their retained memory through a queued_payload_size customization point, and fc::nullable gains raw pack/unpack parity with std::optional so reflected results containing it can be sized. The regression test parks the single http pool thread inside a latched emitter, produces a 1 MiB bind_stream result on the app thread, and asserts the sampled bytes_in_flight covers the payload while its emitter waits in the queue, draining to zero afterwards. --- libraries/libfc/include/fc/io/raw.hpp | 14 ++ .../sysio/chain_plugin/chain_plugin.hpp | 11 ++ plugins/chain_plugin/src/chain_plugin.cpp | 23 ++++ .../include/sysio/http_plugin/bind_stream.hpp | 51 ++++++- .../include/sysio/http_plugin/common.hpp | 16 +++ .../include/sysio/http_plugin/http_plugin.hpp | 12 ++ plugins/http_plugin/src/http_plugin.cpp | 11 ++ plugins/http_plugin/test/unit_tests.cpp | 130 ++++++++++++++++++ 8 files changed, 266 insertions(+), 2 deletions(-) diff --git a/libraries/libfc/include/fc/io/raw.hpp b/libraries/libfc/include/fc/io/raw.hpp index 636d8fca37..062503360b 100644 --- a/libraries/libfc/include/fc/io/raw.hpp +++ b/libraries/libfc/include/fc/io/raw.hpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -259,6 +260,19 @@ namespace fc { else { v.reset(); } // in case v has already has a value } FC_RETHROW_EXCEPTIONS( warn, "optional<{}>", fc::get_typename::name() ) } + // fc::nullable -- binary serialization parity with std::optional: the wrapper only + // changes JSON emission (key always present, explicit null when disengaged), so its + // wire form delegates to the wrapped optional unchanged. + template + void pack( Stream& s, const fc::nullable& v ) { + fc::raw::pack( s, v.value ); + } + + template + void unpack( Stream& s, fc::nullable& v ) { + fc::raw::unpack( s, v.value ); + } + // std::vector template inline void pack( Stream& s, const std::vector& value ) { FC_ASSERT( value.size() <= MAX_SIZE_OF_BYTE_ARRAYS ); diff --git a/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp b/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp index 8cf41a747c..f68c7e5aa2 100644 --- a/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp +++ b/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp @@ -852,6 +852,17 @@ class read_write : public api_base { constexpr const char dec[] = "dec"; constexpr const char hex[] = "hex"; +/// Queued-payload size (http_detail::source_size_estimate ADL customization point) for +/// trace-bearing results: the generic reflected raw-pack cannot traverse +/// streamed_processed_trace (it is emitted, never wire-packed), so report the dominant +/// retained memory directly -- per-action payload bytes plus the captured ABI bytes. +/// Keeps the results visible to --http-max-bytes-in-flight-mb while they sit captured in +/// a queued stream_emitter. +size_t queued_payload_size(const streamed_processed_trace& t); +inline size_t queued_payload_size(const read_only::compute_transaction_results& r) { return queued_payload_size(r.processed); } +inline size_t queued_payload_size(const read_only::send_read_only_transaction_results& r) { return queued_payload_size(r.processed); } +inline size_t queued_payload_size(const read_write::send_transaction_results& r) { return queued_payload_size(r.processed); } + } // namespace chain_apis class chain_plugin : public plugin { diff --git a/plugins/chain_plugin/src/chain_plugin.cpp b/plugins/chain_plugin/src/chain_plugin.cpp index bb0a1a4182..36af32ec5b 100644 --- a/plugins/chain_plugin/src/chain_plugin.cpp +++ b/plugins/chain_plugin/src/chain_plugin.cpp @@ -3924,6 +3924,29 @@ read_only::get_consensus_parameters(const get_consensus_parameters_params&, cons return results; } +namespace { + /// Dominant retained memory of a transaction trace held by a queued response closure: + /// per-action payload bytes, console output, and return values. A reservation-grade + /// estimate, deliberately ignoring small fixed-size fields -- same semantics as + /// detail::in_flight_sizeof. + size_t trace_retained_size(const chain::transaction_trace& trace) { + size_t sz = sizeof(trace); + for (const auto& at : trace.action_traces) { + sz += sizeof(at) + at.act.data.size() + at.console.size() + at.return_value.size(); + } + return sz; + } +} + +size_t queued_payload_size(const streamed_processed_trace& t) { + size_t sz = 0; + if (t.trace) + sz += trace_retained_size(*t.trace); + for (const auto& [account, abi] : t.raw_abis.raw_abis) + sz += abi.size(); + return sz; +} + } // namespace chain_apis fc::variant chain_plugin::get_log_trx_trace(const transaction_trace_ptr& trx_trace ) const { diff --git a/plugins/http_plugin/include/sysio/http_plugin/bind_stream.hpp b/plugins/http_plugin/include/sysio/http_plugin/bind_stream.hpp index 7d0c7ec87e..2871be68a8 100644 --- a/plugins/http_plugin/include/sysio/http_plugin/bind_stream.hpp +++ b/plugins/http_plugin/include/sysio/http_plugin/bind_stream.hpp @@ -9,6 +9,7 @@ #include #include +#include #include #include @@ -19,6 +20,7 @@ #include #include #include +#include namespace sysio { @@ -184,6 +186,40 @@ namespace http_detail { } } + /// Packed-size estimate of a typed response payload, used to reserve bytes-in-flight + /// (http_plugin::reserve_bytes_in_flight) for as long as the payload sits captured in a + /// stream_emitter on the http thread pool queue. fc::raw::pack_size is an + /// allocation-free byte count over the same data the variant path estimates via + /// detail::in_flight_sizeof (pack_size of the response variant tree). + /// + /// Payload types the generic reflected pack cannot traverse (eg results embedding + /// streamed_processed_trace, which is emitted but never wire-packed) provide an + /// ADL-visible `queued_payload_size(const T&)` overload instead -- checked first. + /// Anything neither customized, reflected, nor a variant (emit-closure payloads) + /// returns 0: those are produced on the http pool and handed to the response callback + /// inline, so no queue window exists to charge, and a std::function cannot be packed. + template + inline size_t source_size_estimate(const T& r) { + if constexpr (requires { { queued_payload_size(r) } -> std::convertible_to; }) { + return queued_payload_size(r); + } else if constexpr (fc::reflector::is_defined::value || std::is_same_v) { + try { + return fc::raw::pack_size(r); + } catch (...) {} + } + return 0; + } + + /// Batch results (eg push_transactions' vector of per-transaction results) size as the + /// sum of their elements' estimates. + template + inline size_t source_size_estimate(const std::vector& v) { + size_t sz = 0; + for (const auto& e : v) + sz += source_size_estimate(e); + return sz; + } + } // namespace http_detail @@ -301,7 +337,13 @@ api_entry_stream bind_stream(http_plugin& http, Handle handle, // ---- Dispatch on Kind -------------------------------------- if constexpr (Kind == dispatch::sync) { auto result = invoke_sync(handle, std::move(params), deadline); - cb(resp_code, [r = std::move(result)](fc::json_writer& w) mutable { + // Reserve the typed result's packed size for as long as it rides the emitter + // closure across the queue to the http pool -- the streaming analogue of the + // variant path's pre-dispatch in_flight_sizeof charge. Released when the + // emitter (and with it the captured result) is destroyed after emission. + auto reservation = http.reserve_bytes_in_flight(source_size_estimate(result)); + cb(resp_code, + [r = std::move(result), reservation = std::move(reservation)](fc::json_writer& w) mutable { fc::to_json_stream(r, w); }); @@ -391,8 +433,13 @@ api_entry_stream bind_stream(http_plugin& http, Handle handle, api_name.c_str(), call_name.c_str(), body, cb); } } else if (std::holds_alternative(result)) { + // Same queued-payload reservation as dispatch::sync: the typed result + // is produced on whichever queue invoked the next_function and rides + // the emitter closure across to the http pool. + auto reservation = + http.reserve_bytes_in_flight(source_size_estimate(std::get(result))); cb(resp_code, - [r = std::get(std::move(result))] + [r = std::get(std::move(result)), reservation = std::move(reservation)] (fc::json_writer& w) mutable { fc::to_json_stream(r, w); }); } else { http.post_http_thread_pool( diff --git a/plugins/http_plugin/include/sysio/http_plugin/common.hpp b/plugins/http_plugin/include/sysio/http_plugin/common.hpp index 819f0052bb..ace206208e 100644 --- a/plugins/http_plugin/include/sysio/http_plugin/common.hpp +++ b/plugins/http_plugin/include/sysio/http_plugin/common.hpp @@ -273,6 +273,14 @@ inline auto make_http_response_handler(http_plugin_state& plugin_state, detail:: * over-budget emissions abort early instead of materializing in full; the variant * path achieves the analogous mid-flight visibility by pre-charging its * in_flight_sizeof estimate of the response variant tree. +* +* The captured source payload is accounted for separately: bind_stream reserves its +* packed-size estimate (http_plugin::reserve_bytes_in_flight) when it captures a typed +* result into the emitter, and that reservation rides the closure across this queue -- +* so results waiting here for a pool thread are already visible to the budget, exactly +* like the variant path's queued response variants. The dispatched lambda below +* re-checks the budget before emitting and drains over-budget work with the busy +* response without running the emitter at all. */ inline auto make_http_stream_response_handler(http_plugin_state& plugin_state, detail::abstract_conn_ptr session_ptr) { return url_response_stream_callback{ @@ -306,6 +314,14 @@ inline auto make_http_stream_response_handler(http_plugin_state& plugin_state, d }; auto on_exit = fc::make_scoped_exit([&settle_to]() { settle_to(0); }); try { + // Queued-payload reservations (bind_stream's source_size_estimate, still + // held by the emitter) and every other request's charges are visible here: + // reject over-budget work before emitting anything, mirroring the variant + // path's pre-serialization verify. + if (auto error_str = session_ptr->verify_max_bytes_in_flight(0); !error_str.empty()) { + session_ptr->send_busy_response(std::move(error_str)); + return; + } std::string body; { fc::json_writer w(body, [&settle_to, &session_ptr](size_t buffer_size) { diff --git a/plugins/http_plugin/include/sysio/http_plugin/http_plugin.hpp b/plugins/http_plugin/include/sysio/http_plugin/http_plugin.hpp index 1db3975064..334c49f3c4 100644 --- a/plugins/http_plugin/include/sysio/http_plugin/http_plugin.hpp +++ b/plugins/http_plugin/include/sysio/http_plugin/http_plugin.hpp @@ -228,6 +228,18 @@ namespace sysio { size_t bytes_in_flight() const; + /** + * @brief RAII charge against --http-max-bytes-in-flight-mb for a payload captured + * into a queued response closure. + * + * The charge is taken immediately and released when the returned token is + * destroyed -- so a typed result captured into a stream_emitter is accounted for + * exactly as long as it is alive, including the time it sits on the http thread + * pool queue before emission (the streaming analogue of the variant path's + * pre-dispatch in_flight_sizeof charge). Returns an empty token for size 0. + */ + std::shared_ptr reserve_bytes_in_flight(size_t size); + std::atomic& listening(); private: std::shared_ptr my; diff --git a/plugins/http_plugin/src/http_plugin.cpp b/plugins/http_plugin/src/http_plugin.cpp index ddcdfb8937..109fae9bf5 100644 --- a/plugins/http_plugin/src/http_plugin.cpp +++ b/plugins/http_plugin/src/http_plugin.cpp @@ -824,6 +824,17 @@ namespace sysio { return my->plugin_state->bytes_in_flight; } + std::shared_ptr http_plugin::reserve_bytes_in_flight(size_t size) { + if (size == 0) + return {}; + my->plugin_state->bytes_in_flight += size; + // Deleter-only token: the null pointer carries no object, but the deleter still runs + // exactly once when the last copy is destroyed, releasing the charge. Capturing the + // plugin_state shared_ptr keeps the counter alive past plugin shutdown. + return std::shared_ptr(static_cast(nullptr), + [state = my->plugin_state, size](void*) { state->bytes_in_flight -= size; }); + } + std::atomic& http_plugin::listening() { return my->listening; } diff --git a/plugins/http_plugin/test/unit_tests.cpp b/plugins/http_plugin/test/unit_tests.cpp index f5e4fbfc58..51571b48da 100644 --- a/plugins/http_plugin/test/unit_tests.cpp +++ b/plugins/http_plugin/test/unit_tests.cpp @@ -50,6 +50,7 @@ constexpr uint32_t stream_request_body_index = 8; constexpr uint32_t stream_response_bytes_index = 9; constexpr uint32_t stream_over_budget_index = 10; constexpr uint32_t stream_single_token_index = 11; +constexpr uint32_t stream_queued_payload_index = 12; // Keeps unsharded IPv6 probe coverage on the historical port 9999. constexpr uint32_t ipv6_probe_index = 2; @@ -1242,5 +1243,134 @@ BOOST_FIXTURE_TEST_CASE(stream_response_single_token_over_budget, http_plugin_te BOOST_CHECK_EQUAL(http_plugin->bytes_in_flight(), 0u); } +namespace { + /// FC_REFLECT'd payload big enough that its packed-size reservation is unmistakable in + /// the bytes_in_flight sample (and comfortably below the test budget). + struct queued_payload_result { + std::vector data; + }; + + /// Minimal api handle for driving bind_stream's dispatch::sync queued-payload + /// reservation end-to-end. The method blocks (on the app thread) until the test has + /// stalled the single http pool thread, so the emitter -- and the typed result it + /// captures -- must sit queued behind the stall the moment it is produced. + struct queued_payload_api { + std::atomic* entered; + std::atomic* proceed; + size_t payload_size; + + queued_payload_result make_big() { + entered->store(true); + while (!proceed->load()) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + return queued_payload_result{std::vector(payload_size, 'q')}; + } + }; +} +FC_REFLECT(queued_payload_result, (data)) + +// Typed results captured into a stream_emitter and queued to the http pool must be charged +// against bytes_in_flight for the time they sit in the queue -- the variant path charges its +// in_flight_sizeof estimate before dispatch, and the streaming path must not be blinder. +// With --http-threads=1, /stall_stream parks the only pool thread inside a latched emitter +// while the bind_stream endpoint's result is produced on the app thread; the peak sampled +// below reads ~0 without the source-size reservation and >= the payload size with it. The +// budget must then drain to zero, proving the reservation releases exactly once. +BOOST_FIXTURE_TEST_CASE(stream_queued_payload_reserved, http_plugin_test_fixture) { + const std::string endpoint = test_http_endpoint("127.0.0.1", stream_queued_payload_index); + const std::string server_address = "--http-server-address=" + endpoint; + const std::string port = test_http_port(stream_queued_payload_index); + + http_plugin* http_plugin = init({"--plugin=sysio::http_plugin", + server_address.c_str(), + "--http-threads=1", + "--http-max-bytes-in-flight-mb=64"}); + BOOST_REQUIRE(http_plugin); + + constexpr size_t payload_size = 1024 * 1024; + std::atomic entered{false}; + std::atomic proceed{false}; + std::atomic stalled{false}; + std::atomic release{false}; + + http_plugin->add_api_stream({ + bind_stream<&queued_payload_api::make_big, dispatch::sync>( + *http_plugin, queued_payload_api{&entered, &proceed, payload_size}, + "/v1/test/queued_payload", api_category::node, http_params_types::no_params, 200), + }, appbase::exec_queue::read_write); + + // Async streaming handlers run inline on the pool thread that read the request, and the + // response dispatch short-circuits on that same thread -- so this emitter's latch parks + // the pool's only thread. + http_plugin->add_async_api_stream({{std::string("/stall_stream"), api_category::node, + [&](string&&, string&&, url_response_stream_callback&& cb) { + cb(200, [&](fc::json_writer& w) { + stalled.store(true); + while (!release.load()) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + w.value_string("ok"); + }); + }}}); + + boost::asio::io_context ctx; + boost::asio::ip::tcp::resolver resolver(ctx); + + auto send_request = [&](const char* path) { + auto s = std::make_shared(ctx, boost::asio::ip::tcp::v4()); + boost::asio::connect(*s, resolver.resolve("127.0.0.1", port)); + boost::beast::http::request req(boost::beast::http::verb::get, path, 11); + req.set(http::field::host, endpoint); + boost::beast::http::write(*s, req); + return s; + }; + auto read_response = [&](boost::asio::ip::tcp::socket& s) { + boost::beast::http::response_parser parser; + parser.body_limit(16 * 1024 * 1024); // hex-encoded payload is ~2x its byte size + boost::beast::flat_buffer buffer; + boost::beast::http::read(s, buffer, parser); + return parser.release(); + }; + auto wait_for = [](auto&& pred) { // ~10s cap: pool scheduling is slow under sanitizers + for (int i = 0; i < 10000; ++i) { + if (pred()) + return true; + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + return pred(); + }; + + auto payload_socket = send_request("/v1/test/queued_payload"); + BOOST_REQUIRE(wait_for([&] { return entered.load(); })); // request read; app thread inside make_big + auto stall_socket = send_request("/stall_stream"); + BOOST_REQUIRE(wait_for([&] { return stalled.load(); })); // pool's only thread now parked + proceed.store(true); // result produced -> emitter queued behind the stall + + // Regression sample: reads ~0 without the reservation riding the queued emitter closure. + size_t peak = 0; + wait_for([&] { + peak = std::max(peak, http_plugin->bytes_in_flight()); + return peak >= payload_size; + }); + BOOST_CHECK_GE(peak, payload_size); + + release.store(true); + auto stall_resp = read_response(*stall_socket); + BOOST_CHECK(stall_resp.result() == boost::beast::http::status::ok); + auto payload_resp = read_response(*payload_socket); + BOOST_CHECK(payload_resp.result() == boost::beast::http::status::ok); + BOOST_CHECK_GT(payload_resp.body().size(), 2 * payload_size); // hex-encoded vector + + // requests_in_flight tracks session lifetime, so the keep-alive sockets must close + // before the drain below can reach zero. + stall_socket.reset(); + payload_socket.reset(); + + uint16_t max = std::numeric_limits::max(); + while (http_plugin->requests_in_flight() > 0 && --max) + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + BOOST_CHECK(max > 0); + BOOST_CHECK_EQUAL(http_plugin->bytes_in_flight(), 0u); +} + //A warning for future tests: destruction of http_plugin_test_fixture sometimes does not destroy http_plugin's listeners. Tests // added in the future should avoid reusing ports of other tests in http_plugin_unit_tests. From cd02577ab66d86f12bf16a89100c277233b14d31 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Thu, 23 Jul 2026 12:12:33 -0500 Subject: [PATCH 69/71] trace_api: emit trace responses through the guarded streaming writer get_block and get_transaction_trace built their entire response into an unguarded local string and handed it through the legacy json_raw callback, so the http memory cap only saw the body after it was fully allocated. Both endpoints now register via add_async_api_stream: misses resolve to their 404 before an emitter is committed (contains_transaction gates the per-transaction case), and the emitter writes directly into the guarded json_writer the streaming callback provides, so --http-max-bytes-in-flight-mb observes and can abort large trace responses while they serialize. Error responses emit error_results through the same streaming path. response_formatter grows writer-taking process_*_to_stream forms; the buffer-building *_to_json forms remain as thin wrappers for tests and the trace_api_json benchmark. The budget regression test drives the block emitter into a small-threshold guarded writer and asserts the abort lands within a token of the threshold, far below the full body. --- .../sysio/trace_api/request_handler.hpp | 144 ++++++++++++----- .../trace_api_plugin/src/request_handler.cpp | 38 ++++- .../trace_api_plugin/src/trace_api_plugin.cpp | 76 ++++----- .../trace_api_plugin/test/test_responses.cpp | 145 ++++++++++++++++++ 4 files changed, 323 insertions(+), 80 deletions(-) diff --git a/plugins/trace_api_plugin/include/sysio/trace_api/request_handler.hpp b/plugins/trace_api_plugin/include/sysio/trace_api/request_handler.hpp index 087742ea06..b2ee3b3f13 100644 --- a/plugins/trace_api_plugin/include/sysio/trace_api/request_handler.hpp +++ b/plugins/trace_api_plugin/include/sysio/trace_api/request_handler.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -28,16 +29,31 @@ namespace sysio::trace_api { public: static fc::variant process_block( const data_log_entry& trace, bool irreversible, const data_handler_function& data_handler ); - // Streaming counterpart: emit the same JSON payload as process_block but write it - // directly into an output std::string via fc::json_writer, skipping the - // fc::mutable_variant_object tree entirely. Output is byte-for-byte identical - // to fc::json::to_string(process_block(...)). + // Streaming counterpart: emit the same JSON payload as process_block directly into the + // caller-provided fc::json_writer, skipping the fc::mutable_variant_object tree entirely. + // Output is byte-for-byte identical to fc::json::to_string(process_block(...)). The HTTP + // path passes the guarded writer the streaming callback provides, so the memory budget + // observes the response while it serializes. + static void process_block_to_stream( const data_log_entry& trace, bool irreversible, const stream_data_handler_function& data_handler, fc::json_writer& w ); + + // Buffer-building wrapper over process_block_to_stream (unguarded writer into a local + // string). Used by tests and the trace_api_json benchmark. static std::string process_block_to_json( const data_log_entry& trace, bool irreversible, const stream_data_handler_function& data_handler ); - // Streaming counterpart for a single transaction inside a block trace. Walks the block's transaction list and, - // for the first entry whose id matches trxid, emits just that transaction's JSON object (no surrounding block - // wrapper). Returns an empty string if no transaction in the block matches. Output for a matching trx is - // byte-for-byte identical to fc::json::to_string(get_transaction_trace(...)). + // True iff the block trace contains a transaction with the given id. Lets the HTTP layer + // resolve its 404 before committing to a 200 emitter -- a streaming response cannot signal + // "not found" after emission has begun. + static bool contains_transaction( const data_log_entry& trace, const chain::transaction_id_type& trxid ); + + // Streaming counterpart for a single transaction inside a block trace. Walks the block's + // transaction list and, for the first entry whose id matches trxid, emits just that + // transaction's JSON object (no surrounding block wrapper) into the caller-provided writer. + // Returns false (and emits nothing) if no transaction in the block matches. Output for a + // matching trx is byte-for-byte identical to fc::json::to_string(get_transaction_trace(...)). + static bool process_transaction_to_stream( const data_log_entry& trace, const chain::transaction_id_type& trxid, const stream_data_handler_function& data_handler, fc::json_writer& w ); + + // Buffer-building wrapper over process_transaction_to_stream; returns an empty string on a + // miss. Used by tests. static std::string process_transaction_to_json( const data_log_entry& trace, const chain::transaction_id_type& trxid, const stream_data_handler_function& data_handler ); }; } @@ -215,27 +231,47 @@ namespace sysio::trace_api { } /** - * Streaming variant of get_block_trace: emits the JSON response body directly into - * a std::string via fc::json_writer instead of building an intermediate - * fc::mutable_variant_object tree. Returned string is empty if the block does not - * exist; callers should check empty() before dispatching the 200 response. - * Skips the fc::variant -> fc::json::to_string copy on the response path when paired - * with http_content_type::json_raw. + * Streaming variant of get_block_trace for the HTTP streaming-callback path. + * + * Resolves block existence up front so the caller can produce its 404 before + * committing to a 200 emitter; the returned closure (owning the block's decoded + * trace data) emits the response body into the caller-provided fc::json_writer. + * The HTTP layer passes its guarded writer, so the memory budget observes -- and + * can abort -- the response while it serializes instead of after it is allocated. + * + * @param block_height - the height of the block whose trace is requested + * @return an emitter for the block's trace, or std::nullopt if the block does not exist + * @throws bad_data_exception when there are issues with the underlying data preventing processing. */ - std::string get_block_trace_json( uint32_t block_height ) { + std::optional> get_block_trace_emitter( uint32_t block_height ) { auto data = logfile_provider.get_block(block_height); if (!data) { _log("No block found at block height " + std::to_string(block_height) ); - return {}; + return std::nullopt; } - auto data_handler = [this](const std::variant& action, fc::json_writer& w) { - std::visit([&](const auto& a) { - data_handler_provider.serialize_to_json_stream(a, w); - }, action); + return [this, data = std::move(*data)](fc::json_writer& w) { + detail::response_formatter::process_block_to_stream(std::get<0>(data), std::get<1>(data), + make_stream_data_handler(), w); }; + } - return detail::response_formatter::process_block_to_json(std::get<0>(*data), std::get<1>(*data), data_handler); + /** + * Buffer-building wrapper over get_block_trace_emitter: emits into a local string via + * an unguarded writer. Returned string is empty if the block does not exist. Kept + * for direct (non-HTTP) callers and tests; the HTTP path uses the emitter form so the + * guarded writer's budget applies. + */ + std::string get_block_trace_json( uint32_t block_height ) { + auto emit = get_block_trace_emitter(block_height); + if (!emit) + return {}; + std::string out; + { + fc::json_writer w(out); + (*emit)(w); + } + return out; } /** @@ -307,31 +343,54 @@ namespace sysio::trace_api { } /** - * Streaming variant of get_transaction_trace: emits the matching transaction's JSON directly into a std::string - * via fc::json_writer instead of building the full block's fc::variant tree only to extract one transaction. - * Returned string is empty if the block does not exist or the trxid is not present in the block; callers should - * check empty() before dispatching the 200 response. Skips the fc::variant -> fc::json::to_string copy on the - * response path when paired with http_content_type::json_raw. + * Streaming variant of get_transaction_trace for the HTTP streaming-callback path. + * + * Resolves both misses up front -- block absent, or trxid not present in the block + * (via response_formatter::contains_transaction) -- so the caller can produce its + * 404 before committing to a 200 emitter; the returned closure emits just the + * matching transaction's JSON into the caller-provided (guarded) fc::json_writer. + * + * @param trxid - the transaction id whose trace is requested + * @param block_height - the height of the block whose trace contains the transaction + * @return an emitter for the transaction's trace, or std::nullopt on either miss + * @throws bad_data_exception when there are issues with the underlying data preventing processing. */ - std::string get_transaction_trace_json( const chain::transaction_id_type& trxid, uint32_t block_height ) { - _log("get_transaction_trace_json called"); + std::optional> get_transaction_trace_emitter( const chain::transaction_id_type& trxid, uint32_t block_height ) { + _log("get_transaction_trace_emitter called"); auto data = logfile_provider.get_block(block_height); if (!data) { _log("No block found at block height " + std::to_string(block_height)); - return {}; + return std::nullopt; + } + if (!detail::response_formatter::contains_transaction(std::get<0>(*data), trxid)) { + _log("No transaction with id " + trxid.str() + " found in block " + std::to_string(block_height)); + return std::nullopt; } - auto data_handler = [this](const std::variant& action, fc::json_writer& w) { - std::visit([&](const auto& a) { - data_handler_provider.serialize_to_json_stream(a, w); - }, action); + return [this, data = std::move(*data), trxid](fc::json_writer& w) { + [[maybe_unused]] const bool found = detail::response_formatter::process_transaction_to_stream( + std::get<0>(data), trxid, make_stream_data_handler(), w); + assert(found); // contains_transaction gated emitter creation }; + } - auto resp = detail::response_formatter::process_transaction_to_json(std::get<0>(*data), trxid, data_handler); - if (resp.empty()) { - _log("No transaction with id " + trxid.str() + " found in block " + std::to_string(block_height)); + /** + * Buffer-building wrapper over get_transaction_trace_emitter: emits into a local + * string via an unguarded writer. Returned string is empty if the block does not + * exist or the trxid is not present in the block. Kept for direct (non-HTTP) + * callers and tests; the HTTP path uses the emitter form so the guarded writer's + * budget applies. + */ + std::string get_transaction_trace_json( const chain::transaction_id_type& trxid, uint32_t block_height ) { + auto emit = get_transaction_trace_emitter(trxid, block_height); + if (!emit) + return {}; + std::string out; + { + fc::json_writer w(out); + (*emit)(w); } - return resp; + return out; } /** @@ -359,6 +418,17 @@ namespace sysio::trace_api { } private: + /// Adapter from the provider's per-action streaming serializer to the + /// stream_data_handler_function shape the response_formatter consumes. Shared by + /// both emitter factories; binds `this`, so emitters must not outlive the handler. + stream_data_handler_function make_stream_data_handler() { + return [this](const std::variant& action, fc::json_writer& w) { + std::visit([&](const auto& a) { + data_handler_provider.serialize_to_json_stream(a, w); + }, action); + }; + } + actions_result get_actions_impl(const action_query& query, variant_shape shape) { actions_result result; diff --git a/plugins/trace_api_plugin/src/request_handler.cpp b/plugins/trace_api_plugin/src/request_handler.cpp index 202e0dac0e..6bafad0aac 100644 --- a/plugins/trace_api_plugin/src/request_handler.cpp +++ b/plugins/trace_api_plugin/src/request_handler.cpp @@ -249,9 +249,7 @@ namespace { } namespace sysio::trace_api::detail { - std::string response_formatter::process_block_to_json( const data_log_entry& trace, bool irreversible, const stream_data_handler_function& data_handler ) { - std::string out; - fc::json_writer w(out); + void response_formatter::process_block_to_stream( const data_log_entry& trace, bool irreversible, const stream_data_handler_function& data_handler, fc::json_writer& w ) { std::visit([&](auto&& bt) { w.begin_object(); w.set("id", bt.id) @@ -266,20 +264,44 @@ namespace sysio::trace_api::detail { w.key("transactions"); write_transactions(w, bt.transactions, data_handler); w.end_object(); }, trace); - return out; } - std::string response_formatter::process_transaction_to_json( const data_log_entry& trace, const chain::transaction_id_type& trxid, const stream_data_handler_function& data_handler ) { + std::string response_formatter::process_block_to_json( const data_log_entry& trace, bool irreversible, const stream_data_handler_function& data_handler ) { std::string out; - std::visit([&](auto&& bt) { + fc::json_writer w(out); + process_block_to_stream(trace, irreversible, data_handler, w); + return out; + } + + bool response_formatter::contains_transaction( const data_log_entry& trace, const chain::transaction_id_type& trxid ) { + return std::visit([&](auto&& bt) { + for (const auto& t : bt.transactions) { + if (t.id == trxid) + return true; + } + return false; + }, trace); + } + + bool response_formatter::process_transaction_to_stream( const data_log_entry& trace, const chain::transaction_id_type& trxid, const stream_data_handler_function& data_handler, fc::json_writer& w ) { + return std::visit([&](auto&& bt) { for (const auto& t : bt.transactions) { if (t.id == trxid) { - fc::json_writer w(out); write_transaction(w, t, data_handler); - return; + return true; } } + return false; }, trace); + } + + std::string response_formatter::process_transaction_to_json( const data_log_entry& trace, const chain::transaction_id_type& trxid, const stream_data_handler_function& data_handler ) { + std::string out; + { + fc::json_writer w(out); + if (!process_transaction_to_stream(trace, trxid, data_handler, w)) + return {}; // miss: preserve the empty-string contract (out holds only the writer's reserve) + } return out; } } diff --git a/plugins/trace_api_plugin/src/trace_api_plugin.cpp b/plugins/trace_api_plugin/src/trace_api_plugin.cpp index bb9077b59f..154a58363c 100644 --- a/plugins/trace_api_plugin/src/trace_api_plugin.cpp +++ b/plugins/trace_api_plugin/src/trace_api_plugin.cpp @@ -10,6 +10,8 @@ #include +#include + #include #include @@ -142,6 +144,16 @@ namespace { std::shared_ptr store; }; + + /// Build a stream_emitter for an error_results payload -- the streaming twin of the + /// old `cb(code, fc::variant(error_results{...}))` error responses. Emits through the + /// same reflector path http_plugin::handle_exception_stream uses, so the error body + /// shape matches the variant path byte-for-byte. + sysio::stream_emitter make_error_results_emitter(sysio::error_results results) { + return [results = std::move(results)](fc::json_writer& w) mutable { + fc::to_json_stream(results, w); + }; + } } namespace sysio { @@ -283,13 +295,15 @@ struct trace_api_rpc_plugin_impl : public std::enable_shared_from_this(); - // Content-Type: application/json, but response body is built via the streaming - // fc::json_writer path and passed through as-is. Skips the fc::variant tree - // build + fc::json::to_string on the 200 response. Error responses still go via - // the fc::variant path (they're tiny and infrequent). - http.add_async_handler({"/v1/trace_api/get_block", + // Streaming-cb registration: the emitter writes the response body directly into the + // guarded fc::json_writer the http layer provides, so --http-max-bytes-in-flight-mb + // observes -- and can abort -- large trace responses WHILE they serialize, instead + // of only after the full body has been allocated (the old json_raw pass-through). + // Misses resolve to their 404 before an emitter is committed; error responses emit + // tiny error_results objects through the same streaming path. + http.add_async_api_stream({{std::string("/v1/trace_api/get_block"), api_category::trace_api, - [self = shared_from_this()](std::string, std::string body, url_response_callback cb) + [self = shared_from_this()](std::string&&, std::string&& body, url_response_stream_callback&& cb) { auto block_number = ([&body]() -> std::optional { if (body.empty()) { @@ -309,33 +323,28 @@ struct trace_api_rpc_plugin_impl : public std::enable_shared_from_thisreq_handler->get_block_trace_json(*block_number); - if (resp.empty()) { - error_results results{404, "Trace API: block trace missing"}; - cb( 404, fc::variant( results )); + auto emitter = self->req_handler->get_block_trace_emitter(*block_number); + if (!emitter) { + cb( 404, make_error_results_emitter({404, "Trace API: block trace missing"}) ); } else { - // Pre-serialized JSON body: wrap in a string-valued variant so the - // json_raw content-type path sends it verbatim. Error responses in - // this handler (and anything routed through http_plugin::handle_exception) - // stay as object-valued variants; common.hpp detects that and falls - // back to fc::json::to_string. - cb( 200, fc::variant( std::move(resp) ) ); + // self keeps req_handler (bound as `this` inside the emitter) alive until + // emission completes on the http thread pool. + cb( 200, [self, emit = std::move(*emitter)](fc::json_writer& w) { emit(w); } ); } } catch (...) { - http_plugin::handle_exception("trace_api", "get_block", body, cb); + http_plugin::handle_exception_stream("trace_api", "get_block", body, cb); } - }}, http_content_type::json_raw); + }}}); - http.add_async_handler({"/v1/trace_api/get_transaction_trace", + http.add_async_api_stream({{std::string("/v1/trace_api/get_transaction_trace"), api_category::trace_api, - [self = shared_from_this()](std::string, std::string body, url_response_callback cb) + [self = shared_from_this()](std::string&&, std::string&& body, url_response_stream_callback&& cb) { auto trx_id = ([&body]() -> std::optional { if (body.empty()) { @@ -354,8 +363,7 @@ struct trace_api_rpc_plugin_impl : public std::enable_shared_from_thiscommon->store->get_trx_block_number(*trx_id); if (!blk_num.has_value()){ - error_results results{404, "Trace API: transaction id missing in the transaction id log files"}; - cb( 404, fc::variant( results )); + cb( 404, make_error_results_emitter({404, "Trace API: transaction id missing in the transaction id log files"}) ); } else { - std::string resp = self->req_handler->get_transaction_trace_json(*trx_id, *blk_num); - if (resp.empty()) { - error_results results{404, "Trace API: transaction trace missing"}; - cb( 404, fc::variant( results )); + auto emitter = self->req_handler->get_transaction_trace_emitter(*trx_id, *blk_num); + if (!emitter) { + cb( 404, make_error_results_emitter({404, "Trace API: transaction trace missing"}) ); } else { - // Pre-serialized JSON body: wrap in a string-valued variant so the - // json_raw content-type path sends it verbatim, mirroring get_block. - cb( 200, fc::variant( std::move(resp) ) ); + // self keeps req_handler (bound as `this` inside the emitter) alive until + // emission completes on the http thread pool, mirroring get_block. + cb( 200, [self, emit = std::move(*emitter)](fc::json_writer& w) { emit(w); } ); } } } catch (...) { - http_plugin::handle_exception("trace_api", "get_transaction", body, cb); + http_plugin::handle_exception_stream("trace_api", "get_transaction", body, cb); } - }}, http_content_type::json_raw); + }}}); http.add_async_handler({"/v1/trace_api/get_actions", api_category::trace_api, diff --git a/plugins/trace_api_plugin/test/test_responses.cpp b/plugins/trace_api_plugin/test/test_responses.cpp index 0a006b009a..d45164ff82 100644 --- a/plugins/trace_api_plugin/test/test_responses.cpp +++ b/plugins/trace_api_plugin/test/test_responses.cpp @@ -94,6 +94,14 @@ struct response_test_fixture { return response_impl.get_transaction_trace_json( trxid, block_height ); } + std::optional> get_block_trace_emitter( uint32_t block_height ) { + return response_impl.get_block_trace_emitter( block_height ); + } + + std::optional> get_transaction_trace_emitter( const chain::transaction_id_type& trxid, uint32_t block_height ) { + return response_impl.get_transaction_trace_emitter( trxid, block_height ); + } + // fixture data and methods std::function mock_get_block; std::function>(const action_trace_v0&)> mock_data_handler = default_mock_data_handler; @@ -700,4 +708,141 @@ BOOST_AUTO_TEST_SUITE(trace_responses) } } + // The HTTP layer commits to a 200 before the emitter runs, so both emitter factories must + // resolve every miss (absent block, absent trxid) to nullopt up front -- and an engaged + // emitter must produce exactly the bytes the buffer-building wrappers produce. + BOOST_FIXTURE_TEST_CASE(streaming_emitter_miss_resolution_and_parity, response_test_fixture) + { + auto trx_id = "0000000000000000000000000000000000000000000000000000000000000001"_h; + + action_trace_v0 action_trace{}; + action_trace.global_sequence = 0; + action_trace.receiver = "receiver"_n; + action_trace.account = "contract"_n; + action_trace.action = "action"_n; + action_trace.authorization = {{ "alice"_n, "active"_n }}; + action_trace.data = { 0x00, 0x01, 0x02, 0x03 }; + + auto block_trace = block_trace_v0 { + "b000000000000000000000000000000000000000000000000000000000000001"_h, + 1, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + chain::block_timestamp_type(0), + "bp.one"_n, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + std::vector { + { + trx_id, + std::vector { action_trace }, + 10, + 5, + std::vector{ chain::signature_type() }, + { chain::time_point_sec(), 1, 0, 100, 50, 0 }, + 1, + chain::block_timestamp_type(0), + std::optional{} + } + } + }; + + mock_get_block = [&block_trace]( uint32_t height ) -> get_block_t { + if (height != 1) { + return {}; + } + return std::make_tuple(data_log_entry(block_trace), false); + }; + + auto drive = [](const std::function& emit) { + std::string out; + { + fc::json_writer w(out); + emit(w); + } + return out; + }; + + // Engaged emitters emit byte-identically to the wrappers. + auto block_emitter = get_block_trace_emitter( 1 ); + BOOST_REQUIRE(block_emitter.has_value()); + BOOST_CHECK_EQUAL(drive(*block_emitter), get_block_trace_json( 1 )); + + auto trx_emitter = get_transaction_trace_emitter( trx_id, 1 ); + BOOST_REQUIRE(trx_emitter.has_value()); + BOOST_CHECK_EQUAL(drive(*trx_emitter), get_transaction_trace_json( trx_id, 1 )); + + // Misses resolve before any emitter exists: absent block, and absent trxid in a + // present block (contains_transaction gating). + BOOST_CHECK(!get_block_trace_emitter( 2 ).has_value()); + auto missing_id = "00000000000000000000000000000000000000000000000000000000000000ff"_h; + BOOST_CHECK(!get_transaction_trace_emitter( missing_id, 1 ).has_value()); + BOOST_CHECK(!get_transaction_trace_emitter( trx_id, 2 ).has_value()); + } + + // Trace responses must be abortable by the http layer's memory budget WHILE they + // serialize: driving the block emitter into a small-threshold guarded writer (the shape + // make_http_stream_response_handler provides) must throw the guard's abort partway + // through emission, with the buffer far below the full body size -- the old json_raw + // path materialized the entire body before the budget ever saw a byte. + BOOST_FIXTURE_TEST_CASE(streaming_block_response_budget_abort, response_test_fixture) + { + // 32 transactions x 4 KiB action payload: full body far exceeds the 8 KiB threshold + // below (each payload alone hex-expands past it). + constexpr size_t payload_bytes = 4 * 1024; + constexpr size_t transaction_count = 32; + std::vector transactions; + transactions.reserve(transaction_count); + for (size_t i = 0; i < transaction_count; ++i) { + action_trace_v0 action_trace{}; + action_trace.global_sequence = i; + action_trace.receiver = "receiver"_n; + action_trace.account = "contract"_n; + action_trace.action = "action"_n; + action_trace.authorization = {{ "alice"_n, "active"_n }}; + action_trace.data.assign(payload_bytes, static_cast(i)); + transactions.push_back(transaction_trace_v0{ + fc::sha256::hash(static_cast(i)), + std::vector { std::move(action_trace) }, + 10, + 5, + std::vector{ chain::signature_type() }, + { chain::time_point_sec(), 1, 0, 100, 50, 0 }, + 1, + chain::block_timestamp_type(0), + std::optional{} + }); + } + + auto block_trace = block_trace_v0 { + "b000000000000000000000000000000000000000000000000000000000000001"_h, + 1, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + chain::block_timestamp_type(0), + "bp.one"_n, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + std::move(transactions) + }; + + mock_get_block = [&block_trace]( uint32_t height ) -> get_block_t { + return std::make_tuple(data_log_entry(block_trace), false); + }; + + const std::string full_body = get_block_trace_json( 1 ); + + /// Foreign (non-fc, non-std) exception type modeling the http layer's budget abort + /// (sysio::detail::stream_response_budget_exceeded). + struct budget_abort {}; + constexpr size_t threshold = 8 * 1024; + + auto emitter = get_block_trace_emitter( 1 ); + BOOST_REQUIRE(emitter.has_value()); + std::string out; + fc::json_writer w(out, [](size_t sz) { if (sz > threshold) throw budget_abort{}; }, 256); + BOOST_CHECK_THROW((*emitter)(w), budget_abort); + // Aborted within a token or two of the threshold; nowhere near the full body. + BOOST_CHECK_LT(out.size(), 2 * threshold); + BOOST_CHECK_GT(full_body.size(), 8 * threshold); + } + BOOST_AUTO_TEST_SUITE_END() From 046495480c93492409802e89bba4fc2de746aef7 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Thu, 23 Jul 2026 12:12:34 -0500 Subject: [PATCH 70/71] fc: preserve NaN sign in streaming double emission The streaming formatter emitted "nan" for every NaN while the variant path (stringstream << std::fixed, delegating to the C library) prints "-nan" when the sign bit is set, breaking byte-for-byte parity for ABI float bit patterns that carry a negative NaN. NaN never compares, so the sign now comes from signbit for both the nan and inf branches. Parity coverage spans the libfc leaf-type checks (+-NaN, +-inf, float -nan) and an abi_serializer case that unpacks raw float64/float32 negative-NaN bit patterns -- unreachable from JSON input -- through binary_to_json_stream against the variant path. --- .../libfc/include/fc/reflect/json_stream.hpp | 8 ++- libraries/libfc/test/io/test_json_stream.cpp | 13 ++++ unittests/abi_tests.cpp | 66 +++++++++++++++++++ 3 files changed, 85 insertions(+), 2 deletions(-) diff --git a/libraries/libfc/include/fc/reflect/json_stream.hpp b/libraries/libfc/include/fc/reflect/json_stream.hpp index c2325870c5..c3c37cbe43 100644 --- a/libraries/libfc/include/fc/reflect/json_stream.hpp +++ b/libraries/libfc/include/fc/reflect/json_stream.hpp @@ -87,8 +87,12 @@ inline constexpr int double_fixed_precision = std::numeric_limits::di inline constexpr size_t double_fixed_buf_size = std::numeric_limits::max_exponent10 + double_fixed_precision + 8; inline void to_json_stream(double d, json_writer& w) { if (!std::isfinite(d)) { - // Variant emits the literal "nan" / "inf" / "-inf" string for non-finite doubles; match that shape. - w.value_string(std::isnan(d) ? "nan" : (d > 0 ? "inf" : "-inf")); + // Variant (stringstream << std::fixed, delegating to the C library) emits the literal + // "nan" / "-nan" / "inf" / "-inf" strings for non-finite doubles; match that shape. + // NaN never compares, so its sign must come from signbit -- arbitrary ABI float bit + // patterns reach here and negative NaN must round-trip byte-identically. + w.value_string(std::isnan(d) ? (std::signbit(d) ? "-nan" : "nan") + : (std::signbit(d) ? "-inf" : "inf")); return; } char buf[double_fixed_buf_size]; diff --git a/libraries/libfc/test/io/test_json_stream.cpp b/libraries/libfc/test/io/test_json_stream.cpp index 4babe3776c..89c22068e4 100644 --- a/libraries/libfc/test/io/test_json_stream.cpp +++ b/libraries/libfc/test/io/test_json_stream.cpp @@ -23,6 +23,7 @@ #include #include +#include #include #include #include @@ -557,6 +558,18 @@ BOOST_AUTO_TEST_CASE(streaming_vs_variant_parity_libfc_leaf_types) { check_streaming_matches_variant(double{1e10}, "double 1e10"); check_streaming_matches_variant(double{0.0}, "double 0.0"); check_streaming_matches_variant(float{3.5f}, "float 3.5"); + // Non-finite doubles: the legacy path (stringstream << std::fixed, delegating to the + // C library) prints "nan" / "-nan" / "inf" / "-inf". NaN never compares, so its sign + // only reaches the output via signbit -- and arbitrary ABI float bit patterns can + // carry either sign. + const double quiet_nan = std::numeric_limits::quiet_NaN(); + const double inf = std::numeric_limits::infinity(); + check_streaming_matches_variant(quiet_nan, "double nan"); + check_streaming_matches_variant(std::copysign(quiet_nan, -1.0), "double -nan"); + check_streaming_matches_variant(inf, "double inf"); + check_streaming_matches_variant(-inf, "double -inf"); + check_streaming_matches_variant(std::copysign(std::numeric_limits::quiet_NaN(), -1.0f), + "float -nan"); } // bls public_key + signature: derived from a deterministic seed via private_key::generate(). // private_key itself has =delete'd to_json_stream so we don't include it here. diff --git a/unittests/abi_tests.cpp b/unittests/abi_tests.cpp index 2f25767a2b..6e83d5f3ae 100644 --- a/unittests/abi_tests.cpp +++ b/unittests/abi_tests.cpp @@ -1,7 +1,10 @@ #include +#include +#include #include #include #include +#include #include @@ -819,6 +822,69 @@ BOOST_AUTO_TEST_CASE(large_int_quoting_boundaries) } FC_LOG_AND_RETHROW() } +// Non-finite float32/float64 bit patterns through the ABI paths: the variant path +// (stringstream << std::fixed, delegating to the C library) prints "nan" / "-nan" / +// "inf" / "-inf", and binary_to_json_stream must agree byte-for-byte. NaN never +// compares, so its sign only survives via signbit -- and since no JSON input can +// produce a NaN, the packed bytes are built directly from raw bit patterns, exactly +// as an arbitrary on-chain payload could. +BOOST_AUTO_TEST_CASE(non_finite_float_sign_parity) +{ try { + + const char* test_abi = R"=====( + { + "version": "sysio::abi/1.0", + "types": [], + "structs": [{ + "name": "floats", + "base": "", + "fields": [{ + "name": "f64_pos_nan", + "type": "float64" + },{ + "name": "f64_neg_nan", + "type": "float64" + },{ + "name": "f64_neg_inf", + "type": "float64" + },{ + "name": "f32_neg_nan", + "type": "float32" + }] + }], + "actions": [], + "tables": [], + "ricardian_clauses": [] + } + )====="; + + abi_serializer abis(fc::json::from_string(test_abi).as(), yield_fn()); + + const double f64_pos_nan = std::bit_cast(uint64_t{0x7FF8000000000000ULL}); + const double f64_neg_nan = std::bit_cast(uint64_t{0xFFF8000000000000ULL}); + const double f64_neg_inf = -std::numeric_limits::infinity(); + const float f32_neg_nan = std::bit_cast(uint32_t{0xFFC00000u}); + + bytes packed(3 * sizeof(double) + sizeof(float)); + fc::datastream ds(packed.data(), packed.size()); + fc::raw::pack(ds, f64_pos_nan); + fc::raw::pack(ds, f64_neg_nan); + fc::raw::pack(ds, f64_neg_inf); + fc::raw::pack(ds, f32_neg_nan); + + const std::string variant_json = + fc::json::to_string(abis.binary_to_variant("floats", packed, yield_fn()), get_deadline()); + + // Pin the legacy shapes, sign included. + BOOST_CHECK(variant_json.find("\"f64_pos_nan\":\"nan\"") != std::string::npos); + BOOST_CHECK(variant_json.find("\"f64_neg_nan\":\"-nan\"") != std::string::npos); + BOOST_CHECK(variant_json.find("\"f64_neg_inf\":\"-inf\"") != std::string::npos); + BOOST_CHECK(variant_json.find("\"f32_neg_nan\":\"-nan\"") != std::string::npos); + + verify_stream_matches_variant(abis, "floats", packed, variant_json); + +} FC_LOG_AND_RETHROW() } + // Shared fixture for `general` and the streaming-parity test. Top-level fields // cover every built-in plus inherited base structs, large-int quoting boundaries, From e00f963fb493b336112708c6dd33117ed6fa55d6 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Thu, 23 Jul 2026 17:47:13 -0500 Subject: [PATCH 71/71] fc: format non-finite doubles sign-aware in the variant path The variant path left non-finite formatting to the C library behind stringstream, whose negative-NaN spelling is platform-dependent: glibc prints "-nan" while Apple's libc drops the sign and prints "nan". The streaming path's signbit-aware emission therefore matched the variant path on Linux but diverged on macOS, where the parity tests caught the legacy path emitting "nan" for a negative NaN. s_fc_to_string now formats nan/inf explicitly from isnan/isinf + signbit, making both paths byte-identical and deterministic on every platform while leaving Linux output unchanged. Absolute spelling pins join the parity checks so the contract no longer depends on the platform's C library. --- libraries/libfc/include/fc/reflect/json_stream.hpp | 9 +++++---- libraries/libfc/src/variant.cpp | 10 ++++++++++ libraries/libfc/test/io/test_json_stream.cpp | 14 ++++++++++---- unittests/abi_tests.cpp | 11 ++++++----- 4 files changed, 31 insertions(+), 13 deletions(-) diff --git a/libraries/libfc/include/fc/reflect/json_stream.hpp b/libraries/libfc/include/fc/reflect/json_stream.hpp index c3c37cbe43..4c569b45d5 100644 --- a/libraries/libfc/include/fc/reflect/json_stream.hpp +++ b/libraries/libfc/include/fc/reflect/json_stream.hpp @@ -87,10 +87,11 @@ inline constexpr int double_fixed_precision = std::numeric_limits::di inline constexpr size_t double_fixed_buf_size = std::numeric_limits::max_exponent10 + double_fixed_precision + 8; inline void to_json_stream(double d, json_writer& w) { if (!std::isfinite(d)) { - // Variant (stringstream << std::fixed, delegating to the C library) emits the literal - // "nan" / "-nan" / "inf" / "-inf" strings for non-finite doubles; match that shape. - // NaN never compares, so its sign must come from signbit -- arbitrary ABI float bit - // patterns reach here and negative NaN must round-trip byte-identically. + // Variant (s_fc_to_string, variant.cpp) emits the explicit "nan" / "-nan" / "inf" / + // "-inf" spellings for non-finite doubles -- deliberately NOT the C library's, whose + // negative-NaN formatting is platform-dependent (glibc "-nan", Apple "nan"). Match + // that shape. NaN never compares, so its sign must come from signbit -- arbitrary + // ABI float bit patterns reach here and negative NaN must round-trip byte-identically. w.value_string(std::isnan(d) ? (std::signbit(d) ? "-nan" : "nan") : (std::signbit(d) ? "-inf" : "inf")); return; diff --git a/libraries/libfc/src/variant.cpp b/libraries/libfc/src/variant.cpp index 622e06f7f7..ce2b25af10 100644 --- a/libraries/libfc/src/variant.cpp +++ b/libraries/libfc/src/variant.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -786,6 +787,15 @@ bool variant::as_bool()const static std::string s_fc_to_string(double d) { + // Non-finite spellings are formatted explicitly rather than left to the C library the + // stringstream delegates to: glibc prints "-nan" for a negative NaN while Apple's libc + // drops the sign and prints "nan", so platform formatting would make the emitted bytes + // differ across platforms and diverge from the streaming path (fc/reflect/json_stream.hpp), + // which pins these exact shapes. NaN never compares, so its sign must come from signbit. + if( std::isnan( d ) ) + return std::signbit( d ) ? "-nan" : "nan"; + if( std::isinf( d ) ) + return std::signbit( d ) ? "-inf" : "inf"; // +2 is required to ensure that the double is rounded correctly when read back in. http://docs.oracle.com/cd/E19957-01/806-3568/ncg_goldberg.html std::stringstream ss; ss << std::setprecision(std::numeric_limits::digits10 + 2) << std::fixed << d; diff --git a/libraries/libfc/test/io/test_json_stream.cpp b/libraries/libfc/test/io/test_json_stream.cpp index 89c22068e4..959ebe9934 100644 --- a/libraries/libfc/test/io/test_json_stream.cpp +++ b/libraries/libfc/test/io/test_json_stream.cpp @@ -558,10 +558,11 @@ BOOST_AUTO_TEST_CASE(streaming_vs_variant_parity_libfc_leaf_types) { check_streaming_matches_variant(double{1e10}, "double 1e10"); check_streaming_matches_variant(double{0.0}, "double 0.0"); check_streaming_matches_variant(float{3.5f}, "float 3.5"); - // Non-finite doubles: the legacy path (stringstream << std::fixed, delegating to the - // C library) prints "nan" / "-nan" / "inf" / "-inf". NaN never compares, so its sign - // only reaches the output via signbit -- and arbitrary ABI float bit patterns can - // carry either sign. + // Non-finite doubles: the variant path (s_fc_to_string) formats these explicitly as + // "nan" / "-nan" / "inf" / "-inf", platform-independent -- unlike the C library + // formatting it bypasses, which drops negative NaN's sign on Apple. NaN never + // compares, so its sign only reaches the output via signbit -- and arbitrary ABI + // float bit patterns can carry either sign. const double quiet_nan = std::numeric_limits::quiet_NaN(); const double inf = std::numeric_limits::infinity(); check_streaming_matches_variant(quiet_nan, "double nan"); @@ -570,6 +571,11 @@ BOOST_AUTO_TEST_CASE(streaming_vs_variant_parity_libfc_leaf_types) { check_streaming_matches_variant(-inf, "double -inf"); check_streaming_matches_variant(std::copysign(std::numeric_limits::quiet_NaN(), -1.0f), "float -nan"); + // Absolute pins on top of the parity checks: both paths must produce this exact + // spelling on every platform, not merely agree with each other. + BOOST_CHECK_EQUAL(fc::to_json_string(std::copysign(quiet_nan, -1.0)), "\"-nan\""); + BOOST_CHECK_EQUAL(fc::json::to_string(fc::variant(std::copysign(quiet_nan, -1.0)), + fc::json::yield_function_t()), "\"-nan\""); } // bls public_key + signature: derived from a deterministic seed via private_key::generate(). // private_key itself has =delete'd to_json_stream so we don't include it here. diff --git a/unittests/abi_tests.cpp b/unittests/abi_tests.cpp index 6e83d5f3ae..663e5b7b47 100644 --- a/unittests/abi_tests.cpp +++ b/unittests/abi_tests.cpp @@ -823,11 +823,12 @@ BOOST_AUTO_TEST_CASE(large_int_quoting_boundaries) } FC_LOG_AND_RETHROW() } // Non-finite float32/float64 bit patterns through the ABI paths: the variant path -// (stringstream << std::fixed, delegating to the C library) prints "nan" / "-nan" / -// "inf" / "-inf", and binary_to_json_stream must agree byte-for-byte. NaN never -// compares, so its sign only survives via signbit -- and since no JSON input can -// produce a NaN, the packed bytes are built directly from raw bit patterns, exactly -// as an arbitrary on-chain payload could. +// (s_fc_to_string) formats non-finite doubles explicitly as "nan" / "-nan" / "inf" / +// "-inf" -- platform-independent, unlike the C library formatting it bypasses (glibc +// prints "-nan" where Apple prints "nan") -- and binary_to_json_stream must agree +// byte-for-byte. NaN never compares, so its sign only survives via signbit -- and +// since no JSON input can produce a NaN, the packed bytes are built directly from raw +// bit patterns, exactly as an arbitrary on-chain payload could. BOOST_AUTO_TEST_CASE(non_finite_float_sign_parity) { try {