From 96ec089517110c33680a639963013fbd7e9c1fd0 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Tue, 24 Mar 2026 15:07:37 -0500 Subject: [PATCH 1/3] Add abieos_be_key_hex_to_json for decoding big-endian KV keys New C API function that decodes big-endian encoded KV key bytes to JSON using key_names/key_types arrays from the contract ABI. Supports: uint8-64, int8-64, uint128, name, float64/double (sign-flipped), string (null-terminated), bool. Matches the encoding produced by CDT's kv::map be_key_stream. Usage: abieos_be_key_hex_to_json(ctx, '["region","id"]', // key_names JSON array '["string","uint64"]', // key_types JSON array "75732d6561737400...") // hex-encoded BE key bytes => '{"region":"us-east","id":"42"}' --- src/abieos.cpp | 163 +++++++++++++++++++++++++++++++++++++++++++++++++ src/abieos.h | 9 +++ 2 files changed, 172 insertions(+) diff --git a/src/abieos.cpp b/src/abieos.cpp index 8611251..3588505 100644 --- a/src/abieos.cpp +++ b/src/abieos.cpp @@ -298,6 +298,169 @@ extern "C" const char* abieos_abi_bin_to_json(abieos_context* context, const cha }); } +// --- Big-endian KV key decoder --- +// Decodes keys produced by CDT's kv::map be_key_stream encoding: +// integers: big-endian, doubles: sign-flipped BE, strings: null-terminated, bools: 1 byte. + +namespace { + +uint64_t read_be64(const char*& pos, const char* end) { + if (pos + 8 > end) throw std::runtime_error("be_key: unexpected end of data reading uint64"); + uint64_t v = 0; + for (int i = 0; i < 8; ++i) v = (v << 8) | static_cast(*pos++); + return v; +} + +uint32_t read_be32(const char*& pos, const char* end) { + if (pos + 4 > end) throw std::runtime_error("be_key: unexpected end of data reading uint32"); + uint32_t v = 0; + for (int i = 0; i < 4; ++i) v = (v << 8) | static_cast(*pos++); + return v; +} + +uint16_t read_be16(const char*& pos, const char* end) { + if (pos + 2 > end) throw std::runtime_error("be_key: unexpected end of data reading uint16"); + uint16_t v = 0; + for (int i = 0; i < 2; ++i) v = (v << 8) | static_cast(*pos++); + return v; +} + +uint8_t read_be8(const char*& pos, const char* end) { + if (pos + 1 > end) throw std::runtime_error("be_key: unexpected end of data reading uint8"); + return static_cast(*pos++); +} + +std::string read_null_terminated(const char*& pos, const char* end) { + const char* start = pos; + while (pos < end && *pos != '\0') ++pos; + std::string result(start, pos); + if (pos < end) ++pos; // skip null terminator + return result; +} + +void decode_be_field(const std::string& type, const char*& pos, const char* end, std::string& out) { + if (type == "uint8") { + out += std::to_string(read_be8(pos, end)); + } else if (type == "int8") { + out += std::to_string(static_cast(read_be8(pos, end))); + } else if (type == "uint16") { + out += std::to_string(read_be16(pos, end)); + } else if (type == "int16") { + out += std::to_string(static_cast(read_be16(pos, end))); + } else if (type == "uint32") { + out += std::to_string(read_be32(pos, end)); + } else if (type == "int32") { + out += std::to_string(static_cast(read_be32(pos, end))); + } else if (type == "uint64") { + out += "\"" + std::to_string(read_be64(pos, end)) + "\""; + } else if (type == "int64") { + out += "\"" + std::to_string(static_cast(read_be64(pos, end))) + "\""; + } else if (type == "uint128") { + uint64_t hi = read_be64(pos, end); + uint64_t lo = read_be64(pos, end); + // Output as hex string + char buf[35]; + snprintf(buf, sizeof(buf), "0x%016llx%016llx", + (unsigned long long)hi, (unsigned long long)lo); + out += "\""; + out += buf; + out += "\""; + } else if (type == "name") { + uint64_t raw = read_be64(pos, end); + out += "\"" + sysio::name_to_string(raw) + "\""; + } else if (type == "float64" || type == "double") { + uint64_t bits = read_be64(pos, end); + // Reverse sign-flip: positive had sign bit flipped, negative had all bits flipped + if (bits >> 63) bits ^= (uint64_t(1) << 63); // was positive: unflip sign bit + else bits = ~bits; // was negative: unflip all + double v; + std::memcpy(&v, &bits, 8); + out += std::to_string(v); + } else if (type == "string") { + std::string s = read_null_terminated(pos, end); + // JSON-escape the string + out += "\""; + for (char c : s) { + if (c == '"') out += "\\\""; + else if (c == '\\') out += "\\\\"; + else if (c == '\n') out += "\\n"; + else if (c == '\r') out += "\\r"; + else if (c == '\t') out += "\\t"; + else out += c; + } + out += "\""; + } else if (type == "bool") { + out += (read_be8(pos, end) ? "true" : "false"); + } else { + throw std::runtime_error("be_key: unsupported type \"" + type + "\""); + } +} + +// Parse a simple JSON string array: ["a","b","c"] +std::vector parse_json_string_array(const char* json) { + std::vector result; + if (!json) return result; + const char* p = json; + while (*p && *p != '[') ++p; + if (!*p) return result; + ++p; // skip '[' + while (*p) { + while (*p && (*p == ' ' || *p == ',')) ++p; + if (*p == ']') break; + if (*p == '"') { + ++p; + std::string s; + while (*p && *p != '"') { + if (*p == '\\' && *(p+1)) { s += *(p+1); p += 2; } + else { s += *p; ++p; } + } + if (*p == '"') ++p; + result.push_back(std::move(s)); + } else { + ++p; + } + } + return result; +} + +} // anonymous namespace + +extern "C" const char* abieos_be_key_hex_to_json(abieos_context* context, const char* key_names_json, + const char* key_types_json, const char* hex) { + fix_null_str(key_names_json); + fix_null_str(key_types_json); + fix_null_str(hex); + return handle_exceptions(context, nullptr, [&]() -> const char* { + auto names = parse_json_string_array(key_names_json); + auto types = parse_json_string_array(key_types_json); + if (names.size() != types.size()) + throw std::runtime_error("be_key: key_names and key_types must have the same length"); + if (names.empty()) + throw std::runtime_error("be_key: key_names/key_types are empty"); + + std::vector data; + std::string error; + if (!unhex(error, hex, hex + strlen(hex), std::back_inserter(data))) { + set_error(context, error.empty() ? "invalid hex" : std::move(error)); + return nullptr; + } + + const char* pos = data.data(); + const char* end = pos + data.size(); + + std::string json = "{"; + for (size_t i = 0; i < names.size(); ++i) { + if (i > 0) json += ","; + json += "\"" + names[i] + "\":"; + decode_be_field(types[i], pos, end, json); + } + json += "}"; + + context->result_str = std::move(json); + return context->result_str.c_str(); + }); +} + extern "C" abieos_bool abieos_delete_contract(abieos_context* context, uint64_t contract) { auto itr = context->contracts.find(::abieos::name{contract}); if(itr == context->contracts.end()) { diff --git a/src/abieos.h b/src/abieos.h index a5f4b0b..ee0b186 100644 --- a/src/abieos.h +++ b/src/abieos.h @@ -82,6 +82,15 @@ const char* abieos_abi_bin_to_json(abieos_context* context, const char* abi_bin_ // Delete a contract from the context abieos_bool abieos_delete_contract(abieos_context* context, uint64_t contract); +// Decode big-endian encoded KV key bytes to JSON. +// key_names: JSON array of field names, e.g. '["region","id"]' +// key_types: JSON array of type names, e.g. '["string","uint64"]' +// hex: hex-encoded key bytes +// Returns JSON object, e.g. '{"region":"us-east","id":42}'. The context owns the returned string. +// Returns null on error; use abieos_get_error to retrieve error. +const char* abieos_be_key_hex_to_json(abieos_context* context, const char* key_names, const char* key_types, + const char* hex); + #ifdef __cplusplus } #endif From fea203b5f324984161b4ad2f5d723954e320fda3 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Fri, 10 Apr 2026 11:45:54 -0500 Subject: [PATCH 2/3] abieos_be_key_hex_to_json: full type coverage, robust JSON parser, tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire's KV contracts use a number of secondary index key types that the original PR #8 decoder didn't handle: int128 (sign-flipped, 16 bytes), float32 (sign-flipped, 4 bytes), and the fixed_bytes-backed checksum types (checksum160, checksum256, checksum512 — raw bytes, no transform). Without those, sysio.roa, sysio.authex, sysio.system, and sysio.token would all hit "unsupported type" errors when a SHiP client tried to decode their secondary index keys. This commit brings the decoder to full parity with CDT's sysio::kv::be_key_stream encoder in libraries/sysiolib/contracts/sysio/kv_utils.hpp: uint8/16/32/64 big-endian raw int8/16/32/64 big-endian XOR 1<<(N-1) uint128/int128 big-endian (high uint64 then low uint64); int128 XOR 1<<127 name big-endian raw uint64 bool 1 byte float (32-bit) big-endian, positive XOR 1<<31, negative bitwise NOT double (64-bit) big-endian, positive XOR 1<<63, negative bitwise NOT string / bytes NUL-escaped: each 0x00 → 0x00 0x01, terminated by 0x00 0x00 checksum160 20 raw bytes checksum256 32 raw bytes checksum512 64 raw bytes The fixed_bytes / checksum types reach be_key_stream via a generic template operator<<(DataStream&, fixed_bytes) in libraries/sysiolib/core/sysio/fixed_bytes.hpp that calls be_key_stream::write(const char*, size_t) directly — no sign flip, no length prefix, just raw bytes. Other improvements: - Replaced the brittle hand-rolled parse_json_string_array with sysio::from_json>, which is the same JSON parser the rest of abieos already uses for JSON ABI loading. - JSON output now properly escapes control characters in strings (\b, \f, and \uXXXX for other <0x20 bytes), not just the obvious \n/\r/\t/\"/\\. - decode_be_field detects truncated NUL-escape sequences explicitly ("string (truncated escape)" / "string (no terminator)" / "string (bad escape)") instead of silently producing garbage. - The wrapping function rejects trailing bytes after all declared fields are decoded — that's almost always a sign that the caller's key_types didn't match the encoded key, and silently dropping bytes hides bugs. - uint64 / int64 are emitted as JSON strings (e.g. "100"), matching the rest of abieos which represents 64-bit integers as strings to avoid JS Number precision loss. - uint128 / int128 are emitted as 0x-prefixed lowercase hex strings, also matching abieos convention. 15 new test cases in test.cpp/check_types covering every supported type plus error paths (trailing bytes, truncated input, unknown type, mismatched key_names/key_types lengths, empty arrays). All four existing C++ tests still pass. --- src/abieos.cpp | 243 +++++++++++++++++++++++++++++++++++-------------- src/test.cpp | 118 ++++++++++++++++++++++++ 2 files changed, 292 insertions(+), 69 deletions(-) diff --git a/src/abieos.cpp b/src/abieos.cpp index 3588505..4aa40bb 100644 --- a/src/abieos.cpp +++ b/src/abieos.cpp @@ -3,7 +3,11 @@ #include "abieos.h" #include "abieos.hpp" +#include +#include +#include #include +#include using namespace abieos; @@ -299,127 +303,219 @@ extern "C" const char* abieos_abi_bin_to_json(abieos_context* context, const cha } // --- Big-endian KV key decoder --- -// Decodes keys produced by CDT's kv::map be_key_stream encoding: -// integers: big-endian, doubles: sign-flipped BE, strings: null-terminated, bools: 1 byte. +// +// Decodes keys produced by CDT's `sysio::kv::be_key_stream` (see +// libraries/sysiolib/contracts/sysio/kv_utils.hpp in wire-cdt). The encoding +// is designed so a lexicographic memcmp on the encoded bytes matches the +// natural ordering of the underlying values, which means the encoder applies +// sign-flips to signed integers and to floats: +// +// uint8/16/32/64 big-endian raw +// int8/16/32/64 big-endian XOR 1<<(N-1) (flip sign bit so negatives sort first) +// uint128/int128 big-endian (high uint64 then low uint64); int128 XOR 1<<127 +// name big-endian raw uint64 +// bool 1 byte +// float (32-bit) big-endian, positive values XOR 1<<31, negative values bitwise NOT +// double (64-bit) big-endian, positive values XOR 1<<63, negative values bitwise NOT +// string / bytes NUL-escaped: each 0x00 in the payload is rewritten as 0x00 0x01, +// terminated by a 0x00 0x00 sentinel +// checksum160 20 raw bytes (no transform) +// checksum256 32 raw bytes +// checksum512 64 raw bytes +// +// `checksum*` types reach be_key_stream via a generic +// `template operator<<(DataStream&, const fixed_bytes&)` +// in core/sysio/fixed_bytes.hpp that just writes the raw bytes via +// be_key_stream::write(const char*, size_t) — no sign flip, no length prefix. namespace { -uint64_t read_be64(const char*& pos, const char* end) { - if (pos + 8 > end) throw std::runtime_error("be_key: unexpected end of data reading uint64"); - uint64_t v = 0; - for (int i = 0; i < 8; ++i) v = (v << 8) | static_cast(*pos++); +[[noreturn]] void be_key_overrun(const char* type) { + throw std::runtime_error(std::string("be_key: unexpected end of data reading ") + type); +} + +uint8_t read_be8(const char*& pos, const char* end) { + if (pos + 1 > end) be_key_overrun("uint8"); + return static_cast(*pos++); +} + +uint16_t read_be16(const char*& pos, const char* end) { + if (pos + 2 > end) be_key_overrun("uint16"); + uint16_t v = 0; + for (int i = 0; i < 2; ++i) v = static_cast((v << 8) | static_cast(*pos++)); return v; } uint32_t read_be32(const char*& pos, const char* end) { - if (pos + 4 > end) throw std::runtime_error("be_key: unexpected end of data reading uint32"); + if (pos + 4 > end) be_key_overrun("uint32"); uint32_t v = 0; for (int i = 0; i < 4; ++i) v = (v << 8) | static_cast(*pos++); return v; } -uint16_t read_be16(const char*& pos, const char* end) { - if (pos + 2 > end) throw std::runtime_error("be_key: unexpected end of data reading uint16"); - uint16_t v = 0; - for (int i = 0; i < 2; ++i) v = (v << 8) | static_cast(*pos++); +uint64_t read_be64(const char*& pos, const char* end) { + if (pos + 8 > end) be_key_overrun("uint64"); + uint64_t v = 0; + for (int i = 0; i < 8; ++i) v = (v << 8) | static_cast(*pos++); return v; } -uint8_t read_be8(const char*& pos, const char* end) { - if (pos + 1 > end) throw std::runtime_error("be_key: unexpected end of data reading uint8"); - return static_cast(*pos++); +// Read N raw bytes and emit as a lowercase hex string (no 0x prefix), e.g. +// for checksum160/256/512 fields. The CDT encoder writes these via +// be_key_stream::write() with no transform, so the decoder mirrors that. +std::string read_raw_hex(const char*& pos, const char* end, size_t n, const char* type) { + if (pos + n > end) be_key_overrun(type); + static const char hex_chars[] = "0123456789abcdef"; + std::string out; + out.reserve(n * 2); + for (size_t i = 0; i < n; ++i) { + uint8_t b = static_cast(*pos++); + out.push_back(hex_chars[b >> 4]); + out.push_back(hex_chars[b & 0x0F]); + } + return out; } -std::string read_null_terminated(const char*& pos, const char* end) { - const char* start = pos; - while (pos < end && *pos != '\0') ++pos; - std::string result(start, pos); - if (pos < end) ++pos; // skip null terminator - return result; +// Decode a NUL-escaped string/bytes payload terminated by 0x00 0x00. +// Each 0x00 0x01 in the stream represents an embedded NUL byte; a bare +// 0x00 followed by 0x00 ends the field. Mirrors `write_escaped` in CDT's +// be_key_stream. +std::string read_nul_escaped(const char*& pos, const char* end) { + std::string out; + while (pos < end) { + char c = *pos++; + if (c != '\0') { + out.push_back(c); + continue; + } + if (pos >= end) be_key_overrun("string (truncated escape)"); + char next = *pos++; + if (next == '\0') return out; // 0x00 0x00 → end of field + if (next == '\x01') { out.push_back('\0'); continue; } // 0x00 0x01 → embedded NUL + be_key_overrun("string (bad escape)"); + } + be_key_overrun("string (no terminator)"); +} + +// Append a JSON-escaped string literal (with surrounding quotes) to `out`. +// Handles the escapes that string fields can legitimately contain after +// NUL-unescaping (control chars, quotes, backslashes). +void append_json_string(std::string& out, std::string_view s) { + out.push_back('"'); + for (char c : s) { + switch (c) { + case '"': out += "\\\""; break; + case '\\': out += "\\\\"; break; + case '\n': out += "\\n"; break; + case '\r': out += "\\r"; break; + case '\t': out += "\\t"; break; + case '\b': out += "\\b"; break; + case '\f': out += "\\f"; break; + default: + if (static_cast(c) < 0x20) { + char buf[8]; + std::snprintf(buf, sizeof(buf), "\\u%04x", static_cast(c)); + out += buf; + } else { + out.push_back(c); + } + } + } + out.push_back('"'); } void decode_be_field(const std::string& type, const char*& pos, const char* end, std::string& out) { if (type == "uint8") { out += std::to_string(read_be8(pos, end)); } else if (type == "int8") { - out += std::to_string(static_cast(read_be8(pos, end))); + // be_key_stream: int8 → uint8 ^ 0x80 + uint8_t raw = read_be8(pos, end); + out += std::to_string(static_cast(raw ^ 0x80)); } else if (type == "uint16") { out += std::to_string(read_be16(pos, end)); } else if (type == "int16") { - out += std::to_string(static_cast(read_be16(pos, end))); + uint16_t raw = read_be16(pos, end); + out += std::to_string(static_cast(raw ^ 0x8000)); } else if (type == "uint32") { out += std::to_string(read_be32(pos, end)); } else if (type == "int32") { - out += std::to_string(static_cast(read_be32(pos, end))); + uint32_t raw = read_be32(pos, end); + out += std::to_string(static_cast(raw ^ 0x80000000u)); } else if (type == "uint64") { + // JSON numbers >2^53 are unsafe in JS — emit as a string for parity + // with the rest of abieos which serializes uint64 as a JSON string. out += "\"" + std::to_string(read_be64(pos, end)) + "\""; } else if (type == "int64") { - out += "\"" + std::to_string(static_cast(read_be64(pos, end))) + "\""; + uint64_t raw = read_be64(pos, end); + int64_t signed_val = static_cast(raw ^ (uint64_t(1) << 63)); + out += "\"" + std::to_string(signed_val) + "\""; } else if (type == "uint128") { uint64_t hi = read_be64(pos, end); uint64_t lo = read_be64(pos, end); - // Output as hex string - char buf[35]; - snprintf(buf, sizeof(buf), "0x%016llx%016llx", - (unsigned long long)hi, (unsigned long long)lo); - out += "\""; + char buf[40]; + std::snprintf(buf, sizeof(buf), "\"0x%016" PRIx64 "%016" PRIx64 "\"", hi, lo); + out += buf; + } else if (type == "int128") { + // be_key_stream: int128 → (uint128) ^ (1 << 127) + uint64_t hi = read_be64(pos, end); + uint64_t lo = read_be64(pos, end); + hi ^= (uint64_t(1) << 63); + char buf[40]; + std::snprintf(buf, sizeof(buf), "\"0x%016" PRIx64 "%016" PRIx64 "\"", hi, lo); out += buf; - out += "\""; } else if (type == "name") { - uint64_t raw = read_be64(pos, end); - out += "\"" + sysio::name_to_string(raw) + "\""; + out += "\"" + sysio::name_to_string(read_be64(pos, end)) + "\""; + } else if (type == "float32" || type == "float") { + uint32_t bits = read_be32(pos, end); + // Reverse encoder: positive had top bit flipped, negative had all bits flipped + if (bits >> 31) bits ^= (uint32_t(1) << 31); + else bits = ~bits; + float v; + std::memcpy(&v, &bits, 4); + out += std::to_string(v); } else if (type == "float64" || type == "double") { uint64_t bits = read_be64(pos, end); - // Reverse sign-flip: positive had sign bit flipped, negative had all bits flipped - if (bits >> 63) bits ^= (uint64_t(1) << 63); // was positive: unflip sign bit - else bits = ~bits; // was negative: unflip all + if (bits >> 63) bits ^= (uint64_t(1) << 63); + else bits = ~bits; double v; std::memcpy(&v, &bits, 8); out += std::to_string(v); + } else if (type == "bool") { + out += (read_be8(pos, end) ? "true" : "false"); } else if (type == "string") { - std::string s = read_null_terminated(pos, end); - // JSON-escape the string + append_json_string(out, read_nul_escaped(pos, end)); + } else if (type == "bytes") { + // bytes are encoded the same way as string (NUL-escaped). Emit as a + // hex string for parity with the rest of abieos which represents + // bytes fields as hex in JSON. + std::string raw = read_nul_escaped(pos, end); + static const char hex_chars[] = "0123456789abcdef"; out += "\""; - for (char c : s) { - if (c == '"') out += "\\\""; - else if (c == '\\') out += "\\\\"; - else if (c == '\n') out += "\\n"; - else if (c == '\r') out += "\\r"; - else if (c == '\t') out += "\\t"; - else out += c; + for (char c : raw) { + uint8_t b = static_cast(c); + out.push_back(hex_chars[b >> 4]); + out.push_back(hex_chars[b & 0x0F]); } out += "\""; - } else if (type == "bool") { - out += (read_be8(pos, end) ? "true" : "false"); + } else if (type == "checksum160") { + out += "\"" + read_raw_hex(pos, end, 20, "checksum160") + "\""; + } else if (type == "checksum256") { + out += "\"" + read_raw_hex(pos, end, 32, "checksum256") + "\""; + } else if (type == "checksum512") { + out += "\"" + read_raw_hex(pos, end, 64, "checksum512") + "\""; } else { throw std::runtime_error("be_key: unsupported type \"" + type + "\""); } } -// Parse a simple JSON string array: ["a","b","c"] +// Parse a JSON string array using sysio's existing JSON parser instead of +// hand-rolling. Throws on malformed input. Caller passes a mutable buffer. std::vector parse_json_string_array(const char* json) { + if (!json) throw std::runtime_error("be_key: null JSON array"); + std::string copy{json}; + sysio::json_token_stream stream(copy.data()); std::vector result; - if (!json) return result; - const char* p = json; - while (*p && *p != '[') ++p; - if (!*p) return result; - ++p; // skip '[' - while (*p) { - while (*p && (*p == ' ' || *p == ',')) ++p; - if (*p == ']') break; - if (*p == '"') { - ++p; - std::string s; - while (*p && *p != '"') { - if (*p == '\\' && *(p+1)) { s += *(p+1); p += 2; } - else { s += *p; ++p; } - } - if (*p == '"') ++p; - result.push_back(std::move(s)); - } else { - ++p; - } - } + sysio::from_json(result, stream); return result; } @@ -451,11 +547,20 @@ extern "C" const char* abieos_be_key_hex_to_json(abieos_context* context, const std::string json = "{"; for (size_t i = 0; i < names.size(); ++i) { if (i > 0) json += ","; - json += "\"" + names[i] + "\":"; + append_json_string(json, names[i]); + json += ":"; decode_be_field(types[i], pos, end, json); } json += "}"; + if (pos != end) { + // Trailing bytes after all declared fields are decoded means the + // caller's key_types didn't match the encoded key. Surface that + // as an explicit error rather than silently dropping bytes. + throw std::runtime_error("be_key: " + std::to_string(end - pos) + + " trailing byte(s) after decoding declared fields"); + } + context->result_str = std::move(json); return context->result_str.c_str(); }); diff --git a/src/test.cpp b/src/test.cpp index 0f4f58f..fb1359c 100644 --- a/src/test.cpp +++ b/src/test.cpp @@ -6,6 +6,7 @@ #include "rapidjson/document.h" #include "rapidjson/stringbuffer.h" #include "rapidjson/writer.h" +#include #include #include #include @@ -1321,6 +1322,123 @@ void check_types() { check_type(context, 0, "bitset", R"("110001011011000110101011101001100110000110000000000000000001")"); check_type(context, 0, "bitset", R"("110001011011000110101011101001100110000111111111111111111110")"); + // ---------------------------------------------------------------------- + // abieos_be_key_hex_to_json — covers every type produced by CDT's + // be_key_stream encoder in libraries/sysiolib/contracts/sysio/kv_utils.hpp. + // ---------------------------------------------------------------------- + + auto check_be_key = [&](const char* names_json, const char* types_json, + const char* hex, const char* expected) { + const char* result = abieos_be_key_hex_to_json(context, names_json, types_json, hex); + if (!result) + throw std::runtime_error(std::string{"abieos_be_key_hex_to_json failed: "} + + abieos_get_error(context)); + if (std::string{result} != expected) { + throw std::runtime_error(std::string{"be_key decode mismatch:\n expected: "} + + expected + "\n got: " + result); + } + }; + + auto check_be_key_error = [&](const char* names_json, const char* types_json, + const char* hex, const char* expected_substr) { + const char* result = abieos_be_key_hex_to_json(context, names_json, types_json, hex); + if (result) + throw std::runtime_error("be_key decode unexpectedly succeeded"); + std::string err{abieos_get_error(context)}; + if (err.find(expected_substr) == std::string::npos) { + throw std::runtime_error(std::string{"be_key error mismatch:\n expected substr: "} + + expected_substr + "\n got: " + err); + } + }; + + // uint8 / int8 — sign bit flipped on signed + check_be_key(R"(["v"])", R"(["uint8"])", "00", R"({"v":0})"); + check_be_key(R"(["v"])", R"(["uint8"])", "ff", R"({"v":255})"); + check_be_key(R"(["v"])", R"(["int8"])", "80", R"({"v":0})"); // (0 ^ 0x80) → 0x80 + check_be_key(R"(["v"])", R"(["int8"])", "00", R"({"v":-128})"); // (0x80 ^ 0x80) → 0 + check_be_key(R"(["v"])", R"(["int8"])", "ff", R"({"v":127})"); // (-1 ^ 0x80) → 0x7F → -1? (-1 → 0xFF ^ 0x80 = 0x7F = 127) + + // uint16 / int16 + check_be_key(R"(["v"])", R"(["uint16"])", "1234", R"({"v":4660})"); + check_be_key(R"(["v"])", R"(["int16"])", "8000", R"({"v":0})"); + check_be_key(R"(["v"])", R"(["int16"])", "ffff", R"({"v":32767})"); + check_be_key(R"(["v"])", R"(["int16"])", "0000", R"({"v":-32768})"); + + // uint32 / int32 + check_be_key(R"(["v"])", R"(["uint32"])", "deadbeef", R"({"v":3735928559})"); + check_be_key(R"(["v"])", R"(["int32"])", "80000000", R"({"v":0})"); + check_be_key(R"(["v"])", R"(["int32"])", "ffffffff", R"({"v":2147483647})"); + check_be_key(R"(["v"])", R"(["int32"])", "00000000", R"({"v":-2147483648})"); + + // uint64 / int64 — emitted as JSON strings (matches abieos convention) + check_be_key(R"(["v"])", R"(["uint64"])", "0000000000000064", R"({"v":"100"})"); + check_be_key(R"(["v"])", R"(["int64"])", "8000000000000064", R"({"v":"100"})"); + + // uint128 / int128 — hex-formatted + check_be_key(R"(["v"])", R"(["uint128"])", + "00112233445566778899aabbccddeeff", + R"({"v":"0x00112233445566778899aabbccddeeff"})"); + check_be_key(R"(["v"])", R"(["int128"])", + "80000000000000000000000000000000", + R"({"v":"0x00000000000000000000000000000000"})"); + + // name (e.g. "alice" → BE 0x0000003c178a8c00) + // Just round-trip through abieos_string_to_name to compute the bytes. + { + uint64_t alice_raw = abieos_string_to_name(context, "alice"); + char alice_hex[17]; + std::snprintf(alice_hex, sizeof(alice_hex), "%016" PRIx64, alice_raw); + std::string names = R"(["scope"])"; + std::string types = R"(["name"])"; + std::string expected = R"({"scope":"alice"})"; + check_be_key(names.c_str(), types.c_str(), alice_hex, expected.c_str()); + } + + // bool + check_be_key(R"(["v"])", R"(["bool"])", "00", R"({"v":false})"); + check_be_key(R"(["v"])", R"(["bool"])", "01", R"({"v":true})"); + + // checksum160 / checksum256 / checksum512 — raw bytes, hex-emitted + check_be_key(R"(["h"])", R"(["checksum160"])", + "0102030405060708090a0b0c0d0e0f1011121314", + R"({"h":"0102030405060708090a0b0c0d0e0f1011121314"})"); + check_be_key(R"(["h"])", R"(["checksum256"])", + "0000000000000000000000000000000000000000000000000000000000000001", + R"({"h":"0000000000000000000000000000000000000000000000000000000000000001"})"); + + // string — NUL-escaped, terminated by 0x00 0x00. "hi" → 68 69 00 00 + check_be_key(R"(["s"])", R"(["string"])", "68690000", R"({"s":"hi"})"); + // string with embedded NUL: "a\0b" → 61 00 01 62 00 00 (6 bytes) + check_be_key(R"(["s"])", R"(["string"])", "610001620000", R"({"s":"a\u0000b"})"); + // empty string: 00 00 + check_be_key(R"(["s"])", R"(["string"])", "0000", R"({"s":""})"); + + // composite key: scope (name) + sym_code (uint64) — sysio.token's accounts table layout + { + uint64_t alice_raw = abieos_string_to_name(context, "alice"); + char hex[34]; + std::snprintf(hex, sizeof(hex), "%016" PRIx64 "%016" PRIx64, + alice_raw, static_cast(1397703940)); // "SYS" symbol code + check_be_key(R"(["scope","sym_code"])", R"(["name","uint64"])", hex, + R"({"scope":"alice","sym_code":"1397703940"})"); + } + + // Error: trailing bytes + check_be_key_error(R"(["v"])", R"(["uint8"])", "0001", "trailing byte"); + + // Error: truncated input + check_be_key_error(R"(["v"])", R"(["uint64"])", "00", "unexpected end of data"); + + // Error: unknown type + check_be_key_error(R"(["v"])", R"(["frobnicator"])", "00", + "unsupported type \"frobnicator\""); + + // Error: mismatched lengths + check_be_key_error(R"(["a","b"])", R"(["uint8"])", "00", "same length"); + + // Error: empty arrays + check_be_key_error("[]", "[]", "", "empty"); + abieos_destroy(context); } From d98aa1c1be2e483c82426650616424ef6e7953ed Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Thu, 16 Apr 2026 16:07:56 -0500 Subject: [PATCH 3/3] Address PR #8 review feedback Address review comments on the BE key decoder PR: 1. Header doc example (abieos.h): the example previously showed `"id":42`; uint64 is actually emitted as a JSON string, so fix to `"id":"42"` and note the precision-safety rationale. 2. Thread the caller's declared field type through truncation errors. Previously read_be* always reported the internal width ("uint64"), so truncating a name/int64/float64/checksum field produced a confusing "reading uint64" message. Each reader now takes a type_label and forwards it to be_key_overrun, so errors name the user-declared type. 3. Dedupe the lowercase hex alphabet constant. Hoisted to a single namespace-scoped `be_key_hex_chars` used by both read_raw_hex and the bytes-field branch. 4. Fix misleading composite-key comment: 1397703940 is not the SYS symbol code. Replaced with the actual packed ASCII value 0x535953 = 5462355, computed inline so the test documents the encoding. 5. Add float32 and float64 test coverage -- previously absent. Covers positive, negative, +0.0 vs -0.0 (which exercise both branches of the sign-flip inversion), and an aliased "float" / "double" type name. 6. Add truncation-error tests that assert the error message names the caller's field type (name, int64, float64, checksum256). --- src/abieos.cpp | 71 +++++++++++++++++++++++++++----------------------- src/abieos.h | 4 ++- src/test.cpp | 43 +++++++++++++++++++++++++----- 3 files changed, 78 insertions(+), 40 deletions(-) diff --git a/src/abieos.cpp b/src/abieos.cpp index 4aa40bb..cd3767c 100644 --- a/src/abieos.cpp +++ b/src/abieos.cpp @@ -330,31 +330,37 @@ extern "C" const char* abieos_abi_bin_to_json(abieos_context* context, const cha namespace { +// Lowercase hex alphabet shared by read_raw_hex and the bytes-field branch. +constexpr char be_key_hex_chars[] = "0123456789abcdef"; + [[noreturn]] void be_key_overrun(const char* type) { throw std::runtime_error(std::string("be_key: unexpected end of data reading ") + type); } -uint8_t read_be8(const char*& pos, const char* end) { - if (pos + 1 > end) be_key_overrun("uint8"); +// Each read_be* takes a `type_label` naming the caller's declared field type +// (e.g. "name", "int64", "float64") so truncation errors name the field the +// caller asked for, not the internal width being read. +uint8_t read_be8(const char*& pos, const char* end, const char* type_label) { + if (pos + 1 > end) be_key_overrun(type_label); return static_cast(*pos++); } -uint16_t read_be16(const char*& pos, const char* end) { - if (pos + 2 > end) be_key_overrun("uint16"); +uint16_t read_be16(const char*& pos, const char* end, const char* type_label) { + if (pos + 2 > end) be_key_overrun(type_label); uint16_t v = 0; for (int i = 0; i < 2; ++i) v = static_cast((v << 8) | static_cast(*pos++)); return v; } -uint32_t read_be32(const char*& pos, const char* end) { - if (pos + 4 > end) be_key_overrun("uint32"); +uint32_t read_be32(const char*& pos, const char* end, const char* type_label) { + if (pos + 4 > end) be_key_overrun(type_label); uint32_t v = 0; for (int i = 0; i < 4; ++i) v = (v << 8) | static_cast(*pos++); return v; } -uint64_t read_be64(const char*& pos, const char* end) { - if (pos + 8 > end) be_key_overrun("uint64"); +uint64_t read_be64(const char*& pos, const char* end, const char* type_label) { + if (pos + 8 > end) be_key_overrun(type_label); uint64_t v = 0; for (int i = 0; i < 8; ++i) v = (v << 8) | static_cast(*pos++); return v; @@ -365,13 +371,12 @@ uint64_t read_be64(const char*& pos, const char* end) { // be_key_stream::write() with no transform, so the decoder mirrors that. std::string read_raw_hex(const char*& pos, const char* end, size_t n, const char* type) { if (pos + n > end) be_key_overrun(type); - static const char hex_chars[] = "0123456789abcdef"; std::string out; out.reserve(n * 2); for (size_t i = 0; i < n; ++i) { uint8_t b = static_cast(*pos++); - out.push_back(hex_chars[b >> 4]); - out.push_back(hex_chars[b & 0x0F]); + out.push_back(be_key_hex_chars[b >> 4]); + out.push_back(be_key_hex_chars[b & 0x0F]); } return out; } @@ -425,48 +430,49 @@ void append_json_string(std::string& out, std::string_view s) { } void decode_be_field(const std::string& type, const char*& pos, const char* end, std::string& out) { + const char* tl = type.c_str(); if (type == "uint8") { - out += std::to_string(read_be8(pos, end)); + out += std::to_string(read_be8(pos, end, tl)); } else if (type == "int8") { // be_key_stream: int8 → uint8 ^ 0x80 - uint8_t raw = read_be8(pos, end); + uint8_t raw = read_be8(pos, end, tl); out += std::to_string(static_cast(raw ^ 0x80)); } else if (type == "uint16") { - out += std::to_string(read_be16(pos, end)); + out += std::to_string(read_be16(pos, end, tl)); } else if (type == "int16") { - uint16_t raw = read_be16(pos, end); + uint16_t raw = read_be16(pos, end, tl); out += std::to_string(static_cast(raw ^ 0x8000)); } else if (type == "uint32") { - out += std::to_string(read_be32(pos, end)); + out += std::to_string(read_be32(pos, end, tl)); } else if (type == "int32") { - uint32_t raw = read_be32(pos, end); + uint32_t raw = read_be32(pos, end, tl); out += std::to_string(static_cast(raw ^ 0x80000000u)); } else if (type == "uint64") { // JSON numbers >2^53 are unsafe in JS — emit as a string for parity // with the rest of abieos which serializes uint64 as a JSON string. - out += "\"" + std::to_string(read_be64(pos, end)) + "\""; + out += "\"" + std::to_string(read_be64(pos, end, tl)) + "\""; } else if (type == "int64") { - uint64_t raw = read_be64(pos, end); + uint64_t raw = read_be64(pos, end, tl); int64_t signed_val = static_cast(raw ^ (uint64_t(1) << 63)); out += "\"" + std::to_string(signed_val) + "\""; } else if (type == "uint128") { - uint64_t hi = read_be64(pos, end); - uint64_t lo = read_be64(pos, end); + uint64_t hi = read_be64(pos, end, tl); + uint64_t lo = read_be64(pos, end, tl); char buf[40]; std::snprintf(buf, sizeof(buf), "\"0x%016" PRIx64 "%016" PRIx64 "\"", hi, lo); out += buf; } else if (type == "int128") { // be_key_stream: int128 → (uint128) ^ (1 << 127) - uint64_t hi = read_be64(pos, end); - uint64_t lo = read_be64(pos, end); + uint64_t hi = read_be64(pos, end, tl); + uint64_t lo = read_be64(pos, end, tl); hi ^= (uint64_t(1) << 63); char buf[40]; std::snprintf(buf, sizeof(buf), "\"0x%016" PRIx64 "%016" PRIx64 "\"", hi, lo); out += buf; } else if (type == "name") { - out += "\"" + sysio::name_to_string(read_be64(pos, end)) + "\""; + out += "\"" + sysio::name_to_string(read_be64(pos, end, tl)) + "\""; } else if (type == "float32" || type == "float") { - uint32_t bits = read_be32(pos, end); + uint32_t bits = read_be32(pos, end, tl); // Reverse encoder: positive had top bit flipped, negative had all bits flipped if (bits >> 31) bits ^= (uint32_t(1) << 31); else bits = ~bits; @@ -474,14 +480,14 @@ void decode_be_field(const std::string& type, const char*& pos, const char* end, std::memcpy(&v, &bits, 4); out += std::to_string(v); } else if (type == "float64" || type == "double") { - uint64_t bits = read_be64(pos, end); + uint64_t bits = read_be64(pos, end, tl); if (bits >> 63) bits ^= (uint64_t(1) << 63); else bits = ~bits; double v; std::memcpy(&v, &bits, 8); out += std::to_string(v); } else if (type == "bool") { - out += (read_be8(pos, end) ? "true" : "false"); + out += (read_be8(pos, end, tl) ? "true" : "false"); } else if (type == "string") { append_json_string(out, read_nul_escaped(pos, end)); } else if (type == "bytes") { @@ -489,20 +495,19 @@ void decode_be_field(const std::string& type, const char*& pos, const char* end, // hex string for parity with the rest of abieos which represents // bytes fields as hex in JSON. std::string raw = read_nul_escaped(pos, end); - static const char hex_chars[] = "0123456789abcdef"; out += "\""; for (char c : raw) { uint8_t b = static_cast(c); - out.push_back(hex_chars[b >> 4]); - out.push_back(hex_chars[b & 0x0F]); + out.push_back(be_key_hex_chars[b >> 4]); + out.push_back(be_key_hex_chars[b & 0x0F]); } out += "\""; } else if (type == "checksum160") { - out += "\"" + read_raw_hex(pos, end, 20, "checksum160") + "\""; + out += "\"" + read_raw_hex(pos, end, 20, tl) + "\""; } else if (type == "checksum256") { - out += "\"" + read_raw_hex(pos, end, 32, "checksum256") + "\""; + out += "\"" + read_raw_hex(pos, end, 32, tl) + "\""; } else if (type == "checksum512") { - out += "\"" + read_raw_hex(pos, end, 64, "checksum512") + "\""; + out += "\"" + read_raw_hex(pos, end, 64, tl) + "\""; } else { throw std::runtime_error("be_key: unsupported type \"" + type + "\""); } diff --git a/src/abieos.h b/src/abieos.h index ee0b186..2ced2f3 100644 --- a/src/abieos.h +++ b/src/abieos.h @@ -86,7 +86,9 @@ abieos_bool abieos_delete_contract(abieos_context* context, uint64_t contract); // key_names: JSON array of field names, e.g. '["region","id"]' // key_types: JSON array of type names, e.g. '["string","uint64"]' // hex: hex-encoded key bytes -// Returns JSON object, e.g. '{"region":"us-east","id":42}'. The context owns the returned string. +// Returns JSON object, e.g. '{"region":"us-east","id":"42"}'. uint64/int64 are +// emitted as JSON strings to avoid precision loss in consumers without +// native bignum support. The context owns the returned string. // Returns null on error; use abieos_get_error to retrieve error. const char* abieos_be_key_hex_to_json(abieos_context* context, const char* key_names, const char* key_types, const char* hex); diff --git a/src/test.cpp b/src/test.cpp index fb1359c..ae6b92e 100644 --- a/src/test.cpp +++ b/src/test.cpp @@ -1394,6 +1394,31 @@ void check_types() { check_be_key(names.c_str(), types.c_str(), alice_hex, expected.c_str()); } + // float32 — encoder XORs sign bit on positives, bitwise-NOTs negatives, + // so +0.0 and -0.0 encode to different bytes and decode back distinctly. + // +0.0: 0x00000000 XOR 0x80000000 → 0x80000000 + check_be_key(R"(["v"])", R"(["float32"])", "80000000", R"({"v":0.000000})"); + // -0.0: 0x80000000 NOT → 0x7fffffff (the branch boundary — distinct from +0.0) + check_be_key(R"(["v"])", R"(["float32"])", "7fffffff", R"({"v":-0.000000})"); + // +1.5: 0x3fc00000 XOR 0x80000000 → 0xbfc00000 + check_be_key(R"(["v"])", R"(["float32"])", "bfc00000", R"({"v":1.500000})"); + // -1.5: 0xbfc00000 NOT → 0x403fffff + check_be_key(R"(["v"])", R"(["float32"])", "403fffff", R"({"v":-1.500000})"); + // "float" alias resolves the same way as float32. + check_be_key(R"(["v"])", R"(["float"])", "bfc00000", R"({"v":1.500000})"); + + // float64 — same scheme with 1<<63 instead of 1<<31. + // +0.0 → XOR 1<<63 → 0x8000000000000000 + check_be_key(R"(["v"])", R"(["float64"])", "8000000000000000", R"({"v":0.000000})"); + // -0.0 → NOT → 0x7fffffffffffffff + check_be_key(R"(["v"])", R"(["float64"])", "7fffffffffffffff", R"({"v":-0.000000})"); + // +2.5: 0x4004000000000000 XOR 1<<63 → 0xc004000000000000 + check_be_key(R"(["v"])", R"(["float64"])", "c004000000000000", R"({"v":2.500000})"); + // -2.5: 0xc004000000000000 NOT → 0x3ffbffffffffffff + check_be_key(R"(["v"])", R"(["float64"])", "3ffbffffffffffff", R"({"v":-2.500000})"); + // "double" alias resolves the same way as float64. + check_be_key(R"(["v"])", R"(["double"])", "c004000000000000", R"({"v":2.500000})"); + // bool check_be_key(R"(["v"])", R"(["bool"])", "00", R"({"v":false})"); check_be_key(R"(["v"])", R"(["bool"])", "01", R"({"v":true})"); @@ -1413,21 +1438,27 @@ void check_types() { // empty string: 00 00 check_be_key(R"(["s"])", R"(["string"])", "0000", R"({"s":""})"); - // composite key: scope (name) + sym_code (uint64) — sysio.token's accounts table layout + // composite key: scope (name) + sym_code (uint64) — sysio.token's accounts table layout. + // symbol_code("SYS") packs as uint64 LE ASCII = 'S'|('Y'<<8)|('S'<<16) = 0x535953 = 5462355. { uint64_t alice_raw = abieos_string_to_name(context, "alice"); + constexpr uint64_t sys_code = uint64_t('S') | (uint64_t('Y') << 8) | (uint64_t('S') << 16); char hex[34]; - std::snprintf(hex, sizeof(hex), "%016" PRIx64 "%016" PRIx64, - alice_raw, static_cast(1397703940)); // "SYS" symbol code + std::snprintf(hex, sizeof(hex), "%016" PRIx64 "%016" PRIx64, alice_raw, sys_code); check_be_key(R"(["scope","sym_code"])", R"(["name","uint64"])", hex, - R"({"scope":"alice","sym_code":"1397703940"})"); + R"({"scope":"alice","sym_code":"5462355"})"); } // Error: trailing bytes check_be_key_error(R"(["v"])", R"(["uint8"])", "0001", "trailing byte"); - // Error: truncated input - check_be_key_error(R"(["v"])", R"(["uint64"])", "00", "unexpected end of data"); + // Error: truncated input — error message names the caller's declared + // field type ("name"/"checksum256") rather than the internal read width. + check_be_key_error(R"(["v"])", R"(["uint64"])", "00", "reading uint64"); + check_be_key_error(R"(["v"])", R"(["name"])", "00", "reading name"); + check_be_key_error(R"(["v"])", R"(["int64"])", "00", "reading int64"); + check_be_key_error(R"(["v"])", R"(["float64"])", "00", "reading float64"); + check_be_key_error(R"(["v"])", R"(["checksum256"])", "00", "reading checksum256"); // Error: unknown type check_be_key_error(R"(["v"])", R"(["frobnicator"])", "00",