From a5bbfb0dbd9e6288e48991c3f2b2dbb3c5c58a3e Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Thu, 20 Aug 2026 12:34:24 -0300 Subject: [PATCH 01/51] fix: export xl_write_typed from the Windows .def --- rust/excelreader/excelreader.def | 1 + src/ExcelReader.Native/include/excelreader.def | 1 + 2 files changed, 2 insertions(+) diff --git a/rust/excelreader/excelreader.def b/rust/excelreader/excelreader.def index 4140084..d0b5386 100644 --- a/rust/excelreader/excelreader.def +++ b/rust/excelreader/excelreader.def @@ -12,6 +12,7 @@ EXPORTS xl_is_date1904 xl_parse_typed xl_free_table + xl_write_typed xl_infer_schema xl_free_schema xl_last_error_ptr diff --git a/src/ExcelReader.Native/include/excelreader.def b/src/ExcelReader.Native/include/excelreader.def index 4140084..d0b5386 100644 --- a/src/ExcelReader.Native/include/excelreader.def +++ b/src/ExcelReader.Native/include/excelreader.def @@ -12,6 +12,7 @@ EXPORTS xl_is_date1904 xl_parse_typed xl_free_table + xl_write_typed xl_infer_schema xl_free_schema xl_last_error_ptr From 8e175ffa3fc8bac8c5ff4df667863b0dc2e1961b Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Thu, 20 Aug 2026 13:37:48 -0300 Subject: [PATCH 02/51] feat(cpp): add xl::write_columns, a zero-copy columnar write --- cpp/tests/CMakeLists.txt | 13 + cpp/tests/write.cpp | 188 ++++++++++ .../include/excelreader.hpp | 334 +++++++++++++++++- 3 files changed, 533 insertions(+), 2 deletions(-) create mode 100644 cpp/tests/write.cpp diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 17af2a9..056fcf9 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -15,3 +15,16 @@ if(WIN32) "$" "$") endif() + +add_executable(excelreader_cpp_write write.cpp) +target_link_libraries(excelreader_cpp_write PRIVATE xl::excelreader) + +add_test(NAME excelreader_cpp_write COMMAND excelreader_cpp_write) +if(WIN32) + # Same reason as the smoke test: the native DLL has no import-time PATH entry, so copy it next + # to the test executable for the loader to find. + add_custom_command(TARGET excelreader_cpp_write POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "$" + "$") +endif() \ No newline at end of file diff --git a/cpp/tests/write.cpp b/cpp/tests/write.cpp new file mode 100644 index 0000000..3c0149e --- /dev/null +++ b/cpp/tests/write.cpp @@ -0,0 +1,188 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#define CHECK(cond, msg) \ + do \ + { \ + if (!(cond)) \ + { \ + std::fprintf(stderr, "FAIL: %s (%s:%d)\n", msg, __FILE__, __LINE__); \ + return 1; \ + } \ + } while (0) + +struct WrittenRow +{ + std::string_view texto; + int64_t inteiro; + double numero; + std::chrono::year_month_day data; + std::chrono::microseconds hora; + std::chrono::system_clock::time_point instante; +}; + +template <> +struct xl::ExcelMapper +{ + static constexpr auto get_bindings() + { + return std::make_tuple( + xl::make_field("texto", &WrittenRow::texto), + xl::make_field("inteiro", &WrittenRow::inteiro), + xl::make_field("numero", &WrittenRow::numero), + xl::make_field("data", &WrittenRow::data), + xl::make_field("hora", &WrittenRow::hora), + xl::make_field("instante", &WrittenRow::instante)); + } +}; + +// A path under the system temp directory. Deleted by each test that creates it. +static std::filesystem::path temp_path(std::string_view name) +{ + return std::filesystem::temp_directory_path() / + std::filesystem::path(std::string("excelreader-cpp-") + std::string(name)); +} + +static int test_write_columns_round_trip() +{ + const std::vector offsets{0, 3, 6}; + const std::vector blob{'u', 'm', 'a', 'd', 'o', 'i'}; + const std::vector inteiros{1, 2}; + const std::vector numeros{0.5, 1.5}; + const std::vector datas{20454, 20455}; // 2026-01-01, 2026-01-02 + const std::vector horas{3600000000, 7200000000}; + const std::vector instantes{1767225600000000, 1767312000000000}; + + const std::array columns{ + xl::string_column("texto", offsets, blob), + xl::i64_column("inteiro", inteiros), + xl::f64_column("numero", numeros), + xl::date_column("data", datas), + xl::time_column("hora", horas), + xl::timestamp_column("instante", instantes)}; + + const std::filesystem::path path = temp_path("columns.xlsx"); + auto written = xl::write_columns(path.string(), XL_FORMAT_XLSX, columns); + CHECK(written.has_value(), "write_columns must succeed"); + { + auto workbook = xl::Workbook::open(path.string()); + CHECK(workbook.has_value(), "the written file must open"); + auto table = xl::parse_sheet(*workbook); + CHECK(table.has_value(), "the written file must parse back"); + CHECK(table->size() == 2, "two rows were written"); + + WrittenRow first = *table->begin(); + CHECK(first.texto == "uma", "row 0's string must round-trip"); + CHECK(first.inteiro == 1, "row 0's int64 must round-trip"); + CHECK(first.numero == 0.5, "row 0's double must round-trip"); + CHECK(first.data == std::chrono::year{2026} / std::chrono::month{1} / std::chrono::day{1}, + "row 0's date must round-trip"); + CHECK(first.hora == std::chrono::microseconds{3600000000}, "row 0's time must round-trip"); + auto second_row = table->at(1); + CHECK(second_row.has_value(), "row 1 must be in bounds"); + CHECK(second_row->texto == "doi", "row 1's string must round-trip"); + CHECK(second_row->inteiro == 2, "row 1's int64 must round-trip"); + } + std::filesystem::remove(path); + return 0; +} + +static int test_write_columns_rejects_bad_input() +{ + const std::vector two{1, 2}; + const std::vector three{1, 2, 3}; + const std::vector empty_bitmap{}; + const std::filesystem::path path = temp_path("rejected.xlsx"); + + std::array mismatched{xl::i64_column("a", two), xl::i64_column("b", three)}; + CHECK(!xl::write_columns(path.string(), XL_FORMAT_XLSX, mismatched).has_value(), + "columns of different lengths must be rejected"); + + std::array partial_header{xl::i64_column("a", two), xl::i64_column("", two)}; + CHECK(!xl::write_columns(path.string(), XL_FORMAT_XLSX, partial_header).has_value(), + "a partial header row must be rejected"); + + // Two rows need one byte of bitmap. Hand it a non-null pointer with zero length: the ABI takes + // the bitmap without a length, so this is exactly the overrun the wrapper exists to refuse. + std::array short_bitmap{xl::i64_column("a", two)}; + short_bitmap[0].validity = reinterpret_cast(two.data()); + short_bitmap[0].validity_len = 0; + CHECK(!xl::write_columns(path.string(), XL_FORMAT_XLSX, short_bitmap).has_value(), + "a validity bitmap shorter than the row count must be rejected"); + + std::array fine{xl::i64_column("a", two)}; + CHECK(!xl::write_columns(path.string(), XL_FORMAT_AUTO, fine).has_value(), + "XL_FORMAT_AUTO must be rejected: a new file has no signature bytes to sniff"); + + std::span none{}; + CHECK(!xl::write_columns(path.string(), XL_FORMAT_XLSX, none).has_value(), + "an empty column set must be rejected"); + + std::filesystem::remove(path); + return 0; +} + +static int test_write_options() +{ + xl::WriteOptions defaults{}; + xl_write_options raw = defaults.to_c(); + CHECK(raw.struct_size == static_cast(sizeof(xl_write_options)), "to_c must fill struct_size"); + CHECK(raw.sheet_name == nullptr, "an empty sheet_name must lower to NULL, meaning Sheet1"); + CHECK(raw.sheet_name_len == 0, "an empty sheet_name must lower to length 0"); + CHECK(raw.csv_delimiter == 0, "an unset csv_delimiter must lower to 0"); + CHECK(raw.date1904 == XL_OPT_DEFAULT, "an unset tri-state must lower to XL_OPT_DEFAULT"); + + xl::WriteOptions configured{}; + configured.sheet_name = "Dados"; + configured.csv_delimiter = ';'; + configured.use_shared_strings = XL_OPT_TRUE; + xl_write_options set = configured.to_c(); + CHECK(set.sheet_name_len == 5, "sheet_name_len must be the byte length"); + CHECK(set.sheet_name != nullptr, "a non-empty sheet_name must lower to a pointer"); + CHECK(set.csv_delimiter == ';', "csv_delimiter must pass through unchanged"); + CHECK(set.use_shared_strings == XL_OPT_TRUE, "use_shared_strings must pass through unchanged"); + return 0; +} + +static int test_format_from_path() +{ + CHECK(xl::format_from_path("out.xlsx") == XL_FORMAT_XLSX, ".xlsx must resolve to XL_FORMAT_XLSX"); + CHECK(xl::format_from_path("out.XLSB") == XL_FORMAT_XLSB, "the extension match must be case-insensitive"); + CHECK(xl::format_from_path("out.xls") == XL_FORMAT_XLS, ".xls must resolve to XL_FORMAT_XLS"); + CHECK(xl::format_from_path("out.csv") == XL_FORMAT_CSV, ".csv must resolve to XL_FORMAT_CSV"); + CHECK(xl::format_from_path("out.txt") == XL_FORMAT_AUTO, "an unknown extension must resolve to AUTO"); + CHECK(xl::format_from_path("out") == XL_FORMAT_AUTO, "no extension at all must resolve to AUTO"); + // A dot in a directory name is not an extension. + CHECK(xl::format_from_path("v1.2/report") == XL_FORMAT_AUTO, "a dot before the last separator is not an extension"); + return 0; +} + +int main() +{ + if (test_write_options() != 0) + { + return 1; + } + if (test_format_from_path() != 0) + { + return 1; + } + if (test_write_columns_round_trip() != 0) + { + return 1; + } + if (test_write_columns_rejects_bad_input() != 0) + { + return 1; + } + std::printf("OK\n"); + return 0; +} \ No newline at end of file diff --git a/src/ExcelReader.Native/include/excelreader.hpp b/src/ExcelReader.Native/include/excelreader.hpp index 4c23aa3..ca9247e 100644 --- a/src/ExcelReader.Native/include/excelreader.hpp +++ b/src/ExcelReader.Native/include/excelreader.hpp @@ -1,7 +1,7 @@ /* Header-only C++ wrapper around excelreader.h (the C ABI). * - * Scope of this first pass: opening a workbook and schema-driven typed table parsing - * (xl_parse_typed) only - no writing, no row-by-row decoded reads. + * Scope: opening a workbook, schema-driven typed table parsing (xl_parse_typed), and + * schema-driven writing (xl_write_typed). No row-by-row decoded reads. * * Design constraints, matching the native library's own perf/memory posture: * - No exceptions anywhere in this header. Every fallible operation returns @@ -15,6 +15,10 @@ * - std::string_view fields are zero-copy views into the xl_table's own string blob: * valid ONLY as long as the owning TableView is alive. Use std::string for a * field that needs to outlive the view (e.g. after to_vector()). + * - Writing borrows. xl::write_columns hands xl_write_typed the caller's own buffers, + * which the ABI reads without copying or freeing; they must outlive the call. + * xl::write_sheet is the one place a copy happens, and only because a range of + * structs has to be transposed into columns. */ #pragma once @@ -27,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -182,6 +187,270 @@ namespace xl } }; + // ---- Write options --------------------------------------------------------------------------- + + // C++ mirror of xl_write_options: same fields, same meaning (0 or XL_OPT_DEFAULT for "use the + // library default" on every field), with default member initializers so a caller sets only what + // they want to override. to_c() fills in struct_size. + // + // sheet_name is BORROWED, like every other buffer this library hands the ABI: the string it + // views must outlive the write call. Its rules (1-31 characters, none of : \ / ? * [ ]) are + // validated natively and reported through xl_last_error, so they are deliberately not + // re-checked here - one set of bounds, one place to change them. + struct WriteOptions + { + std::string_view sheet_name{}; // empty = "Sheet1". Ignored for XL_FORMAT_CSV. + + // CSV only; ignored for every other format. Byte value 1-255; 0 = default (',' and '"'). + int32_t csv_delimiter = 0; + int32_t csv_quote = 0; + + int32_t date1904 = XL_OPT_DEFAULT; // XLS/XLSB only + int32_t use_shared_strings = XL_OPT_DEFAULT; // XLSX/XLSB only + + xl_write_options to_c() const noexcept + { + xl_write_options opts{}; + opts.struct_size = sizeof(xl_write_options); + opts.sheet_name_len = static_cast(sheet_name.size()); + opts.sheet_name = sheet_name.empty() + ? nullptr + : reinterpret_cast(sheet_name.data()); + opts.csv_delimiter = csv_delimiter; + opts.csv_quote = csv_quote; + opts.date1904 = date1904; + opts.use_shared_strings = use_shared_strings; + return opts; + } + }; + + namespace detail + { + + // Case-insensitive suffix match over ASCII, which is all a file extension can be here. + // constexpr and allocation-free so format_from_path stays usable in a constant expression. + constexpr bool ends_with_ci(std::string_view text, std::string_view suffix) noexcept + { + if (text.size() < suffix.size()) + { + return false; + } + const std::string_view tail = text.substr(text.size() - suffix.size()); + for (size_t i = 0; i < suffix.size(); ++i) + { + const char c = tail[i]; + const char lowered = (c >= 'A' && c <= 'Z') ? static_cast(c - 'A' + 'a') : c; + if (lowered != suffix[i]) + { + return false; + } + } + return true; + } + + } // namespace detail + + // Infers an XL_FORMAT_* from a path's extension. Returns XL_FORMAT_AUTO when the extension is + // absent or unrecognized - and since xl_write_typed rejects AUTO with a message of its own, an + // unrecognized path fails the write rather than silently picking a format. + constexpr int32_t format_from_path(std::string_view path) noexcept + { + const size_t separator = path.find_last_of("/\\"); + const std::string_view name = (separator == std::string_view::npos) + ? path + : path.substr(separator + 1); + if (detail::ends_with_ci(name, ".xlsx")) + { + return XL_FORMAT_XLSX; + } + if (detail::ends_with_ci(name, ".xlsb")) + { + return XL_FORMAT_XLSB; + } + if (detail::ends_with_ci(name, ".xls")) + { + return XL_FORMAT_XLS; + } + if (detail::ends_with_ci(name, ".csv")) + { + return XL_FORMAT_CSV; + } + return XL_FORMAT_AUTO; + } + + // ---- Columnar write -------------------------------------------------------------------------- + + // One INPUT column, pointing at the caller's own buffers. Nothing here is copied: every pointer + // must stay valid until write_columns returns. + // + // Build one through the typed constructors below rather than by hand - they derive `length`, + // `type` and `validity_len` from the spans they are handed, which is what makes the bounds check + // in write_columns possible at all. + struct ColumnRef + { + std::string_view name{}; // empty = no header row (all-or-nothing across the column set) + int32_t type = XL_T_STRING; + int64_t length = 0; + const void *values = nullptr; + const uint8_t *validity = nullptr; // nullptr = the column has no nulls + // NOT part of the ABI struct: xl_write_typed takes the bitmap without a length and reads + // (length + 7) / 8 bytes on trust. Carrying the length here is what lets write_columns + // refuse a short one instead of handing the native side a buffer overrun. + int64_t validity_len = 0; + const uint8_t *data = nullptr; // XL_T_STRING only: the UTF-8 blob + int64_t data_len = 0; + }; + + namespace detail + { + + constexpr const uint8_t *validity_pointer(std::span validity) noexcept + { + return validity.empty() ? nullptr : validity.data(); + } + + } // namespace detail + + // One constructor per column type rather than an overload set: XL_T_BOOL's buffer and a string + // blob are both std::span, and XL_T_I64/TIME/TIMESTAMP are all + // std::span, so overload resolution could not tell them apart. + inline constexpr ColumnRef i64_column(std::string_view name, std::span values, + std::span validity = {}) noexcept + { + return ColumnRef{name, XL_T_I64, static_cast(values.size()), values.data(), + detail::validity_pointer(validity), static_cast(validity.size()), + nullptr, 0}; + } + + inline constexpr ColumnRef f64_column(std::string_view name, std::span values, + std::span validity = {}) noexcept + { + return ColumnRef{name, XL_T_F64, static_cast(values.size()), values.data(), + detail::validity_pointer(validity), static_cast(validity.size()), + nullptr, 0}; + } + + // `values` is one byte per row, 0 or 1 - NOT a bit-packed bitmap. + inline constexpr ColumnRef bool_column(std::string_view name, std::span values, + std::span validity = {}) noexcept + { + return ColumnRef{name, XL_T_BOOL, static_cast(values.size()), values.data(), + detail::validity_pointer(validity), static_cast(validity.size()), + nullptr, 0}; + } + + inline constexpr ColumnRef date_column(std::string_view name, std::span days_since_epoch, + std::span validity = {}) noexcept + { + return ColumnRef{name, XL_T_DATE, static_cast(days_since_epoch.size()), + days_since_epoch.data(), detail::validity_pointer(validity), + static_cast(validity.size()), nullptr, 0}; + } + + inline constexpr ColumnRef time_column(std::string_view name, std::span micros_since_midnight, + std::span validity = {}) noexcept + { + return ColumnRef{name, XL_T_TIME, static_cast(micros_since_midnight.size()), + micros_since_midnight.data(), detail::validity_pointer(validity), + static_cast(validity.size()), nullptr, 0}; + } + + inline constexpr ColumnRef timestamp_column(std::string_view name, std::span micros_since_epoch, + std::span validity = {}) noexcept + { + return ColumnRef{name, XL_T_TIMESTAMP, static_cast(micros_since_epoch.size()), + micros_since_epoch.data(), detail::validity_pointer(validity), + static_cast(validity.size()), nullptr, 0}; + } + + // `offsets` has length + 1 entries; `data` is every row's UTF-8 bytes concatenated. Unlike the + // table xl_parse_typed returns, `data` need not be interior to `offsets` here. + inline constexpr ColumnRef string_column(std::string_view name, std::span offsets, + std::span data, + std::span validity = {}) noexcept + { + const int64_t rows = offsets.empty() ? 0 : static_cast(offsets.size()) - 1; + return ColumnRef{name, XL_T_STRING, rows, offsets.data(), detail::validity_pointer(validity), + static_cast(validity.size()), data.empty() ? nullptr : data.data(), + static_cast(data.size())}; + } + + namespace detail + { + + // Split out of validate_write_columns to stay inside the style guide's nesting and length + // limits. Returns nullopt when the column is acceptable. + inline std::optional validate_one_write_column(const ColumnRef &column, size_t index, + int64_t row_count, bool has_header) + { + const std::string at = " (column " + std::to_string(index) + ")"; + if (column.length != row_count) + { + return Error{XL_INVALID_ARGUMENT, + "every column must have the same length; column 0 has " + + std::to_string(row_count) + " rows but this one has " + + std::to_string(column.length) + at}; + } + if (column.name.empty() == has_header) + { + return Error{XL_INVALID_ARGUMENT, + "every column must have a name, or none may - xl_write_typed cannot write " + "a partial header row" + + at}; + } + if (column.validity != nullptr && column.validity_len < (row_count + 7) / 8) + { + return Error{XL_INVALID_ARGUMENT, + "the validity bitmap is " + std::to_string(column.validity_len) + + " bytes, but " + std::to_string(row_count) + " rows need " + + std::to_string((row_count + 7) / 8) + at}; + } + if (column.type == XL_T_STRING && column.data_len > INT32_MAX) + { + return Error{XL_INVALID_ARGUMENT, + "the string blob is larger than 2 GiB, which int32 offsets cannot address" + at}; + } + return std::nullopt; + } + + // Returns the row count every column agreed on, or the first problem found. Runs to + // completion before anything reaches the native side, matching xl_write_typed's own + // "validate everything, then write" posture - a partially written file plus a buffer + // overrun is strictly worse than a rejected call. + inline std::expected validate_write_columns(std::span columns) + { + if (columns.empty()) + { + return std::unexpected(Error{XL_INVALID_ARGUMENT, "write_columns needs at least one column."}); + } + const int64_t row_count = columns.front().length; + const bool has_header = !columns.front().name.empty(); + for (size_t i = 0; i < columns.size(); ++i) + { + std::optional problem = validate_one_write_column(columns[i], i, row_count, has_header); + if (problem.has_value()) + { + return std::unexpected(std::move(*problem)); + } + } + return row_count; + } + + // Lowers one ColumnRef into the two ABI structs. `name_slot` and `len_slot` are elements of + // arrays the caller keeps alive: xl_column_spec::names is a pointer to an ARRAY of name + // pointers, so each spec needs a stable address to point at, not a temporary. + inline void fill_write_column(const ColumnRef &column, const uint8_t *&name_slot, int32_t &len_slot, + xl_column_spec &spec, xl_column &raw) noexcept + { + name_slot = column.name.empty() ? nullptr : reinterpret_cast(column.name.data()); + len_slot = static_cast(column.name.size()); + spec = xl_column_spec{&name_slot, &len_slot, column.name.empty() ? 0 : 1, 0, column.type, 0}; + raw = xl_column{column.type, column.length, column.values, column.validity, column.data, + column.data_len}; + } + + } // namespace detail + // ---- Workbook (RAII) ------------------------------------------------------------------------ class Workbook @@ -813,4 +1082,65 @@ namespace xl return view->to_vector(); } + // Writes `columns` to `path` as a single sheet, then closes the file. One-shot: no writer handle + // exists before or after, and every buffer reachable from `columns` and `options` is borrowed + // for the duration of the call and never freed by this library. + // + // `format` must be XL_FORMAT_XLS/XLSX/XLSB/CSV. XL_FORMAT_AUTO is an error, because a file being + // created has no signature bytes to sniff. On failure the destination may exist and be + // incomplete - cleaning it up is the caller's. + inline std::expected write_columns(std::string_view path, int32_t format, + std::span columns, + const WriteOptions *options = nullptr) + { + const std::expected &abi = detail::check_abi_version(); + if (!abi.has_value()) + { + return std::unexpected(abi.error()); + } + std::expected row_count = detail::validate_write_columns(columns); + if (!row_count.has_value()) + { + return std::unexpected(std::move(row_count.error())); + } + + const size_t count = columns.size(); + std::vector name_slots(count); + std::vector name_lens(count); + std::vector specs(count); + std::vector raw_columns(count); + for (size_t i = 0; i < count; ++i) + { + detail::fill_write_column(columns[i], name_slots[i], name_lens[i], specs[i], raw_columns[i]); + } + + xl_table table{static_cast(count), *row_count, raw_columns.data()}; + // A zeroed xl_write_options is NOT the same as no options: its struct_size of 0 is rejected. + // NULL is what means "every default". + xl_write_options raw_options{}; + const xl_write_options *options_pointer = nullptr; + if (options != nullptr) + { + raw_options = options->to_c(); + options_pointer = &raw_options; + } + + const int32_t status = xl_write_typed(reinterpret_cast(path.data()), + static_cast(path.size()), format, specs.data(), + &table, options_pointer); + if (status != XL_OK) + { + return std::unexpected(detail::make_error(status)); + } + return {}; + } + + // Infers the format from the path's extension. An unrecognized extension yields XL_FORMAT_AUTO, + // which xl_write_typed then rejects by name. + inline std::expected write_columns(std::string_view path, std::span columns, + const WriteOptions *options = nullptr) + { + return write_columns(path, format_from_path(path), columns, options); + } + } // namespace xl From 578913960828569d2a19e7ad42308b9fadd6b10d Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Thu, 20 Aug 2026 14:49:50 -0300 Subject: [PATCH 03/51] fix(cpp): decode bool fields from the XL_T_BOOL byte buffer, not as int64 --- cpp/tests/write.cpp | 44 ++++++++++++++++++- .../include/excelreader.hpp | 8 ++-- 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/cpp/tests/write.cpp b/cpp/tests/write.cpp index 3c0149e..a3a8950 100644 --- a/cpp/tests/write.cpp +++ b/cpp/tests/write.cpp @@ -78,7 +78,7 @@ static int test_write_columns_round_trip() auto table = xl::parse_sheet(*workbook); CHECK(table.has_value(), "the written file must parse back"); CHECK(table->size() == 2, "two rows were written"); - + WrittenRow first = *table->begin(); CHECK(first.texto == "uma", "row 0's string must round-trip"); CHECK(first.inteiro == 1, "row 0's int64 must round-trip"); @@ -165,6 +165,44 @@ static int test_format_from_path() return 0; } +struct FlagRow +{ + bool ativo; +}; + +template <> +struct xl::ExcelMapper +{ + static constexpr auto get_bindings() + { + return std::make_tuple(xl::make_field("ativo", &FlagRow::ativo)); + } +}; + +static int test_bool_round_trip() +{ + // One byte per row, 0 or 1 - XL_T_BOOL's wire layout, not a bit-packed bitmap. + const std::vector flags{1, 0, 1}; + const std::array columns{xl::bool_column("ativo", flags)}; + + const std::filesystem::path path = temp_path("bools.xlsx"); + CHECK(xl::write_columns(path.string(), XL_FORMAT_XLSX, columns).has_value(), + "writing a bool column must succeed"); + { + auto workbook = xl::Workbook::open(path.string()); + CHECK(workbook.has_value(), "the written file must open"); + auto table = xl::parse_sheet(*workbook); + CHECK(table.has_value(), "the written file must parse back"); + CHECK(table->size() == 3, "three rows were written"); + + CHECK(table->at(0)->ativo, "row 0 was written true"); + CHECK(!table->at(1)->ativo, "row 1 was written false"); + CHECK(table->at(2)->ativo, "row 2 was written true"); + } + std::filesystem::remove(path); + return 0; +} + int main() { if (test_write_options() != 0) @@ -183,6 +221,10 @@ int main() { return 1; } + if (test_bool_round_trip() != 0) + { + return 1; + } std::printf("OK\n"); return 0; } \ No newline at end of file diff --git a/src/ExcelReader.Native/include/excelreader.hpp b/src/ExcelReader.Native/include/excelreader.hpp index ca9247e..736abc4 100644 --- a/src/ExcelReader.Native/include/excelreader.hpp +++ b/src/ExcelReader.Native/include/excelreader.hpp @@ -820,6 +820,10 @@ namespace xl instance.*(binding.member) = T(str_data, static_cast(end - start)); } } + else if constexpr (std::is_same_v) + { + instance.*(binding.member) = (static_cast(col.values)[row] != 0); + } else if constexpr (std::is_integral_v) { instance.*(binding.member) = T(static_cast(col.values)[row]); @@ -828,10 +832,6 @@ namespace xl { instance.*(binding.member) = T(static_cast(col.values)[row]); } - else if constexpr (std::is_same_v) - { - instance.*(binding.member) = (static_cast(col.values)[row] != 0); - } else if constexpr (std::is_same_v) { int32_t days = static_cast(col.values)[row]; From 387cfa5e4464bd1b9e91e73edbf6e4b515ae44c9 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Thu, 20 Aug 2026 14:56:03 -0300 Subject: [PATCH 04/51] feat(cpp): support std::optional fields for nullable columns --- cpp/tests/write.cpp | 43 ++++++++++++++++++ .../include/excelreader.hpp | 44 ++++++++++++++++++- 2 files changed, 85 insertions(+), 2 deletions(-) diff --git a/cpp/tests/write.cpp b/cpp/tests/write.cpp index a3a8950..9587f1c 100644 --- a/cpp/tests/write.cpp +++ b/cpp/tests/write.cpp @@ -203,6 +203,45 @@ static int test_bool_round_trip() return 0; } +struct NullableRow +{ + std::optional quantidade; +}; + +template <> +struct xl::ExcelMapper +{ + static constexpr auto get_bindings() + { + return std::make_tuple(xl::make_field("quantidade", &NullableRow::quantidade)); + } +}; + +static int test_optional_round_trip() +{ + const std::vector valores{10, 0, 30}; + // LSB-first: bit 0 and bit 2 set, bit 1 clear - row 1 is null. + const std::vector validity{0b00000101}; + const std::array columns{xl::i64_column("quantidade", valores, validity)}; + + const std::filesystem::path path = temp_path("nullable.xlsx"); + CHECK(xl::write_columns(path.string(), XL_FORMAT_XLSX, columns).has_value(), + "writing a nullable column must succeed"); + { + auto workbook = xl::Workbook::open(path.string()); + CHECK(workbook.has_value(), "the written file must open"); + auto table = xl::parse_sheet(*workbook); + CHECK(table.has_value(), "the written file must parse back"); + CHECK(table->size() == 3, "three rows were written"); + + CHECK(table->at(0)->quantidade == 10, "row 0 must round-trip its value"); + CHECK(!table->at(1)->quantidade.has_value(), "row 1 was written null and must come back empty"); + CHECK(table->at(2)->quantidade == 30, "row 2 must round-trip its value"); + } + std::filesystem::remove(path); + return 0; +} + int main() { if (test_write_options() != 0) @@ -225,6 +264,10 @@ int main() { return 1; } + if (test_optional_round_trip() != 0) + { + return 1; + } std::printf("OK\n"); return 0; } \ No newline at end of file diff --git a/src/ExcelReader.Native/include/excelreader.hpp b/src/ExcelReader.Native/include/excelreader.hpp index 736abc4..c7228f7 100644 --- a/src/ExcelReader.Native/include/excelreader.hpp +++ b/src/ExcelReader.Native/include/excelreader.hpp @@ -732,6 +732,35 @@ namespace xl static constexpr int32_t value = XL_T_TIME; }; + template + struct XlType> + { + static constexpr int32_t value = XlType::value; + }; + + namespace detail + { + + template + struct IsOptional : std::false_type + { + using Inner = T; + }; + + template + struct IsOptional> : std::true_type + { + using Inner = T; + }; + + template + inline constexpr bool is_optional_v = IsOptional::value; + + template + using unwrap_optional_t = typename IsOptional::Inner; + + } // namespace detail + // ---- Struct <-> column bindings --------------------------------------------------------------- template @@ -808,8 +837,19 @@ namespace xl { return; // leave the struct member default-initialized } - - if constexpr (std::is_same_v || std::is_same_v) + if constexpr (detail::is_optional_v) + { + using Inner = detail::unwrap_optional_t; + struct Holder + { + Inner value{}; + }; + Holder holder{}; + const FieldBinding inner_binding{binding.column_names, &Holder::value}; + assign_field(holder, col, row, inner_binding); + instance.*(binding.member) = std::move(holder.value); + } + else if constexpr (std::is_same_v || std::is_same_v) { const int32_t *offsets = static_cast(col.values); int32_t start = offsets[row]; From aaf02932df12abfa880f96bd3b6703f910317c43 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Thu, 20 Aug 2026 15:10:10 -0300 Subject: [PATCH 05/51] feat(cpp): add xl::write_sheet over the existing ExcelMapper --- cpp/tests/write.cpp | 106 ++++++ .../include/excelreader.hpp | 307 ++++++++++++++++++ 2 files changed, 413 insertions(+) diff --git a/cpp/tests/write.cpp b/cpp/tests/write.cpp index 9587f1c..c08433a 100644 --- a/cpp/tests/write.cpp +++ b/cpp/tests/write.cpp @@ -242,6 +242,104 @@ static int test_optional_round_trip() return 0; } +struct FullRow +{ + std::string texto; + int64_t inteiro; + double numero; + bool ativo; + std::chrono::year_month_day data; + std::chrono::microseconds hora; + std::chrono::system_clock::time_point instante; + std::optional opcional; +}; + +template <> +struct xl::ExcelMapper +{ + static constexpr auto get_bindings() + { + return std::make_tuple( + xl::make_field("texto", &FullRow::texto), + xl::make_field("inteiro", &FullRow::inteiro), + xl::make_field("numero", &FullRow::numero), + xl::make_field("ativo", &FullRow::ativo), + xl::make_field("data", &FullRow::data), + xl::make_field("hora", &FullRow::hora), + xl::make_field("instante", &FullRow::instante), + xl::make_field("opcional", &FullRow::opcional)); + } +}; + +static int test_write_sheet_round_trip() +{ + const std::vector rows{ + FullRow{"uma", 1, 0.5, true, + std::chrono::year{2026} / std::chrono::month{1} / std::chrono::day{1}, + std::chrono::microseconds{3600000000}, + std::chrono::system_clock::time_point{std::chrono::microseconds{1767225600000000}}, + std::optional{7}}, + FullRow{"duas", 2, 1.5, false, + std::chrono::year{2026} / std::chrono::month{1} / std::chrono::day{2}, + std::chrono::microseconds{7200000000}, + std::chrono::system_clock::time_point{std::chrono::microseconds{1767312000000000}}, + std::nullopt}}; + + const std::filesystem::path path = temp_path("sheet.xlsx"); + // The format is inferred from the .xlsx extension by this overload. + CHECK(xl::write_sheet(path.string(), rows).has_value(), "write_sheet must succeed"); + { + auto workbook = xl::Workbook::open(path.string()); + CHECK(workbook.has_value(), "the written file must open"); + auto table = xl::parse_sheet(*workbook); + CHECK(table.has_value(), "the written file must parse back"); + CHECK(table->size() == 2, "two rows were written"); + + auto first = table->at(0); + CHECK(first.has_value(), "row 0 must be in bounds"); + CHECK(first->texto == "uma", "row 0's string must round-trip"); + CHECK(first->inteiro == 1, "row 0's int64 must round-trip"); + CHECK(first->numero == 0.5, "row 0's double must round-trip"); + CHECK(first->ativo, "row 0's bool must round-trip"); + CHECK(first->data == std::chrono::year{2026} / std::chrono::month{1} / std::chrono::day{1}, + "row 0's date must round-trip"); + CHECK(first->hora == std::chrono::microseconds{3600000000}, "row 0's time must round-trip"); + CHECK(first->opcional == 7, "row 0's optional must round-trip its value"); + + auto second = table->at(1); + CHECK(second.has_value(), "row 1 must be in bounds"); + CHECK(second->texto == "duas", "row 1's string must round-trip"); + CHECK(!second->ativo, "row 1's bool must round-trip as false"); + CHECK(!second->opcional.has_value(), "row 1's nullopt must come back empty"); + } + std::filesystem::remove(path); + return 0; +} + +static int test_write_sheet_options_and_csv() +{ + const std::vector rows{}; + + const std::filesystem::path path = temp_path("named.xlsx"); + xl::WriteOptions options{}; + options.sheet_name = "Dados"; + CHECK(xl::write_sheet(path.string(), XL_FORMAT_XLSX, rows, &options).has_value(), + "writing an empty sheet with a custom name must succeed"); + { + auto workbook = xl::Workbook::open(path.string()); + CHECK(workbook.has_value(), "the written file must open"); + auto name = workbook->sheet_name(); + CHECK(name.has_value(), "the sheet name must be readable"); + CHECK(*name == "Dados", "WriteOptions::sheet_name must reach the file"); + } + std::filesystem::remove(path); + + const std::filesystem::path csv = temp_path("sheet.csv"); + CHECK(xl::write_sheet(csv.string(), rows).has_value(), "a .csv path must infer XL_FORMAT_CSV"); + std::filesystem::remove(csv); + return 0; +} + int main() { if (test_write_options() != 0) @@ -268,6 +366,14 @@ int main() { return 1; } + if (test_write_sheet_round_trip() != 0) + { + return 1; + } + if (test_write_sheet_options_and_csv() != 0) + { + return 1; + } std::printf("OK\n"); return 0; } \ No newline at end of file diff --git a/src/ExcelReader.Native/include/excelreader.hpp b/src/ExcelReader.Native/include/excelreader.hpp index c7228f7..6838813 100644 --- a/src/ExcelReader.Native/include/excelreader.hpp +++ b/src/ExcelReader.Native/include/excelreader.hpp @@ -1183,4 +1183,311 @@ namespace xl return write_columns(path, format_from_path(path), columns, options); } + // ---- write_sheet: transposing a range of structs into columns ----------------------------- + + namespace detail + { + + // An XL_T_STRING column's two output buffers. `overflowed` latches rather than throwing: + // this header has no exceptions anywhere, and write_sheet checks it once before the write. + struct StringBuffer + { + std::vector offsets{0}; + std::vector data{}; + bool overflowed = false; + + void reserve(size_t rows) + { + offsets.reserve(rows + 1); + } + + void push(std::string_view value) + { + if (data.size() + value.size() > static_cast(INT32_MAX)) + { + // Record the failure and keep the offsets array well-formed, so nothing + // downstream reads a half-built column before write_sheet bails out. + overflowed = true; + offsets.push_back(offsets.back()); + return; + } + const uint8_t *bytes = reinterpret_cast(value.data()); + data.insert(data.end(), bytes, bytes + value.size()); + offsets.push_back(static_cast(data.size())); + } + }; + + // The output buffer each XL_T_* needs, at that type's exact wire width. + template + struct ColumnStorage; + + template <> + struct ColumnStorage + { + using type = StringBuffer; + }; + template <> + struct ColumnStorage + { + using type = std::vector; + }; + template <> + struct ColumnStorage + { + using type = std::vector; + }; + template <> + struct ColumnStorage + { + using type = std::vector; + }; + template <> + struct ColumnStorage + { + using type = std::vector; + }; + template <> + struct ColumnStorage + { + using type = std::vector; + }; + template <> + struct ColumnStorage + { + using type = std::vector; + }; + + // One column's accumulating buffers, built from the FIELD type. `validity` stays empty + // unless the field is std::optional - the ABI reads validity == NULL as "no nulls", so a + // non-nullable column costs no bitmap at all. + template + struct ColumnBuilder + { + using Field = unwrap_optional_t; + static constexpr bool nullable = is_optional_v; + static constexpr int32_t column_type = XlType::value; + + typename ColumnStorage::type storage{}; + std::vector validity{}; + int64_t rows = 0; + + void reserve(size_t count) + { + storage.reserve(count); + if constexpr (nullable) + { + validity.reserve((count + 7) / 8); + } + } + + void push(const T &value) + { + if constexpr (nullable) + { + // Grows one byte every eight rows, so the bitmap is always exactly big enough + // for the rows pushed so far. + validity.resize(static_cast((rows + 8) / 8), 0); + if (value.has_value()) + { + validity[static_cast(rows / 8)] |= + static_cast(1u << static_cast(rows % 8)); + append(*value); + } + else + { + append_placeholder(); + } + } + else + { + append(value); + } + ++rows; + } + + bool overflowed() const + { + if constexpr (column_type == XL_T_STRING) + { + return storage.overflowed; + } + else + { + return false; + } + } + + ColumnRef to_ref(std::string_view name) const + { + if constexpr (column_type == XL_T_STRING) + { + return string_column(name, storage.offsets, storage.data, validity); + } + else if constexpr (column_type == XL_T_I64) + { + return i64_column(name, storage, validity); + } + else if constexpr (column_type == XL_T_F64) + { + return f64_column(name, storage, validity); + } + else if constexpr (column_type == XL_T_BOOL) + { + return bool_column(name, storage, validity); + } + else if constexpr (column_type == XL_T_DATE) + { + return date_column(name, storage, validity); + } + else if constexpr (column_type == XL_T_TIME) + { + return time_column(name, storage, validity); + } + else + { + return timestamp_column(name, storage, validity); + } + } + + private: + // A null row still occupies a slot in the values buffer; its bit is what marks it + // absent. Zero (or the empty string) is the placeholder the writer never reads. + void append_placeholder() + { + if constexpr (column_type == XL_T_STRING) + { + storage.push(std::string_view{}); + } + else + { + storage.push_back({}); + } + } + + // The exact inverse of detail::assign_field - same chain, same conversions, opposite + // direction. If one of them gains a type, so must the other. + void append(const Field &value) + { + if constexpr (std::is_same_v || std::is_same_v) + { + storage.push(std::string_view(value)); + } + else if constexpr (std::is_same_v) + { + storage.push_back(static_cast(value ? 1 : 0)); + } + else if constexpr (std::is_integral_v) + { + storage.push_back(static_cast(value)); + } + else if constexpr (std::is_floating_point_v) + { + storage.push_back(static_cast(value)); + } + else if constexpr (std::is_same_v) + { + storage.push_back(static_cast(value.time_since_epoch().count())); + } + else if constexpr (std::is_same_v) + { + storage.push_back( + static_cast(std::chrono::sys_days{value}.time_since_epoch().count())); + } + else if constexpr (std::is_same_v) + { + storage.push_back(value.count()); + } + else if constexpr (std::is_same_v>) + { + storage.push_back(value.to_duration().count()); + } + else if constexpr (std::is_same_v) + { + storage.push_back(std::chrono::time_point_cast(value) + .time_since_epoch() + .count()); + } + } + }; + + // The tuple of ColumnBuilders matching a bindings tuple, one per field, in the same order. + template + struct BuildersFor; + template + struct BuildersFor> + { + using type = std::tuple::FieldType>...>; + }; + + template + void push_row(Builders &builders, const T &row, const Tuple &bindings, std::index_sequence) + { + (..., std::get(builders).push(row.*(std::get(bindings).member))); + } + + template + void reserve_all(Builders &builders, size_t count, std::index_sequence) + { + (..., std::get(builders).reserve(count)); + } + + template + bool any_overflowed(const Builders &builders, std::index_sequence) + { + return (... || std::get(builders).overflowed()); + } + + template + std::array to_refs(const Builders &builders, const Tuple &bindings, + std::index_sequence) + { + // Only the FIRST candidate name is used: xl_write_typed rejects a write spec carrying + // more than one, and the alias list exists to resolve a header on the way IN. + return {std::get(builders).to_ref(std::string_view(std::get(bindings).column_names[0]))...}; + } + } // namespace detail + + // Writes `rows` to `path` as a single sheet, using the same xl::ExcelMapper specialization + // that xl::parse_sheet reads with - so reading a sheet into structs and writing it back out + // needs one mapping, not two. + // + // The range is walked ONCE, and each field is appended to its own column buffer through a + // compile-time dispatch. That transpose is the only copy this makes; it is what the ABI's + // columnar shape costs a row-shaped caller. If you already hold columnar buffers, call + // write_columns instead and pay nothing. + template + std::expected write_sheet(std::string_view path, int32_t format, R &&rows, + const WriteOptions *options = nullptr) + { + using T = std::remove_cvref_t>; + static constexpr auto bindings = ExcelMapper::get_bindings(); + static constexpr size_t field_count = std::tuple_size_v; + static constexpr auto indices = std::make_index_sequence{}; + typename detail::BuildersFor>::type builders{}; + if constexpr (std::ranges::sized_range) + { + detail::reserve_all(builders, static_cast(std::ranges::size(rows)), indices); + } + for (const auto &row : rows) + { + detail::push_row(builders, row, bindings, indices); + } + + if (detail::any_overflowed(builders, indices)) + { + return std::unexpected(Error{XL_INVALID_ARGUMENT, + "a string column exceeds 2 GiB, which int32 offsets cannot address."}); + } + + const std::array refs = detail::to_refs(builders, bindings, indices); + return write_columns(path, format, refs, options); + } + + // Infers the format from the path's extension. + template + std::expected write_sheet(std::string_view path, R &&rows, + const WriteOptions *options = nullptr) + { + return write_sheet(path, format_from_path(path), std::forward(rows), options); + } } // namespace xl From 9f13866f471d9ef3886449f5e80b4bd6bc412e0c Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Thu, 20 Aug 2026 15:19:34 -0300 Subject: [PATCH 06/51] feat(rust): declare xl_write_typed and add WriteOptions --- rust/excelreader/src/lib.rs | 27 +++++++- rust/excelreader/src/options.rs | 112 ++++++++++++++++++++++++++++++- rust/excelreader/src/workbook.rs | 6 +- rust/excelreader/src/writer.rs | 0 4 files changed, 140 insertions(+), 5 deletions(-) create mode 100644 rust/excelreader/src/writer.rs diff --git a/rust/excelreader/src/lib.rs b/rust/excelreader/src/lib.rs index 27824c8..6b5785a 100644 --- a/rust/excelreader/src/lib.rs +++ b/rust/excelreader/src/lib.rs @@ -6,9 +6,10 @@ mod error; mod options; mod temporal; pub mod workbook; +pub mod writer; pub use error::Error; -pub use options::OpenOptions; +pub use options::{OpenOptions, WriteOptions}; pub use temporal::{Date, Time, Timestamp}; use std::os::raw::{c_int, c_void}; @@ -74,6 +75,22 @@ pub struct XlOpenOptions { pub intern_strings: i32, } +/// Mirrors `xl_write_options`. Field ORDER is the C struct's, not a tidied-up version of it: with +/// `repr(C)` the 4 bytes of padding after `sheet_name_len` land exactly where a C compiler puts +/// them, giving the 32-byte x64 layout `tests/ExcelReader.NativeSmoke/smoke.c` static-asserts. +/// Build one through [`WriteOptions`] rather than by hand. +#[repr(C)] +#[derive(Clone, Copy, Debug)] +pub struct XlWriteOptions { + pub struct_size: i32, + pub sheet_name_len: i32, + pub sheet_name: *const u8, + pub csv_delimiter: i32, + pub csv_quote: i32, + pub date1904: i32, + pub use_shared_strings: i32, +} + #[repr(C)] pub struct XlColumnSpec { pub names: *const *const u8, @@ -168,6 +185,14 @@ extern "C" { ) -> c_int; pub fn xl_free_schema(schema: *mut XlInferredSchema); + pub fn xl_write_typed( + path: *const u8, + path_len: i32, + format: i32, + specs: *const XlColumnSpec, + table: *const XlTable, + options: *const XlWriteOptions, + ) -> c_int; pub fn xl_last_error_ptr(out_len: *mut i32) -> *const u8; } diff --git a/rust/excelreader/src/options.rs b/rust/excelreader/src/options.rs index 4377929..3192bc9 100644 --- a/rust/excelreader/src/options.rs +++ b/rust/excelreader/src/options.rs @@ -9,7 +9,7 @@ //! reports a rejection through `xl_last_error`, so checking them here too would give those bounds a //! second place to drift from. -use crate::{XlOpenOptions, XL_OPT_DEFAULT, XL_OPT_FALSE, XL_OPT_TRUE}; +use crate::{XL_OPT_DEFAULT, XL_OPT_FALSE, XL_OPT_TRUE, XlOpenOptions, XlWriteOptions}; /// Options for [`Workbook::open_with`](crate::workbook::Workbook::open_with) and /// [`Workbook::open_memory`](crate::workbook::Workbook::open_memory). Every field is `None` by @@ -126,6 +126,80 @@ fn opt_number(value: Option) -> T { value.unwrap_or_default() } +/// Options for [`write_columns`](crate::writer::write_columns) and +/// [`write_sheet`](crate::writer::write_sheet). Every field is `None` by default, meaning "use the +/// library default"; set only the ones you want to override. +/// +/// ```no_run +/// use excelreader::WriteOptions; +/// +/// let options = WriteOptions::new().sheet_name("Dados").use_shared_strings(true); +/// ``` +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct WriteOptions { + /// Sheet name. `None` = `"Sheet1"`. Ignored for CSV. Excel's rules (1-31 characters, none of + /// `: \ / ? * [ ]`) are enforced natively and reported through `xl_last_error`. + pub sheet_name: Option, + + // CSV only; ignored for every other format. Byte value 1-255. + pub csv_delimiter: Option, + pub csv_quote: Option, + + /// XLS/XLSB only; ignored for XLSX and CSV. Library default is `false`. + pub date1904: Option, + /// XLSX/XLSB only; ignored for XLS and CSV. Shrinks files with many repeated strings, at the + /// cost of a string table. Library default is `false`. + pub use_shared_strings: Option, +} + +impl WriteOptions { + /// Every field unset - identical to passing no options at all. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Written by hand rather than generated by `setters!`: this is the one field that is not a + /// plain `Copy` scalar. + #[must_use] + pub fn sheet_name(mut self, value: impl Into) -> Self { + self.sheet_name = Some(value.into()); + self + } + + setters! { + /// CSV only. Byte value 1-255; the library default is `,`. + csv_delimiter: i32, + /// CSV only. Byte value 1-255; the library default is `"`. + csv_quote: i32, + /// XLS/XLSB only. Library default is `false`. + date1904: bool, + /// XLSX/XLSB only. Library default is `false`. + use_shared_strings: bool, + } + + /// Lowers this into the raw ABI struct, with `struct_size` filled in. + /// + /// `pub(crate)`, unlike [`OpenOptions::to_raw`], which is public: the struct this returns holds + /// a raw pointer INTO `self.sheet_name`, so handing it to a caller who can outlive `self` is a + /// use-after-free waiting to happen. `write_columns` takes `Option<&WriteOptions>` and does the + /// lowering itself, where the borrow provably covers the FFI call. + pub(crate) fn to_raw(&self) -> XlWriteOptions { + XlWriteOptions { + struct_size: std::mem::size_of::() as i32, + sheet_name_len: self.sheet_name.as_ref().map_or(0, |n| n.len() as i32), + sheet_name: self + .sheet_name + .as_ref() + .map_or(std::ptr::null(), |n| n.as_ptr()), + csv_delimiter: opt_number(self.csv_delimiter), + csv_quote: opt_number(self.csv_quote), + date1904: opt_state(self.date1904), + use_shared_strings: opt_state(self.use_shared_strings), + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -163,4 +237,40 @@ mod tests { assert_eq!(raw.max_zip_entries, 1024); assert_eq!(raw.max_shared_string_bytes, 1 << 20); } + + #[test] + fn write_options_lower_to_the_abi_defaults() { + let raw = WriteOptions::new().to_raw(); + assert_eq!( + raw.struct_size, + std::mem::size_of::() as i32 + ); + assert!(raw.sheet_name.is_null(), "an unset sheet name means Sheet1"); + assert_eq!(raw.sheet_name_len, 0); + assert_eq!(raw.csv_delimiter, 0); + assert_eq!(raw.date1904, XL_OPT_DEFAULT); + } + + #[test] + fn write_options_lower_every_set_field() { + let options = WriteOptions::new() + .sheet_name("Dados") + .csv_delimiter(b';' as i32) + .date1904(false) + .use_shared_strings(true); + let raw = options.to_raw(); + assert_eq!(raw.sheet_name_len, 5); + assert!(!raw.sheet_name.is_null()); + assert_eq!(raw.csv_delimiter, b';' as i32); + assert_eq!(raw.date1904, XL_OPT_FALSE); + assert_eq!(raw.use_shared_strings, XL_OPT_TRUE); + } + + /// The C struct's x64 layout is pinned by XL_STATIC_ASSERT in + /// tests/ExcelReader.NativeSmoke/smoke.c (lines 66-73). A repr(C) struct that disagrees would + /// hand the native side garbage where it expects a length and a pointer. + #[test] + fn write_options_match_the_c_struct_size() { + assert_eq!(std::mem::size_of::(), 32); + } } diff --git a/rust/excelreader/src/workbook.rs b/rust/excelreader/src/workbook.rs index 1712b13..1a09a97 100644 --- a/rust/excelreader/src/workbook.rs +++ b/rust/excelreader/src/workbook.rs @@ -5,7 +5,7 @@ use crate::{ }; use std::marker::PhantomData; -fn last_error(code: i32) -> Error { +pub(crate) fn last_error(code: i32) -> Error { unsafe { let mut len: i32 = 0; let ptr = crate::xl_last_error_ptr(&mut len); @@ -19,7 +19,7 @@ fn last_error(code: i32) -> Error { } } -fn check(code: i32) -> Result<(), Error> { +pub(crate) fn check(code: i32) -> Result<(), Error> { if code == XL_OK { Ok(()) } else { @@ -37,7 +37,7 @@ fn check(code: i32) -> Result<(), Error> { /// /// The result is cached: it cannot change for the lifetime of the process, and every /// `Workbook::open` would otherwise pay an FFI call for it. -fn check_abi_version() -> Result<(), Error> { +pub(crate) fn check_abi_version() -> Result<(), Error> { use std::sync::OnceLock; static CHECKED: OnceLock> = OnceLock::new(); diff --git a/rust/excelreader/src/writer.rs b/rust/excelreader/src/writer.rs new file mode 100644 index 0000000..e69de29 From 3868b97256ca6d653d1a6ad397e793287560c6d7 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Thu, 20 Aug 2026 15:39:31 -0300 Subject: [PATCH 07/51] feat(rust): add ExcelWriter and writer::write_sheet --- rust/excelreader/src/writer.rs | 489 ++++++++++++++++++++++++++ rust/excelreader/tests/write_typed.rs | 333 ++++++++++++++++++ 2 files changed, 822 insertions(+) create mode 100644 rust/excelreader/tests/write_typed.rs diff --git a/rust/excelreader/src/writer.rs b/rust/excelreader/src/writer.rs index e69de29..ab7dcaf 100644 --- a/rust/excelreader/src/writer.rs +++ b/rust/excelreader/src/writer.rs @@ -0,0 +1,489 @@ +//! Writing a columnar table to a workbook file, through the ABI's single `xl_write_typed` export. +//! +//! Two layers sit here. [`write_columns`] takes buffers the caller already owns and hands them +//! straight to the ABI - the borrow is the whole point, and the lifetimes on [`Column`] are what +//! turn the ABI's prose contract ("every buffer is borrowed for the duration of the call") into +//! something the compiler enforces. [`write_sheet`] sits on top for callers holding a slice of +//! structs, and pays exactly one transpose to get them into columns. + +use crate::workbook::{check, check_abi_version}; +use crate::{ + Error, WriteOptions, XlColumn, XlColumnSpec, XlTable, XlWriteOptions, XL_FORMAT_AUTO, + XL_FORMAT_CSV, XL_FORMAT_XLS, XL_FORMAT_XLSB, XL_FORMAT_XLSX, XL_INVALID_ARGUMENT, XL_T_BOOL, + XL_T_DATE, XL_T_F64, XL_T_I64, XL_T_STRING, XL_T_TIME, XL_T_TIMESTAMP, +}; +use std::os::raw::c_void; + +/// One column's buffers, borrowed from the caller. Each variant's slice is that column type's +/// exact wire layout (see `excelreader.h`), so nothing is converted on the way out. +pub enum ColumnData<'a> { + /// `offsets` has `rows + 1` entries into `data`, which is every row's UTF-8 bytes + /// concatenated. Unlike a table returned by the reader, `data` need not be interior to + /// `offsets` here. + Str { + offsets: &'a [i32], + data: &'a [u8], + }, + I64(&'a [i64]), + F64(&'a [f64]), + /// One byte per row, 0 or 1 - NOT a bit-packed bitmap. + Bool(&'a [u8]), + /// Days since 1970-01-01. + Date(&'a [i32]), + /// Microseconds since midnight. + Time(&'a [i64]), + /// Microseconds since 1970-01-01T00:00:00Z. + Timestamp(&'a [i64]), +} + +impl ColumnData<'_> { + #[must_use] + pub fn len(&self) -> i64 { + match self { + // An offsets array of n + 1 entries describes n rows; an empty one describes none. + ColumnData::Str { offsets, .. } => (offsets.len().max(1) - 1) as i64, + ColumnData::I64(values) | ColumnData::Time(values) | ColumnData::Timestamp(values) => { + values.len() as i64 + } + ColumnData::F64(values) => values.len() as i64, + ColumnData::Bool(values) => values.len() as i64, + ColumnData::Date(values) => values.len() as i64, + } + } + + #[must_use] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// The `XL_T_*` tag this variant writes as. + #[must_use] + pub fn xl_type(&self) -> i32 { + match self { + ColumnData::Str { .. } => XL_T_STRING, + ColumnData::I64(_) => XL_T_I64, + ColumnData::F64(_) => XL_T_F64, + ColumnData::Bool(_) => XL_T_BOOL, + ColumnData::Date(_) => XL_T_DATE, + ColumnData::Time(_) => XL_T_TIME, + ColumnData::Timestamp(_) => XL_T_TIMESTAMP, + } + } + + fn pointers(&self) -> (*const c_void, *const u8, i64) { + match self { + ColumnData::Str { offsets, data } => ( + offsets.as_ptr().cast::(), + data.as_ptr(), + data.len() as i64, + ), + ColumnData::I64(v) | ColumnData::Time(v) | ColumnData::Timestamp(v) => { + (v.as_ptr().cast::(), std::ptr::null(), 0) + } + ColumnData::F64(v) => (v.as_ptr().cast::(), std::ptr::null(), 0), + ColumnData::Bool(v) => (v.as_ptr().cast::(), std::ptr::null(), 0), + ColumnData::Date(v) => (v.as_ptr().cast::(), std::ptr::null(), 0), + } + } +} + +/// One input column: a name, its buffers, and an optional validity bitmap. +pub struct Column<'a> { + /// The header text. `None` in EVERY column means no header row is written; mixing `Some` and + /// `None` across the set is an error, not a partial header. + pub name: Option<&'a str>, + pub data: ColumnData<'a>, + /// LSB-first bitmap, bit `r` set = row `r` is valid. `None` = the column has no nulls. + pub validity: Option<&'a [u8]>, +} + +impl Column<'_> { + fn to_raw(&self) -> XlColumn { + let (values, data, data_len) = self.data.pointers(); + XlColumn { + r#type: self.data.xl_type(), + length: self.data.len(), + values, + validity: self.validity.map_or(std::ptr::null(), <[u8]>::as_ptr), + data, + data_len, + } + } +} + +/// Infers an `XL_FORMAT_*` from a path's extension, case-insensitively. Returns +/// [`XL_FORMAT_AUTO`] when the extension is absent or unrecognized - and since `xl_write_typed` +/// rejects `AUTO` with a message of its own, an unrecognized path fails the write rather than +/// silently picking a format. +#[must_use] +pub fn format_from_path(path: &str) -> i32 { + let name = path.rsplit(['/', '\\']).next().unwrap_or(path); + let Some((_, extension)) = name.rsplit_once('.') else { + return XL_FORMAT_AUTO; + }; + match extension.to_ascii_lowercase().as_str() { + "xlsx" => XL_FORMAT_XLSX, + "xlsb" => XL_FORMAT_XLSB, + "xls" => XL_FORMAT_XLS, + "csv" => XL_FORMAT_CSV, + _ => XL_FORMAT_AUTO, + } +} + +fn invalid(message: String) -> Error { + Error { + code: XL_INVALID_ARGUMENT, + message, + } +} + +/// Returns the row count every column agreed on, or the first problem found. +/// +/// Runs to completion before anything reaches the native side. The bitmap length check is the one +/// that must live here and cannot be delegated: `xl_write_typed` takes `validity` without a length +/// and reads `(rows + 7) / 8` bytes on trust, so a short slice is a buffer overrun the ABI has no +/// way to catch. +fn validate(columns: &[Column<'_>]) -> Result { + let Some(first) = columns.first() else { + return Err(invalid( + "write_columns needs at least one column.".to_string(), + )); + }; + let rows = first.data.len(); + let has_header = first.name.is_some(); + + for (index, column) in columns.iter().enumerate() { + if column.data.len() != rows { + return Err(invalid(format!( + "every column must have the same length; column 0 has {rows} rows but column \ + {index} has {}", + column.data.len() + ))); + } + if column.name.is_some() != has_header { + return Err(invalid(format!( + "every column must have a name, or none may - xl_write_typed cannot write a \ + partial header row (column {index})" + ))); + } + if let Some(bitmap) = column.validity { + let needed = (rows as usize).div_ceil(8); + if bitmap.len() < needed { + return Err(invalid(format!( + "the validity bitmap is {} bytes, but {rows} rows need {needed} (column \ + {index})", + bitmap.len() + ))); + } + } + if let ColumnData::Str { offsets, data } = &column.data { + if data.len() > i32::MAX as usize { + return Err(invalid(format!( + "the string blob is larger than 2 GiB, which int32 offsets cannot address \ + (column {index})" + ))); + } + if offsets.len() as i64 != rows + 1 { + return Err(invalid(format!( + "a string column needs {} offsets for {rows} rows; column {index} has {}", + rows + 1, + offsets.len() + ))); + } + } + } + Ok(rows) +} + +/// Writes `columns` to `path` as a single sheet, then closes the file. One-shot: no writer handle +/// exists before or after, and every buffer reachable from `columns` and `options` is borrowed for +/// the duration of the call and never freed by the native library. +/// +/// `format` must be one of `XL_FORMAT_XLS`/`XLSX`/`XLSB`/`CSV`. [`XL_FORMAT_AUTO`] is an error: a +/// file being created has no signature bytes to sniff. Use [`format_from_path`] to infer one. +/// +/// On failure the destination may exist and be incomplete; cleaning it up is the caller's. +pub fn write_columns( + path: &str, + format: i32, + columns: &[Column<'_>], + options: Option<&WriteOptions>, +) -> Result<(), Error> { + check_abi_version()?; + let row_count = validate(columns)?; + + // Three parallel arrays that must outlive the call: each spec's `names` points at one slot of + // `names`, and its `name_lens` at one slot of `name_lens`. A temporary would dangle. + let names: Vec<*const u8> = columns + .iter() + .map(|c| c.name.map_or(std::ptr::null(), str::as_ptr)) + .collect(); + let name_lens: Vec = columns + .iter() + .map(|c| c.name.map_or(0, |n| n.len() as i32)) + .collect(); + let specs: Vec = columns + .iter() + .enumerate() + .map(|(index, column)| XlColumnSpec { + names: &names[index], + name_lens: &name_lens[index], + // Exactly one name per write spec, or none: the ABI rejects a spec carrying an alias + // list, which only exists to resolve a header on the way IN. + name_count: i32::from(column.name.is_some()), + index: 0, + r#type: column.data.xl_type(), + nullable: 0, + }) + .collect(); + let raw_columns: Vec = columns.iter().map(Column::to_raw).collect(); + + let table = XlTable { + column_count: columns.len() as i32, + row_count, + // `columns` is `*mut` in the C struct only because the reader fills one in; the writer + // takes the table as `const` and never writes through it. + columns: raw_columns.as_ptr().cast_mut(), + }; + // Lowered here rather than by the caller: the raw struct holds a pointer into + // `options.sheet_name`, and this borrow provably covers the FFI call below. + let raw_options = options.map(WriteOptions::to_raw); + let options_ptr = raw_options + .as_ref() + .map_or(std::ptr::null(), |o| o as *const XlWriteOptions); + + let status = unsafe { + crate::xl_write_typed( + path.as_ptr(), + path.len() as i32, + format, + specs.as_ptr(), + &table, + options_ptr, + ) + }; + check(status) +} + +/// The owning twin of [`ColumnData`], produced by [`ExcelWriter::to_columns`]. A transposed range +/// of structs has to own its columns somewhere; this is that somewhere. +pub enum OwnedColumnData { + Str { offsets: Vec, data: Vec }, + I64(Vec), + F64(Vec), + Bool(Vec), + Date(Vec), + Time(Vec), + Timestamp(Vec), +} + +/// One owned column. `name` is `&'static str` rather than `String` because it always comes from a +/// literal in a `#[excel(name = "...")]` attribute - keeping it borrowed means transposing a +/// million rows allocates nothing for names. +pub struct OwnedColumn { + pub name: Option<&'static str>, + pub data: OwnedColumnData, + /// LSB-first bitmap. `None` = the column has no nulls. + pub validity: Option>, +} + +impl OwnedColumn { + /// Borrows this column in the shape [`write_columns`] takes. + #[must_use] + pub fn as_column(&self) -> Column<'_> { + let data = match &self.data { + OwnedColumnData::Str { offsets, data } => ColumnData::Str { offsets, data }, + OwnedColumnData::I64(v) => ColumnData::I64(v), + OwnedColumnData::F64(v) => ColumnData::F64(v), + OwnedColumnData::Bool(v) => ColumnData::Bool(v), + OwnedColumnData::Date(v) => ColumnData::Date(v), + OwnedColumnData::Time(v) => ColumnData::Time(v), + OwnedColumnData::Timestamp(v) => ColumnData::Timestamp(v), + }; + Column { + name: self.name, + data, + validity: self.validity.as_deref(), + } + } +} + +/// Whether appending `added` bytes to a blob already `current` bytes long would push the next +/// offset past what an `int32` can hold. Split out of `push_str` so the arithmetic is testable +/// without materializing a 2 GiB buffer. +fn offset_ceiling_exceeded(current: usize, added: usize) -> bool { + current.saturating_add(added) > i32::MAX as usize +} + +/// Appends one string to a column's offsets/data pair. +/// +/// This is where the `int32` offset overflow is caught, and it has to be caught HERE rather than +/// after the fact: once `data` has grown past `i32::MAX` the offset that would record it has +/// already wrapped, and a wrapped offset is indistinguishable from a real one. +/// +/// # Errors +/// When appending `value` would push the blob past `i32::MAX` bytes. +pub fn push_str(offsets: &mut Vec, data: &mut Vec, value: &str) -> Result<(), Error> { + if offset_ceiling_exceeded(data.len(), value.len()) { + return Err(invalid( + "a string column exceeds 2 GiB, which int32 offsets cannot address.".to_string(), + )); + } + data.extend_from_slice(value.as_bytes()); + offsets.push(data.len() as i32); + Ok(()) +} + +/// Marks row `row` valid in an LSB-first bitmap. +/// +/// # Panics +/// If `validity` is shorter than `row / 8 + 1` bytes - a caller-side sizing bug, not recoverable +/// input. +pub fn set_valid(validity: &mut [u8], row: usize) { + validity[row / 8] |= 1 << (row % 8); +} + +/// Implemented by any struct [`write_sheet`] can write. Derive it with +/// `#[derive(ExcelMapper)]`, which emits this alongside the reading half, or write it by hand. +pub trait ExcelWriter: Sized { + /// Transposes `rows` into one column per field, in field order. + /// + /// # Errors + /// Only for values the ABI cannot represent: a string column past 2 GiB, or an integer that + /// does not fit `i64`. Ordinary data never fails here. + fn to_columns(rows: &[Self]) -> Result, Error>; +} + +/// Writes `rows` to `path` as a single sheet, using the same field mapping +/// [`parse_sheet`](crate::workbook::parse_sheet) reads with. +/// +/// `rows` is walked ONCE and each field appended to its own column buffer. That transpose is the +/// only copy this makes - it is what the ABI's columnar shape costs a row-shaped caller. If you +/// already hold columnar buffers, call [`write_columns`] and pay nothing. +/// +/// # Errors +/// Anything [`ExcelWriter::to_columns`] or [`write_columns`] reports. +pub fn write_sheet( + path: &str, + format: i32, + rows: &[T], + options: Option<&WriteOptions>, +) -> Result<(), Error> { + let owned = T::to_columns(rows)?; + let borrowed: Vec> = owned.iter().map(OwnedColumn::as_column).collect(); + write_columns(path, format, &borrowed, options) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn format_is_inferred_from_the_extension() { + assert_eq!(format_from_path("out.xlsx"), XL_FORMAT_XLSX); + assert_eq!(format_from_path("out.XLSB"), XL_FORMAT_XLSB); + assert_eq!(format_from_path("out.xls"), XL_FORMAT_XLS); + assert_eq!(format_from_path("out.csv"), XL_FORMAT_CSV); + assert_eq!(format_from_path("out.txt"), XL_FORMAT_AUTO); + assert_eq!(format_from_path("out"), XL_FORMAT_AUTO); + // A dot in a directory name is not an extension. + assert_eq!(format_from_path("v1.2/report"), XL_FORMAT_AUTO); + } + + #[test] + fn validate_rejects_a_short_validity_bitmap() { + let values = [1i64, 2, 3, 4, 5, 6, 7, 8, 9]; + let bitmap = [0u8]; // 9 rows need 2 bytes + let columns = [Column { + name: Some("a"), + data: ColumnData::I64(&values), + validity: Some(&bitmap), + }]; + let error = validate(&columns).expect_err("a 1-byte bitmap cannot cover 9 rows"); + assert_eq!(error.code, XL_INVALID_ARGUMENT); + } + + #[test] + fn validate_accepts_an_exactly_sized_validity_bitmap() { + let values = [1i64; 8]; + let bitmap = [0xFFu8]; + let columns = [Column { + name: Some("a"), + data: ColumnData::I64(&values), + validity: Some(&bitmap), + }]; + assert_eq!(validate(&columns).expect("8 rows fit in 1 byte"), 8); + } + + #[test] + fn validate_rejects_a_string_column_with_the_wrong_offset_count() { + let offsets = [0i32, 3]; + let columns = [Column { + name: Some("a"), + data: ColumnData::Str { + offsets: &offsets, + data: b"abc", + }, + validity: None, + }]; + // One row, so the offsets array is right - but claiming two rows' worth is not. + assert_eq!(validate(&columns).expect("1 row, 2 offsets"), 1); + + let bad = [0i32, 3, 6, 9]; + let mixed = [ + Column { + name: Some("a"), + data: ColumnData::I64(&[1, 2]), + validity: None, + }, + Column { + name: Some("b"), + data: ColumnData::Str { + offsets: &bad, + data: b"abcdefghi", + }, + validity: None, + }, + ]; + assert!( + validate(&mixed).is_err(), + "3 rows next to a 2-row column must be rejected" + ); + } + #[test] + fn push_str_appends_bytes_and_one_offset_per_value() { + let mut offsets = vec![0i32]; + let mut data = Vec::new(); + push_str(&mut offsets, &mut data, "uma").expect("a short string must fit"); + push_str(&mut offsets, &mut data, "").expect("an empty string must fit"); + push_str(&mut offsets, &mut data, "duas").expect("a short string must fit"); + assert_eq!(offsets, vec![0, 3, 3, 7]); + assert_eq!(data, b"umaduas"); + } + + /// The ceiling itself, tested without allocating 2 GiB. `push_str` is a two-liner around this + /// predicate; the predicate is where the arithmetic that could be wrong lives. + #[test] + fn the_offset_ceiling_is_exceeded_exactly_at_int32_max() { + let ceiling = i32::MAX as usize; + assert!(!offset_ceiling_exceeded(0, 0)); + assert!( + !offset_ceiling_exceeded(ceiling - 1, 1), + "landing exactly on i32::MAX fits" + ); + assert!( + offset_ceiling_exceeded(ceiling, 1), + "one byte past i32::MAX does not" + ); + assert!(offset_ceiling_exceeded(ceiling - 1, 2)); + } + + #[test] + fn set_valid_sets_the_lsb_first_bit_for_a_row() { + let mut bitmap = vec![0u8; 2]; + set_valid(&mut bitmap, 0); + set_valid(&mut bitmap, 2); + set_valid(&mut bitmap, 9); + assert_eq!(bitmap, vec![0b0000_0101, 0b0000_0010]); + } +} diff --git a/rust/excelreader/tests/write_typed.rs b/rust/excelreader/tests/write_typed.rs new file mode 100644 index 0000000..e300b93 --- /dev/null +++ b/rust/excelreader/tests/write_typed.rs @@ -0,0 +1,333 @@ +//! Round-trips through `xl_write_typed`: everything written here is read back with the same +//! crate's reader, so a layout mistake on either side shows up as a value mismatch rather than +//! as a file only Excel could judge. + +use excelreader::workbook::{parse_sheet, ExcelMapper, Workbook}; +use excelreader::writer::{Column, ColumnData, ExcelWriter, OwnedColumn, OwnedColumnData}; +use excelreader::{Date, Time, Timestamp, XL_FORMAT_CSV, XL_FORMAT_XLSX}; +use std::path::PathBuf; +use std::sync::atomic::{AtomicU32, Ordering}; + +static COUNTER: AtomicU32 = AtomicU32::new(0); + +/// A unique path under the system temp directory. `tempfile` would do this too, but a dev +/// dependency to build one path is more machinery than the job needs. +fn temp_path(name: &str) -> PathBuf { + let mut path = std::env::temp_dir(); + path.push(format!( + "excelreader-rs-{}-{}-{name}", + std::process::id(), + COUNTER.fetch_add(1, Ordering::Relaxed) + )); + path +} + +#[derive(Default, Debug, ExcelMapper)] +struct WrittenRow { + #[excel(name = "texto")] + texto: String, + #[excel(name = "inteiro")] + inteiro: i64, + #[excel(name = "numero")] + numero: f64, + #[excel(name = "ativo")] + ativo: bool, + #[excel(name = "data")] + data: Date, + #[excel(name = "hora")] + hora: Time, + #[excel(name = "instante")] + instante: Timestamp, +} + +#[test] +fn write_columns_round_trips_every_column_type() { + let offsets = [0i32, 3, 6]; + let blob = b"umadoi"; + let inteiros = [1i64, 2]; + let numeros = [0.5f64, 1.5]; + let ativos = [1u8, 0]; + let datas = [20454i32, 20455]; // 2026-01-01, 2026-01-02 + let horas = [3_600_000_000i64, 7_200_000_000]; + let instantes = [1_767_225_600_000_000i64, 1_767_312_000_000_000]; + + let columns = [ + Column { + name: Some("texto"), + data: ColumnData::Str { + offsets: &offsets, + data: blob, + }, + validity: None, + }, + Column { + name: Some("inteiro"), + data: ColumnData::I64(&inteiros), + validity: None, + }, + Column { + name: Some("numero"), + data: ColumnData::F64(&numeros), + validity: None, + }, + Column { + name: Some("ativo"), + data: ColumnData::Bool(&ativos), + validity: None, + }, + Column { + name: Some("data"), + data: ColumnData::Date(&datas), + validity: None, + }, + Column { + name: Some("hora"), + data: ColumnData::Time(&horas), + validity: None, + }, + Column { + name: Some("instante"), + data: ColumnData::Timestamp(&instantes), + validity: None, + }, + ]; + + let path = temp_path("columns.xlsx"); + let target = path.to_str().expect("temp path must be UTF-8"); + excelreader::writer::write_columns(target, XL_FORMAT_XLSX, &columns, None) + .expect("write_columns must succeed"); + + let mut workbook = Workbook::open(target).expect("the written file must open"); + let table = + parse_sheet::(&mut workbook, 1).expect("the written file must parse back"); + assert_eq!(table.len(), 2); + + let first = table.get(0).expect("row 0 must exist"); + assert_eq!(first.texto, "uma"); + assert_eq!(first.inteiro, 1); + assert_eq!(first.numero, 0.5); + assert!(first.ativo); + assert_eq!(first.data, Date::new(20454)); + assert_eq!(first.hora, Time::new(3_600_000_000)); + assert_eq!(first.instante, Timestamp::new(1_767_225_600_000_000)); + + let second = table.get(1).expect("row 1 must exist"); + assert_eq!(second.texto, "doi"); + assert!(!second.ativo); + + drop(table); + std::fs::remove_file(&path).ok(); +} + +#[test] +fn write_columns_writes_nulls_from_the_validity_bitmap() { + let valores = [10i64, 0, 30]; + // LSB-first: bits 0 and 2 set, bit 1 clear - row 1 is null. + let validity = [0b0000_0101u8]; + let columns = [Column { + name: Some("quantidade"), + data: ColumnData::I64(&valores), + validity: Some(&validity), + }]; + + #[derive(Default, Debug, ExcelMapper)] + struct NullableRow { + #[excel(name = "quantidade")] + quantidade: Option, + } + + let path = temp_path("nullable.xlsx"); + let target = path.to_str().expect("temp path must be UTF-8"); + excelreader::writer::write_columns(target, XL_FORMAT_XLSX, &columns, None) + .expect("write_columns must succeed"); + + let mut workbook = Workbook::open(target).expect("the written file must open"); + let table = + parse_sheet::(&mut workbook, 1).expect("the written file must parse back"); + assert_eq!(table.get(0).unwrap().quantidade, Some(10)); + assert_eq!(table.get(1).unwrap().quantidade, None); + assert_eq!(table.get(2).unwrap().quantidade, Some(30)); + + drop(table); + std::fs::remove_file(&path).ok(); +} + +#[test] +fn write_columns_rejects_inconsistent_input() { + let two = [1i64, 2]; + let three = [1i64, 2, 3]; + let path = temp_path("rejected.xlsx"); + let target = path.to_str().expect("temp path must be UTF-8"); + + let mismatched = [ + Column { + name: Some("a"), + data: ColumnData::I64(&two), + validity: None, + }, + Column { + name: Some("b"), + data: ColumnData::I64(&three), + validity: None, + }, + ]; + assert!(excelreader::writer::write_columns(target, XL_FORMAT_XLSX, &mismatched, None).is_err()); + + let partial_header = [ + Column { + name: Some("a"), + data: ColumnData::I64(&two), + validity: None, + }, + Column { + name: None, + data: ColumnData::I64(&two), + validity: None, + }, + ]; + assert!( + excelreader::writer::write_columns(target, XL_FORMAT_XLSX, &partial_header, None).is_err() + ); + + // Two rows need one byte of bitmap; hand it an empty slice. + let short_bitmap = [Column { + name: Some("a"), + data: ColumnData::I64(&two), + validity: Some(&[]), + }]; + assert!( + excelreader::writer::write_columns(target, XL_FORMAT_XLSX, &short_bitmap, None).is_err() + ); + + let fine = [Column { + name: Some("a"), + data: ColumnData::I64(&two), + validity: None, + }]; + assert!( + excelreader::writer::write_columns(target, excelreader::XL_FORMAT_AUTO, &fine, None) + .is_err(), + "XL_FORMAT_AUTO must be rejected: a new file has no signature bytes to sniff" + ); + + assert!(excelreader::writer::write_columns(target, XL_FORMAT_XLSX, &[], None).is_err()); +} + +#[test] +fn write_columns_honors_the_sheet_name_and_csv_format() { + let valores = [1i64]; + let columns = [Column { + name: Some("a"), + data: ColumnData::I64(&valores), + validity: None, + }]; + + let path = temp_path("named.xlsx"); + let target = path.to_str().expect("temp path must be UTF-8"); + let options = excelreader::WriteOptions::new().sheet_name("Dados"); + excelreader::writer::write_columns(target, XL_FORMAT_XLSX, &columns, Some(&options)) + .expect("write must succeed"); + + let workbook = Workbook::open(target).expect("the written file must open"); + assert_eq!(workbook.sheet_name().expect("name must read back"), "Dados"); + drop(workbook); + std::fs::remove_file(&path).ok(); + + let csv = temp_path("out.csv"); + let csv_target = csv.to_str().expect("temp path must be UTF-8"); + assert_eq!( + excelreader::writer::format_from_path(csv_target), + XL_FORMAT_CSV + ); + excelreader::writer::write_columns(csv_target, XL_FORMAT_CSV, &columns, None) + .expect("a CSV write must succeed"); + std::fs::remove_file(&csv).ok(); +} + +struct ManualRow { + nome: String, + idade: i64, + peso: Option, +} + +impl ExcelWriter for ManualRow { + fn to_columns(rows: &[Self]) -> Result, excelreader::Error> { + let n = rows.len(); + let mut nome_offsets: Vec = Vec::with_capacity(n + 1); + nome_offsets.push(0); + let mut nome_data: Vec = Vec::new(); + let mut idade: Vec = Vec::with_capacity(n); + let mut peso: Vec = Vec::with_capacity(n); + let mut peso_validity: Vec = vec![0; n.div_ceil(8)]; + + for (row, r) in rows.iter().enumerate() { + excelreader::writer::push_str(&mut nome_offsets, &mut nome_data, r.nome.as_str())?; + idade.push(r.idade); + match &r.peso { + Some(value) => { + excelreader::writer::set_valid(&mut peso_validity, row); + peso.push(*value); + } + None => peso.push(f64::default()), + } + } + + Ok(vec![ + OwnedColumn { + name: Some("nome"), + data: OwnedColumnData::Str { offsets: nome_offsets, data: nome_data }, + validity: None, + }, + OwnedColumn { + name: Some("idade"), + data: OwnedColumnData::I64(idade), + validity: None, + }, + OwnedColumn { + name: Some("peso"), + data: OwnedColumnData::F64(peso), + validity: Some(peso_validity), + }, + ]) + } +} + +#[derive(Default, Debug, ExcelMapper)] +struct ManualRowRead { + #[excel(name = "nome")] + nome: String, + #[excel(name = "idade")] + idade: i64, + #[excel(name = "peso")] + peso: Option, +} + +#[test] +fn write_sheet_round_trips_a_hand_written_excel_writer() { + let rows = vec![ + ManualRow { nome: "Ana".to_string(), idade: 30, peso: Some(62.5) }, + ManualRow { nome: "Bruno".to_string(), idade: 41, peso: None }, + ]; + + let path = temp_path("manual.xlsx"); + let target = path.to_str().expect("temp path must be UTF-8"); + excelreader::writer::write_sheet(target, XL_FORMAT_XLSX, &rows, None) + .expect("write_sheet must succeed"); + + let mut workbook = Workbook::open(target).expect("the written file must open"); + let table = + parse_sheet::(&mut workbook, 1).expect("the written file must parse back"); + assert_eq!(table.len(), 2); + + let first = table.get(0).expect("row 0 must exist"); + assert_eq!(first.nome, "Ana"); + assert_eq!(first.idade, 30); + assert_eq!(first.peso, Some(62.5)); + + let second = table.get(1).expect("row 1 must exist"); + assert_eq!(second.nome, "Bruno"); + assert_eq!(second.peso, None); + + drop(table); + std::fs::remove_file(&path).ok(); +} \ No newline at end of file From 68840ff54d11d8cb76aa82056131f2df07a4e0e6 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Thu, 20 Aug 2026 15:42:28 -0300 Subject: [PATCH 08/51] feat(rust): generate ExcelWriter from #[derive(ExcelMapper)] --- rust/excelreader-derive/src/lib.rs | 250 ++++++++++++++++++++++++++ rust/excelreader/tests/write_typed.rs | 104 ++++++++++- 2 files changed, 350 insertions(+), 4 deletions(-) diff --git a/rust/excelreader-derive/src/lib.rs b/rust/excelreader-derive/src/lib.rs index ba4fc47..521ada1 100644 --- a/rust/excelreader-derive/src/lib.rs +++ b/rust/excelreader-derive/src/lib.rs @@ -28,12 +28,44 @@ fn expand(input: DeriveInput) -> syn::Result { .map(field_binding) .collect::>>()?; + // The write half needs one set of buffer identifiers per field, so it is generated by index + // rather than field by field: the declarations, the per-row pushes, and the OwnedColumn + // construction all have to agree on the same names. + let mut declarations = Vec::new(); + let mut pushes = Vec::new(); + let mut columns = Vec::new(); + for (index, field) in fields.iter().enumerate() { + let plan = WriteField::new(index, field)?; + declarations.push(plan.declarations()); + pushes.push(plan.push()); + columns.push(plan.column()); + } + Ok(quote! { impl ::excelreader::workbook::ExcelMapper for #struct_name { fn bindings() -> ::std::vec::Vec<::excelreader::workbook::ColumnBinding> { ::std::vec![ #(#bindings),* ] } } + + impl ::excelreader::writer::ExcelWriter for #struct_name { + fn to_columns( + rows: &[Self], + ) -> ::core::result::Result< + ::std::vec::Vec<::excelreader::writer::OwnedColumn>, + ::excelreader::Error, + > { + let n = rows.len(); + #(#declarations)* + // ONE pass over the rows. Every field's append is monomorphic and inlined - there + // is no per-cell match, no boxed closure, and no trait object anywhere in here. + for (row, r) in rows.iter().enumerate() { + let _ = row; // unused when no field is Option<_> + #(#pushes)* + } + ::core::result::Result::Ok(::std::vec![ #(#columns),* ]) + } + } }) } @@ -140,6 +172,175 @@ enum FieldKind { Timestamp, } +struct WriteField { + ident: syn::Ident, + /// The PRIMARY #[excel(name = "...")] only. An alias must never reach a write spec: the ABI + /// rejects a write spec carrying more than one name. + name: LitStr, + kind: FieldKind, + is_option: bool, + values: syn::Ident, + offsets: syn::Ident, + data: syn::Ident, + validity: syn::Ident, +} + +impl WriteField { + fn new(index: usize, field: &Field) -> syn::Result { + let names = excel_names(field)?; + let (inner_ty, is_option) = unwrap_option(&field.ty); + Ok(WriteField { + ident: field.ident.clone().expect("named_fields guarantees Some"), + name: names + .into_iter() + .next() + .expect("excel_names always returns the primary name first"), + kind: FieldKind::from_type(inner_ty)?, + is_option, + values: quote::format_ident!("values_{index}"), + offsets: quote::format_ident!("offsets_{index}"), + data: quote::format_ident!("data_{index}"), + validity: quote::format_ident!("validity_{index}"), + }) + } + + fn declarations(&self) -> proc_macro2::TokenStream { + let (values, offsets, data, validity) = + (&self.values, &self.offsets, &self.data, &self.validity); + let buffers = if matches!(self.kind, FieldKind::Str) { + quote! { + let mut #offsets: ::std::vec::Vec = ::std::vec::Vec::with_capacity(n + 1); + #offsets.push(0); + let mut #data: ::std::vec::Vec = ::std::vec::Vec::new(); + } + } else { + let element = self.kind.element_type(); + quote! { + let mut #values: ::std::vec::Vec<#element> = ::std::vec::Vec::with_capacity(n); + } + }; + if !self.is_option { + return buffers; + } + // ponytail: the bitmap is materialized for every Option<_> field, even one where no row + // turns out to be None - n/8 bytes spent for nothing in that case. Tracking a `saw_none` + // flag and passing None instead would reclaim them; not worth the branch until a profile + // says otherwise. + quote! { + #buffers + let mut #validity: ::std::vec::Vec = ::std::vec![0u8; n.div_ceil(8)]; + } + } + + fn push(&self) -> proc_macro2::TokenStream { + let ident = &self.ident; + let append = self.append_tokens(); + let placeholder = self.placeholder_tokens(); + if !self.is_option { + return quote! { + { + let value = &r.#ident; + #append + } + }; + } + let validity = &self.validity; + quote! { + match &r.#ident { + ::core::option::Option::Some(value) => { + ::excelreader::writer::set_valid(&mut #validity, row); + #append + } + ::core::option::Option::None => { #placeholder } + } + } + } + + /// Appends `value: &Inner` to this field's buffers. The conversions are the exact inverses of + /// `FieldKind::value_tokens` on the read side - if one gains a type, so must the other. + fn append_tokens(&self) -> proc_macro2::TokenStream { + let (values, offsets, data) = (&self.values, &self.offsets, &self.data); + let field = self.name.value(); + match self.kind { + FieldKind::Str => quote! { + ::excelreader::writer::push_str(&mut #offsets, &mut #data, value.as_str())?; + }, + // Through TryFrom, not `as`, for the same reason the read side does: an `as` cast + // would silently wrap a u64 that does not fit an i64 column, and a writer that quietly + // changes the number it was given is worse than one that stops. + FieldKind::Int => quote! { + #values.push( + i64::try_from(*value).map_err(|_| ::excelreader::Error { + code: ::excelreader::XL_INVALID_ARGUMENT, + message: ::std::format!( + "field `{}` holds a value that does not fit an i64 column", + #field + ), + })?, + ); + }, + FieldKind::Float => quote! { #values.push(f64::from(*value)); }, + FieldKind::Bool => quote! { #values.push(u8::from(*value)); }, + FieldKind::Date => quote! { + #values.push( + ::core::convert::Into::<::excelreader::Date>::into(*value).days_since_epoch, + ); + }, + FieldKind::Time => quote! { + #values.push( + ::core::convert::Into::<::excelreader::Time>::into(*value).micros_since_midnight, + ); + }, + FieldKind::Timestamp => quote! { + #values.push( + ::core::convert::Into::<::excelreader::Timestamp>::into(*value).micros_since_epoch, + ); + }, + } + } + + /// A null row still occupies a slot in the values buffer; its cleared validity bit is what + /// marks it absent. Zero (or the empty string) is the placeholder the writer never reads. + fn placeholder_tokens(&self) -> proc_macro2::TokenStream { + let (values, offsets, data) = (&self.values, &self.offsets, &self.data); + if matches!(self.kind, FieldKind::Str) { + return quote! { ::excelreader::writer::push_str(&mut #offsets, &mut #data, "")?; }; + } + quote! { #values.push(::core::default::Default::default()); } + } + + fn column(&self) -> proc_macro2::TokenStream { + let (values, offsets, data, validity) = + (&self.values, &self.offsets, &self.data, &self.validity); + let name = &self.name; + let payload = match self.kind { + FieldKind::Str => quote! { + ::excelreader::writer::OwnedColumnData::Str { offsets: #offsets, data: #data } + }, + FieldKind::Int => quote! { ::excelreader::writer::OwnedColumnData::I64(#values) }, + FieldKind::Float => quote! { ::excelreader::writer::OwnedColumnData::F64(#values) }, + FieldKind::Bool => quote! { ::excelreader::writer::OwnedColumnData::Bool(#values) }, + FieldKind::Date => quote! { ::excelreader::writer::OwnedColumnData::Date(#values) }, + FieldKind::Time => quote! { ::excelreader::writer::OwnedColumnData::Time(#values) }, + FieldKind::Timestamp => { + quote! { ::excelreader::writer::OwnedColumnData::Timestamp(#values) } + } + }; + let validity_expr = if self.is_option { + quote! { ::core::option::Option::Some(#validity) } + } else { + quote! { ::core::option::Option::None } + }; + quote! { + ::excelreader::writer::OwnedColumn { + name: ::core::option::Option::Some(#name), + data: #payload, + validity: #validity_expr, + } + } + } +} + /// Every integer type that maps onto an `XL_T_I64` column. `i64` is included so the conversion in /// `value_tokens` stays uniform - `TryFrom for i64` exists via the blanket `From` impl and /// compiles away to nothing. @@ -222,6 +423,17 @@ impl FieldKind { } } } + /// The Rust element type of this kind's output buffer, at the column type's exact wire width. + fn element_type(&self) -> proc_macro2::TokenStream { + match self { + // Str has no single element type - it uses an offsets/data pair instead. + FieldKind::Str => quote! { u8 }, + FieldKind::Int | FieldKind::Time | FieldKind::Timestamp => quote! { i64 }, + FieldKind::Float => quote! { f64 }, + FieldKind::Bool => quote! { u8 }, + FieldKind::Date => quote! { i32 }, + } + } } #[cfg(test)] @@ -401,4 +613,42 @@ mod tests { assert!(nome_pos < nom_pos, "primary name must precede its aliases"); assert!(nom_pos < name_pos, "aliases must stay in declared order"); } + #[test] + fn generates_an_excel_writer_impl_alongside_the_mapper() { + let output = expand_str( + r#" + struct Row { + #[excel(name = "Nome", alias = "Name")] + nome: String, + #[excel(name = "Peso")] + peso: Option, + } + "#, + ) + .expect("expand must succeed"); + + assert!(output.contains("impl :: excelreader :: writer :: ExcelWriter for Row")); + assert!( + output.contains("push_str"), + "a String field must go through push_str" + ); + assert!( + output.contains("set_valid"), + "an Option field must set a validity bit" + ); + // The alias belongs to the read side only: a write spec carrying two names is rejected by + // the ABI. + let writer_half = output + .split("ExcelWriter for Row") + .nth(1) + .expect("the writer impl must be present"); + assert!( + !writer_half.contains("\"Name\""), + "an alias must never reach a write column" + ); + assert!( + writer_half.contains("\"Nome\""), + "the primary name must reach the write column" + ); + } } diff --git a/rust/excelreader/tests/write_typed.rs b/rust/excelreader/tests/write_typed.rs index e300b93..78fae9f 100644 --- a/rust/excelreader/tests/write_typed.rs +++ b/rust/excelreader/tests/write_typed.rs @@ -275,7 +275,10 @@ impl ExcelWriter for ManualRow { Ok(vec![ OwnedColumn { name: Some("nome"), - data: OwnedColumnData::Str { offsets: nome_offsets, data: nome_data }, + data: OwnedColumnData::Str { + offsets: nome_offsets, + data: nome_data, + }, validity: None, }, OwnedColumn { @@ -305,8 +308,16 @@ struct ManualRowRead { #[test] fn write_sheet_round_trips_a_hand_written_excel_writer() { let rows = vec![ - ManualRow { nome: "Ana".to_string(), idade: 30, peso: Some(62.5) }, - ManualRow { nome: "Bruno".to_string(), idade: 41, peso: None }, + ManualRow { + nome: "Ana".to_string(), + idade: 30, + peso: Some(62.5), + }, + ManualRow { + nome: "Bruno".to_string(), + idade: 41, + peso: None, + }, ]; let path = temp_path("manual.xlsx"); @@ -330,4 +341,89 @@ fn write_sheet_round_trips_a_hand_written_excel_writer() { drop(table); std::fs::remove_file(&path).ok(); -} \ No newline at end of file +} + +#[derive(Default, Debug, PartialEq, ExcelMapper)] +struct DerivedRow { + #[excel(name = "texto")] + texto: String, + #[excel(name = "inteiro", alias = "int")] + inteiro: i32, + #[excel(name = "numero")] + numero: f32, + #[excel(name = "ativo")] + ativo: bool, + #[excel(name = "data")] + data: Date, + #[excel(name = "hora")] + hora: Time, + #[excel(name = "instante")] + instante: Timestamp, + #[excel(name = "opcional")] + opcional: Option, +} + +#[test] +fn the_derive_round_trips_every_supported_field_type() { + let rows = vec![ + DerivedRow { + texto: "uma".to_string(), + inteiro: 1, + numero: 0.5, + ativo: true, + data: Date::new(20454), + hora: Time::new(3_600_000_000), + instante: Timestamp::new(1_767_225_600_000_000), + opcional: Some(7), + }, + DerivedRow { + texto: "duas".to_string(), + inteiro: 2, + numero: 1.5, + ativo: false, + data: Date::new(20455), + hora: Time::new(7_200_000_000), + instante: Timestamp::new(1_767_312_000_000_000), + opcional: None, + }, + ]; + + let path = temp_path("derived.xlsx"); + let target = path.to_str().expect("temp path must be UTF-8"); + excelreader::writer::write_sheet(target, XL_FORMAT_XLSX, &rows, None) + .expect("write_sheet must succeed"); + + let mut workbook = Workbook::open(target).expect("the written file must open"); + let table = + parse_sheet::(&mut workbook, 1).expect("the written file must parse back"); + assert_eq!(table.len(), 2); + assert_eq!(table.get(0).expect("row 0"), rows[0]); + assert_eq!(table.get(1).expect("row 1"), rows[1]); + + drop(table); + std::fs::remove_file(&path).ok(); +} + +/// The write side uses only the PRIMARY name. An alias that reached a write spec would be +/// rejected by the ABI ("must have exactly one name"), so this asserts the header actually +/// written is `inteiro`, never `int`. +#[test] +fn the_derive_writes_only_the_primary_column_name() { + #[derive(Default, Debug, ExcelMapper)] + struct AliasedRead { + #[excel(name = "inteiro")] + inteiro: i32, + } + + let rows = vec![DerivedRow::default()]; + let path = temp_path("aliased.xlsx"); + let target = path.to_str().expect("temp path must be UTF-8"); + excelreader::writer::write_sheet(target, XL_FORMAT_XLSX, &rows, None) + .expect("write_sheet must succeed"); + + let mut workbook = Workbook::open(target).expect("the written file must open"); + parse_sheet::(&mut workbook, 1) + .expect("the header must carry the primary name, not the alias"); + + std::fs::remove_file(&path).ok(); +} From bd0e96ddd1812c14915dfa466285427c021dc131 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Thu, 20 Aug 2026 15:45:25 -0300 Subject: [PATCH 09/51] bench(rust): add write benchmarks against rust_xlsxwriter --- rust/excelreader/Cargo.toml | 5 + rust/excelreader/benches/write_bench.rs | 123 ++++++++++++++++++++++++ 2 files changed, 128 insertions(+) create mode 100644 rust/excelreader/benches/write_bench.rs diff --git a/rust/excelreader/Cargo.toml b/rust/excelreader/Cargo.toml index 787276a..4f5508d 100644 --- a/rust/excelreader/Cargo.toml +++ b/rust/excelreader/Cargo.toml @@ -20,6 +20,7 @@ excelreader-derive = { path = "../excelreader-derive", version = "0.0.0" } # `default-features = false` drops chrono's `oldtime`/`clock` stack: this crate only converts # between integer counts and calendar types, and never reads the system clock or a timezone database. chrono = { version = "0.4", optional = true, default-features = false, features = ["std"] } +rust_xlsxwriter = "0.98.2" [dev-dependencies] trybuild = "1" @@ -34,5 +35,9 @@ harness = false name = "compare_bench" harness = false +[[bench]] +name = "write_bench" +harness = false + [package.metadata.docs.rs] all-features = true diff --git a/rust/excelreader/benches/write_bench.rs b/rust/excelreader/benches/write_bench.rs new file mode 100644 index 0000000..07e2a2b --- /dev/null +++ b/rust/excelreader/benches/write_bench.rs @@ -0,0 +1,123 @@ +//! Write benchmarks over tests/ExcelReader.Benchmarks/Data/65K_Records_Data.xlsx (65,535 data +//! rows, 14 columns), the same fixture the C++, Python and .NET suites use. +//! +//! WORK IS NOT MATCHED across all three groups, and the difference is structural rather than an +//! oversight: +//! +//! * `columns` hands the ABI buffers that are already columnar. Nothing is transposed, nothing +//! is copied. This is the ceiling, and no cell-at-a-time API can be compared to it fairly. +//! * `sheet` starts from a Vec and pays the row-to-column transpose. This is the +//! matched-work sibling: it does the same job rust_xlsxwriter does, from the same starting +//! shape. +//! * `rust_xlsxwriter` writes cell by cell through an API that also owns styling and formula +//! support this library does not expose. +//! +//! Read `sheet` against `rust_xlsxwriter`. Read `columns` only against `sheet`, as the cost of +//! having row-shaped data in the first place. +//! +//! Machine: state the CPU, OS and toolchain version alongside any number published from this file. + +use criterion::{black_box, criterion_group, criterion_main, Criterion}; +use excelreader::workbook::{parse_sheet, ExcelMapper, Workbook}; +use excelreader::writer::{write_columns, write_sheet, Column, ColumnData, OwnedColumn}; +use excelreader::{Date, XL_FORMAT_XLSX}; +use std::path::{Path, PathBuf}; + +const FIXTURE: &str = "../../tests/ExcelReader.Benchmarks/Data/65K_Records_Data.xlsx"; + +#[derive(Default, Clone, ExcelMapper)] +struct Row { + #[excel(name = "Region")] + region: String, + #[excel(name = "Country")] + country: String, + #[excel(name = "Item Type")] + item_type: String, + #[excel(name = "Order Date")] + order_date: Date, + #[excel(name = "Order ID")] + order_id: i64, + #[excel(name = "Units Sold")] + units_sold: i64, + #[excel(name = "Total Revenue")] + total_revenue: f64, +} + +/// Reads the fixture once into row structs. Panics rather than silently benchmarking nothing when +/// the fixture is missing or empty - a suite that measures an empty input is worse than no suite. +fn load_rows() -> Vec { + let path = Path::new(FIXTURE); + assert!( + path.exists(), + "missing benchmark fixture {FIXTURE} - run from rust/excelreader, and check the file is \ + present in the repo" + ); + let mut workbook = Workbook::open(FIXTURE).expect("the fixture must open"); + let table = parse_sheet::(&mut workbook, 1).expect("the fixture must parse"); + let rows: Vec = table.iter().collect(); + assert!(!rows.is_empty(), "the fixture parsed to zero rows"); + rows +} + +fn output_path(name: &str) -> PathBuf { + let mut path = std::env::temp_dir(); + path.push(format!("excelreader-bench-{}-{name}", std::process::id())); + path +} + +fn benchmark_write(criterion: &mut Criterion) { + let rows = load_rows(); + // Transposed once, outside the measured region: the `columns` group exists to measure the + // write, not the transpose the `sheet` group already covers. + let owned: Vec = + ::to_columns(&rows).expect("transpose must succeed"); + let borrowed: Vec> = owned.iter().map(OwnedColumn::as_column).collect(); + + let mut group = criterion.benchmark_group("write_xlsx_65k"); + group.sample_size(20); + + group.bench_function("columns", |b| { + let path = output_path("columns.xlsx"); + let target = path.to_str().expect("temp path must be UTF-8"); + b.iter(|| { + write_columns(target, XL_FORMAT_XLSX, black_box(&borrowed), None) + .expect("write_columns must succeed"); + }); + std::fs::remove_file(&path).ok(); + }); + + group.bench_function("sheet", |b| { + let path = output_path("sheet.xlsx"); + let target = path.to_str().expect("temp path must be UTF-8"); + b.iter(|| { + write_sheet(target, XL_FORMAT_XLSX, black_box(&rows), None) + .expect("write_sheet must succeed"); + }); + std::fs::remove_file(&path).ok(); + }); + + group.bench_function("rust_xlsxwriter", |b| { + let path = output_path("xlsxwriter.xlsx"); + b.iter(|| { + let mut workbook = rust_xlsxwriter::Workbook::new(); + let sheet = workbook.add_worksheet(); + for (index, row) in black_box(&rows).iter().enumerate() { + let r = (index + 1) as u32; + sheet.write_string(r, 0, &row.region).unwrap(); + sheet.write_string(r, 1, &row.country).unwrap(); + sheet.write_string(r, 2, &row.item_type).unwrap(); + sheet.write_number(r, 3, row.order_date.days_since_epoch as f64).unwrap(); + sheet.write_number(r, 4, row.order_id as f64).unwrap(); + sheet.write_number(r, 5, row.units_sold as f64).unwrap(); + sheet.write_number(r, 6, row.total_revenue).unwrap(); + } + workbook.save(&path).unwrap(); + }); + std::fs::remove_file(&path).ok(); + }); + + group.finish(); +} + +criterion_group!(benches, benchmark_write); +criterion_main!(benches); \ No newline at end of file From 6a24fa637dfae058f425ff7e112c9c039bdc0059 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Thu, 20 Aug 2026 15:48:11 -0300 Subject: [PATCH 10/51] bench(cpp): add write benchmarks for write_columns and write_sheet --- cpp/benchmarks/CMakeLists.txt | 12 +++ cpp/benchmarks/benchmark_write.cpp | 143 +++++++++++++++++++++++++++++ 2 files changed, 155 insertions(+) create mode 100644 cpp/benchmarks/benchmark_write.cpp diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index e5f160e..47451dd 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -33,6 +33,18 @@ if(WIN32) "$") endif() +add_executable(excelreader_cpp_write_benchmarks benchmark_write.cpp) +target_link_libraries(excelreader_cpp_write_benchmarks PRIVATE xl::excelreader benchmark::benchmark_main) +target_compile_definitions(excelreader_cpp_write_benchmarks PRIVATE + EXCELREADER_LARGE_FIXTURE_PATH="${EXCELREADER_LARGE_FIXTURE_PATH}") + +if(WIN32) + add_custom_command(TARGET excelreader_cpp_write_benchmarks POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "$" + "$") +endif() + # --- Comparison benchmark against xlnt (https://github.com/tfussell/xlnt) and xlsxio # (https://github.com/brechtsanders/xlsxio) - separate opt-in flag because both are much heavier # FetchContent builds (their own zip/xml stacks) than the suite above. diff --git a/cpp/benchmarks/benchmark_write.cpp b/cpp/benchmarks/benchmark_write.cpp new file mode 100644 index 0000000..9685d0f --- /dev/null +++ b/cpp/benchmarks/benchmark_write.cpp @@ -0,0 +1,143 @@ +// Write benchmarks over tests/ExcelReader.Benchmarks/Data/65K_Records_Data.xlsb (65,535 data rows), +// the same fixture the Rust, Python and .NET suites use. +// +// WORK IS NOT MATCHED between the two cases, by construction: +// +// * write_columns is handed buffers that are already columnar - nothing is transposed and +// nothing is copied. It is the ceiling. +// * write_sheet starts from a std::vector and pays the row-to-column transpose. It is what +// a row-shaped caller actually experiences, and the number to compare against any cell-at-a- +// time writer. +// +// State the CPU, OS and compiler version alongside any number published from this file. + +#include + +#include + +#include +#include +#include +#include + +struct Row +{ + std::string region; + std::string country; + std::string item_type; + std::chrono::sys_days order_date; + int64_t order_id; + int64_t units_sold; + double total_revenue; +}; + +template <> +struct xl::ExcelMapper +{ + static constexpr auto get_bindings() + { + return std::make_tuple( + xl::make_field("Region", &Row::region), + xl::make_field("Country", &Row::country), + xl::make_field("Item Type", &Row::item_type), + xl::make_field("Order Date", &Row::order_date), + xl::make_field("Order ID", &Row::order_id), + xl::make_field("Units Sold", &Row::units_sold), + xl::make_field("Total Revenue", &Row::total_revenue)); + } +}; + +// Reads the fixture once into row structs. Aborts rather than silently benchmarking an empty +// input - a suite that measures nothing is worse than no suite. +static const std::vector &fixture_rows() +{ + static const std::vector rows = [] + { + auto workbook = xl::Workbook::open(EXCELREADER_LARGE_FIXTURE_PATH); + if (!workbook.has_value()) + { + std::fprintf(stderr, "missing or unreadable fixture %s\n", EXCELREADER_LARGE_FIXTURE_PATH); + std::abort(); + } + auto table = xl::parse_sheet(*workbook); + if (!table.has_value() || table->size() == 0) + { + std::fprintf(stderr, "fixture %s parsed to zero rows\n", EXCELREADER_LARGE_FIXTURE_PATH); + std::abort(); + } + return table->to_vector(); + }(); + return rows; +} + +static std::filesystem::path bench_path(std::string_view name) +{ + return std::filesystem::temp_directory_path() / + std::filesystem::path(std::string("excelreader-bench-") + std::string(name)); +} + +static void BM_WriteSheet(benchmark::State &state) +{ + const std::vector &rows = fixture_rows(); + const std::filesystem::path path = bench_path("sheet.xlsx"); + for (auto _ : state) + { + auto result = xl::write_sheet(path.string(), XL_FORMAT_XLSX, rows); + benchmark::DoNotOptimize(result); + if (!result.has_value()) + { + state.SkipWithError("write_sheet failed"); + break; + } + } + state.SetItemsProcessed(static_cast(state.iterations()) * static_cast(rows.size())); + std::filesystem::remove(path); +} +BENCHMARK(BM_WriteSheet); + +static void BM_WriteColumns(benchmark::State &state) +{ + const std::vector &rows = fixture_rows(); + + // Transposed once, outside the measured region: this case exists to measure the write, not the + // transpose BM_WriteSheet already covers. + std::vector region_offsets{0}; + std::vector region_data; + std::vector order_ids; + std::vector units; + std::vector revenue; + order_ids.reserve(rows.size()); + units.reserve(rows.size()); + revenue.reserve(rows.size()); + region_offsets.reserve(rows.size() + 1); + for (const Row &row : rows) + { + const uint8_t *bytes = reinterpret_cast(row.region.data()); + region_data.insert(region_data.end(), bytes, bytes + row.region.size()); + region_offsets.push_back(static_cast(region_data.size())); + order_ids.push_back(row.order_id); + units.push_back(row.units_sold); + revenue.push_back(row.total_revenue); + } + + const std::array columns{ + xl::string_column("Region", region_offsets, region_data), + xl::i64_column("Order ID", order_ids), + xl::i64_column("Units Sold", units), + xl::f64_column("Total Revenue", revenue)}; + + const std::filesystem::path path = bench_path("columns.xlsx"); + for (auto _ : state) + { + auto result = xl::write_columns(path.string(), XL_FORMAT_XLSX, columns); + benchmark::DoNotOptimize(result); + if (!result.has_value()) + { + state.SkipWithError("write_columns failed"); + break; + } + } + state.SetItemsProcessed(static_cast(state.iterations()) * static_cast(rows.size())); + std::filesystem::remove(path); +} +BENCHMARK(BM_WriteColumns); \ No newline at end of file From fd3af8714a265072fe80e8a7bc9d3ceb9bbdf916 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Thu, 20 Aug 2026 15:51:39 -0300 Subject: [PATCH 11/51] @ docs: document the C++ and Rust writers The root README still claimed the writers were not exposed across the ABI, which stopped being true when the Python binding shipped write_workbook and is now wrong for all three bindings. Co-Authored-By: Claude Opus 5 @ --- README.md | 28 ++++++++++++++--- cpp/README.md | 59 ++++++++++++++++++++++++++++++++-- rust/excelreader/Cargo.toml | 2 +- rust/excelreader/README.md | 63 ++++++++++++++++++++++++++++++++++--- 4 files changed, 138 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index ad5f04c..3f48034 100644 --- a/README.md +++ b/README.md @@ -863,13 +863,13 @@ dotnet test --project tests/ExcelReader.Tests/ExcelReader.Tests.csproj --configu ## Other languages -ExcelReader ships a NativeAOT shared library with a C ABI, so non-.NET languages can read XLSX, -XLSB, XLS and CSV without a .NET runtime installed. +ExcelReader ships a NativeAOT shared library with a C ABI, so non-.NET languages can read and write +XLSX, XLSB, XLS and CSV without a .NET runtime installed. - C ABI header: [`src/ExcelReader.Native/include/excelreader.h`](src/ExcelReader.Native/include/excelreader.h) - Python package: [`python/`](python/README.md) -- C++ package: header-only CMake wrapper, `xl::Workbook`/`xl::parse_sheet` over the same ABI — see [`cpp/README.md`](cpp/README.md). -- Rust crate: safe `Workbook`/`parse_sheet` bindings, downloadable via `cargo add excelreader` — see [`rust/excelreader/README.md`](rust/excelreader/README.md). +- C++ package: header-only CMake wrapper, `xl::Workbook`/`xl::parse_sheet`/`xl::write_sheet` over the same ABI — see [`cpp/README.md`](cpp/README.md). +- Rust crate: safe `Workbook`/`parse_sheet`/`write_sheet` bindings, downloadable via `cargo add excelreader` — see [`rust/excelreader/README.md`](rust/excelreader/README.md). ```python from excelreader import open_workbook @@ -879,7 +879,25 @@ with open_workbook("book.xlsx") as workbook: print([cell.value for cell in row]) ``` -Reading only — the writers are not exposed across the ABI yet. +Writing goes through one export, `xl_write_typed`: a whole sheet in a single call, from columnar +buffers the ABI borrows rather than copies. All three bindings expose it — Python as +`write_workbook`/`write_arrow`/`write_pandas`/`write_polars`, C++ as `xl::write_columns` and +`xl::write_sheet`, Rust as `writer::write_columns` and `writer::write_sheet`. In C++ and Rust the +same struct mapping drives both directions, so reading a sheet and writing it back needs one +mapping, not two: + +```rust +use excelreader::writer::write_sheet; +use excelreader::XL_FORMAT_XLSX; + +write_sheet("out.xlsx", XL_FORMAT_XLSX, &rows, None)?; +``` + +```cpp +auto written = xl::write_sheet("out.xlsx", rows); // format inferred from the extension +``` + +Row-by-row decoded reads and the Arrow export remain Python-only. ## Contributing diff --git a/cpp/README.md b/cpp/README.md index 947a423..d1eaacd 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -1,9 +1,9 @@ # excelreader (C++) Header-only C++23 wrapper around ExcelReader's native C ABI: opening a workbook (from a path or -memory, with the full open-options surface), sheet navigation, schema inference, and schema-driven -typed table parsing. No writing, no Arrow, no row-by-row decode yet — see the root README's Python -section for what those look like. +memory, with the full open-options surface), sheet navigation, schema inference, schema-driven +typed table parsing, and schema-driven writing. No Arrow, no row-by-row decode yet — see the root +README's Python section for what those look like. ## Requirements @@ -78,6 +78,53 @@ for (const auto& column : *workbook->infer_schema(1, 100)) { Every entry point returns `std::expected` — this header throws nothing. +## Writing + +Two layers, mirroring the two on the reading side. + +`xl::write_sheet` uses the same `xl::ExcelMapper` specialization `xl::parse_sheet` reads +with, so a round trip needs one mapping, not two: + +```cpp +std::vector rows = /* ... */; +auto written = xl::write_sheet("out.xlsx", rows); // format inferred from the extension +if (!written) { + std::fprintf(stderr, "%s\n", written.error().message.c_str()); +} +``` + +If you already hold columnar buffers, `xl::write_columns` hands them to the ABI with **no copy** — +they are borrowed for the duration of the call and must outlive it: + +```cpp +std::vector ids{1, 2, 3}; +std::vector values{0.5, 1.5, 2.5}; +std::array columns{ + xl::i64_column("id", ids), + xl::f64_column("value", values)}; + +auto written = xl::write_columns("out.xlsx", XL_FORMAT_XLSX, columns); +``` + +One constructor per column type: `i64_column`, `f64_column`, `bool_column`, `date_column`, +`time_column`, `timestamp_column`, and `string_column` (which takes an `int32` offsets span of +`rows + 1` entries plus the UTF-8 blob). + +A nullable column is a values buffer plus an LSB-first validity bitmap — bit set means the row is +valid — passed as the last argument to any of those constructors. `write_columns` checks the bitmap +is long enough for the row count before calling: the ABI takes it without a length and reads +`(rows + 7) / 8` bytes on trust, so a short one would be a buffer overrun. On the struct side, +declare the field `std::optional` and `write_sheet` builds the bitmap for you. + +`xl::WriteOptions` sets the sheet name, the CSV dialect, and the XLS/XLSB and XLSX/XLSB toggles. +`XL_FORMAT_AUTO` is rejected — a file being created has no signature bytes to sniff — so +`xl::format_from_path` returning `XL_FORMAT_AUTO` for an unrecognized extension surfaces as a failed +write rather than a silently chosen format. + +`write_sheet` walks the range once and appends each field to its own column buffer, with the +per-field dispatch resolved at compile time. That transpose is the only copy it makes, and it is +what the ABI's columnar shape costs a row-shaped caller; `write_columns` pays nothing. + ## Bounds and ABI `TableView::operator[]` is unchecked, like `std::vector`'s. Use `TableView::at(row)`, which returns @@ -134,3 +181,9 @@ cmake --build build --config Release --target excelreader_cpp_benchmarks Add `-DEXCELREADER_BUILD_BENCHMARKS_COMPARE=ON -DCMAKE_POLICY_VERSION_MINIMUM=3.5` (xlnt's own `CMakeLists.txt` predates CMake's minimum-version floor) and build/run `excelreader_cpp_compare_benchmarks` for the xlnt/xlsxio comparison. + +`excelreader_cpp_write_benchmarks` (same `-DEXCELREADER_BUILD_BENCHMARKS=ON` flag) measures the two +write layers. Work is deliberately **not** matched between its two cases: `BM_WriteColumns` is +handed buffers that are already columnar and transposes nothing, while `BM_WriteSheet` starts from a +`std::vector` and pays the row-to-column transpose. Read the second as what a row-shaped caller +actually experiences; read the first only as the ceiling. diff --git a/rust/excelreader/Cargo.toml b/rust/excelreader/Cargo.toml index 4f5508d..50a470b 100644 --- a/rust/excelreader/Cargo.toml +++ b/rust/excelreader/Cargo.toml @@ -3,7 +3,7 @@ name = "excelreader" version = "0.0.0" edition = "2021" license = "MIT" -description = "Read Excel/CSV workbooks via ExcelReader's native ABI (open + schema-driven typed parse)." +description = "Read and write Excel/CSV workbooks via ExcelReader's native ABI (schema-driven typed parse and write)." repository = "https://github.com/GabrielMarquezMatte/ExcelReader" readme = "README.md" build = "build.rs" diff --git a/rust/excelreader/README.md b/rust/excelreader/README.md index 75d22c9..3af6eac 100644 --- a/rust/excelreader/README.md +++ b/rust/excelreader/README.md @@ -1,9 +1,9 @@ # excelreader (Rust) -Read Excel/CSV workbooks via ExcelReader's native ABI: opening a workbook (from a path or memory, -with the full open-options surface), sheet navigation, schema inference, and schema-driven typed -parse. No writing, no Arrow, no row-by-row decode yet - see the root README's Python section for -what those look like. +Read and write Excel/CSV workbooks via ExcelReader's native ABI: opening a workbook (from a path or +memory, with the full open-options surface), sheet navigation, schema inference, schema-driven typed +parse, and schema-driven writing. No Arrow, no row-by-row decode yet - see the root README's Python +section for what those look like. ## Usage @@ -71,6 +71,51 @@ for column in workbook.infer_schema(1, 100)? { `Workbook::open_with` takes an explicit format and `OpenOptions`; `Workbook::open_memory` reads from a byte slice. Note that format sniffing does not detect CSV - pass `XL_FORMAT_CSV` explicitly. +## Writing + +`#[derive(ExcelMapper)]` generates both halves, so the same struct reads and writes - the field +types in the table above apply unchanged: + +```rust +use excelreader::writer::write_sheet; +use excelreader::XL_FORMAT_XLSX; + +write_sheet("out.xlsx", XL_FORMAT_XLSX, &rows, None)?; +``` + +`Option` fields become an LSB-first validity bitmap: `None` writes a blank cell rather than a +zero. Only the primary `#[excel(name = "...")]` reaches the header - the `alias` list exists to +resolve a header on the way *in*, and the ABI rejects a write column carrying more than one name. + +For buffers that are already columnar, `write_columns` borrows them and copies nothing. The +lifetimes on `Column<'a>` are what turn the ABI's borrow contract into something the compiler +checks: + +```rust +use excelreader::writer::{write_columns, Column, ColumnData}; +use excelreader::XL_FORMAT_XLSX; + +let ids = [1i64, 2, 3]; +let columns = [Column { + name: Some("id"), + data: ColumnData::I64(&ids), + validity: None, +}]; +write_columns("out.xlsx", XL_FORMAT_XLSX, &columns, None)?; +``` + +`validity` is checked against the row count before the call: the ABI takes the bitmap without a +length and reads `(rows + 7) / 8` bytes on trust, so a short slice would be a buffer overrun rather +than a wrong answer. + +`WriteOptions` sets the sheet name, the CSV dialect, and the XLS/XLSB and XLSX/XLSB toggles. +`format_from_path` infers the format from an extension; it returns `XL_FORMAT_AUTO` for anything it +does not recognize, which the write then rejects - a file being created has no signature bytes to +sniff, so there is nothing to fall back on. + +`write_sheet` walks the slice once and appends each field to its own buffer, monomorphized per +field. That transpose is the only copy it makes; `write_columns` pays nothing. + ## Bounds and panics `TableView::get` returns `Option` and is `None` outside `0..len()`. The `column_*` accessors used @@ -120,6 +165,14 @@ ExcelReader is ~2.2x faster than calamine for XLSX and ~1.2x faster for XLSB on calamine is a fast, well-optimized reader in its own right, so the gap is real but not the order of magnitude seen against slower libraries. +`benches/write_bench.rs` measures the two write layers and +[rust_xlsxwriter](https://github.com/jmcnamara/rust_xlsxwriter) on the same rows. Work is **not** +matched across all three, by construction: `columns` is handed buffers that are already columnar and +transposes nothing, `sheet` starts from a `Vec` and pays the transpose, and `rust_xlsxwriter` +writes cell by cell through an API that also owns styling this library does not expose. Read `sheet` +against `rust_xlsxwriter`; read `columns` only against `sheet`, as the cost of having row-shaped +data in the first place. + Run locally: ```bash @@ -129,4 +182,4 @@ EXCELREADER_NATIVE_LIB_DIR=/path/to/native/lib/dir cargo bench -p excelreader `EXCELREADER_NATIVE_LIB_DIR` should point at a directory containing a locally-built `ExcelReader.Native.{dll,so,dylib}` - see [Build notes](#build-notes) above. Pass -`--bench parse_bench` or `--bench compare_bench` to run one suite only. +`--bench parse_bench`, `--bench compare_bench` or `--bench write_bench` to run one suite only. From 481fecd1a60b1bc5f45ad15ff1f300cf4c4e7c00 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Thu, 20 Aug 2026 16:02:40 -0300 Subject: [PATCH 12/51] bench(cpp): compare writing against xlnt and xlsxio Adds excelreader_cpp_write_compare_benchmarks under the existing EXCELREADER_BUILD_BENCHMARKS_COMPARE flag: all four cases write the same 14-column, 65,535-row shape from the same in-memory rows, so the numbers differ by writer rather than by workload. xlsxio's write side is built as its own static library against the minizip compat shim the read side already set up. Also fixes benchmark_write.cpp's BM_WriteColumns, which wrote 4 columns against BM_WriteSheet's 7. That made the columnar path look ~2x faster when most of the gap was three fewer columns of work; with both at 7 the real difference is a few percent. --- cpp/README.md | 44 ++- cpp/benchmarks/CMakeLists.txt | 37 ++- cpp/benchmarks/benchmark_write.cpp | 59 +++- cpp/benchmarks/benchmark_write_compare.cpp | 353 +++++++++++++++++++++ rust/excelreader/README.md | 29 +- rust/excelreader/benches/write_bench.rs | 2 +- 6 files changed, 496 insertions(+), 28 deletions(-) create mode 100644 cpp/benchmarks/benchmark_write_compare.cpp diff --git a/cpp/README.md b/cpp/README.md index d1eaacd..7854c25 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -182,8 +182,44 @@ Add `-DEXCELREADER_BUILD_BENCHMARKS_COMPARE=ON -DCMAKE_POLICY_VERSION_MINIMUM=3. `CMakeLists.txt` predates CMake's minimum-version floor) and build/run `excelreader_cpp_compare_benchmarks` for the xlnt/xlsxio comparison. +### Writing + `excelreader_cpp_write_benchmarks` (same `-DEXCELREADER_BUILD_BENCHMARKS=ON` flag) measures the two -write layers. Work is deliberately **not** matched between its two cases: `BM_WriteColumns` is -handed buffers that are already columnar and transposes nothing, while `BM_WriteSheet` starts from a -`std::vector` and pays the row-to-column transpose. Read the second as what a row-shaped caller -actually experiences; read the first only as the ceiling. +write layers against each other over 7 columns of the same fixture. Both cases write the same +columns, so the gap between them is only the cost of starting from row-shaped data: `BM_WriteColumns` +is handed buffers that are already columnar, while `BM_WriteSheet` starts from a +`std::vector` and pays the row-to-column transpose. + +`excelreader_cpp_write_compare_benchmarks` (under `-DEXCELREADER_BUILD_BENCHMARKS_COMPARE=ON`) puts +that against xlnt and xlsxio's writer, all four writing the full 14-column, 65,535-row shape of +`65K_Records_Data.xlsx` from the same in-memory rows: + +| Library | Wall | CPU | +|---|---:|---:| +| ExcelReader (`xl::write_columns`, pre-transposed) | 127.8 ms | 127.6 ms | +| ExcelReader (`xl::write_sheet`) | 136.5 ms | 137.5 ms | +| xlsxio (`xlsxiowrite_add_cell_*`) | 2,383.5 ms | 843.8 ms | +| xlnt (`worksheet::cell().value()` + `save()`) | 5,398.0 ms | 5,406.3 ms | + +Same machine as above; Google Benchmark's own iteration counts, no `--benchmark_repetitions` (each +iteration writes a whole 65,535-row file, so the slower cases run once or twice). + +`write_sheet` — the matched-work number, since it starts from the same `std::vector` both +competitors are handed — is ~17.5x faster than xlsxio and ~40x faster than xlnt on wall time. + +Three caveats, none of them optional when quoting these: + +- **xlsxio's wall time is two thirds I/O wait.** Its CPU time is 843.8 ms against 2,383.5 ms wall, + because it streams through a temp file; every other case here is CPU-bound (wall ≈ CPU). Against + CPU time the gap is ~6.1x, not ~17.5x. Which number is the honest one depends on what you are + asking — ~17.5x is what a caller waits, ~6.1x is what the library costs. +- **ExcelReader does slightly more work here**, not less: it attaches a number format to the two + date columns so Excel shows a date, while both competitor cases write those as bare serial + numbers. That difference favours the competitors. +- **xlnt builds a full document model** (styles, formats, formulas) before serializing, which is + more than this library exposes at all. Its number reflects a different feature set, not only a + slower path. + +The two ExcelReader cases land within ~7% of each other, which is the interesting internal result: +the row-to-column transpose is nearly free next to the cost of producing the file. Reach for +`write_columns` when your data is already columnar, but `write_sheet` is not the slow path. diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index 47451dd..e8a6d85 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -145,20 +145,43 @@ if(EXCELREADER_BUILD_BENCHMARKS_COMPARE) PUBLIC BUILD_XLSXIO_STATIC) target_link_libraries(xlsxio_read PRIVATE MINIZIP::minizip expat::expat) - add_executable(excelreader_cpp_compare_benchmarks benchmark_compare.cpp) - target_link_libraries(excelreader_cpp_compare_benchmarks PRIVATE - xl::excelreader benchmark::benchmark_main xlnt xlsxio_read) + # xlsxio's write side is a separate translation unit from its read side, with no shared state - + # it needs minizip's zip.h (same compat shim as above) but not expat, since writing parses no + # XML. + add_library(xlsxio_write STATIC "${xlsxio_SOURCE_DIR}/lib/xlsxio_write.c") + target_include_directories(xlsxio_write + PUBLIC "${xlsxio_SOURCE_DIR}/include" + PRIVATE "${xlsxio_SOURCE_DIR}/lib" "${_xlsxio_minizip_shim}") + target_compile_definitions(xlsxio_write + PRIVATE BUILD_XLSXIO USE_MINIZIP + PUBLIC BUILD_XLSXIO_STATIC) + target_link_libraries(xlsxio_write PRIVATE MINIZIP::minizip) set(EXCELREADER_XLSX_FIXTURE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../tests/ExcelReader.Benchmarks/Data/65K_Records_Data.xlsx" CACHE FILEPATH "Path to the 65K-row xlsx fixture the excelreader-vs-xlnt benchmark reads") + + add_executable(excelreader_cpp_compare_benchmarks benchmark_compare.cpp) + target_link_libraries(excelreader_cpp_compare_benchmarks PRIVATE + xl::excelreader benchmark::benchmark_main xlnt xlsxio_read) target_compile_definitions(excelreader_cpp_compare_benchmarks PRIVATE EXCELREADER_XLSX_FIXTURE_PATH="${EXCELREADER_XLSX_FIXTURE_PATH}") + # Kept a separate executable from the read comparison rather than a second .cpp in the same + # one: both files define their own FullRow plus an xl::ExcelMapper specialization for it, and + # two targets keep that unambiguous. + add_executable(excelreader_cpp_write_compare_benchmarks benchmark_write_compare.cpp) + target_link_libraries(excelreader_cpp_write_compare_benchmarks PRIVATE + xl::excelreader benchmark::benchmark_main xlnt xlsxio_write) + target_compile_definitions(excelreader_cpp_write_compare_benchmarks PRIVATE + EXCELREADER_XLSX_FIXTURE_PATH="${EXCELREADER_XLSX_FIXTURE_PATH}") + if(WIN32) - add_custom_command(TARGET excelreader_cpp_compare_benchmarks POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different - "$" - "$") + foreach(_compare_target excelreader_cpp_compare_benchmarks excelreader_cpp_write_compare_benchmarks) + add_custom_command(TARGET ${_compare_target} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "$" + "$") + endforeach() endif() endif() diff --git a/cpp/benchmarks/benchmark_write.cpp b/cpp/benchmarks/benchmark_write.cpp index 9685d0f..d704d40 100644 --- a/cpp/benchmarks/benchmark_write.cpp +++ b/cpp/benchmarks/benchmark_write.cpp @@ -1,7 +1,8 @@ // Write benchmarks over tests/ExcelReader.Benchmarks/Data/65K_Records_Data.xlsb (65,535 data rows), // the same fixture the Rust, Python and .NET suites use. // -// WORK IS NOT MATCHED between the two cases, by construction: +// Both cases write the SAME seven columns, so the only difference between them is where the data +// starts out - which is the one thing this file is measuring: // // * write_columns is handed buffers that are already columnar - nothing is transposed and // nothing is copied. It is the ceiling. @@ -9,6 +10,9 @@ // a row-shaped caller actually experiences, and the number to compare against any cell-at-a- // time writer. // +// The gap between them is therefore the cost of holding row-shaped data, not a difference in how +// much gets written. For the comparison against other libraries, see benchmark_write_compare.cpp. +// // State the CPU, OS and compiler version alongside any number published from this file. #include @@ -76,6 +80,30 @@ static std::filesystem::path bench_path(std::string_view name) std::filesystem::path(std::string("excelreader-bench-") + std::string(name)); } +namespace +{ + // The offsets/blob pair an XL_T_STRING column needs. xl::write_sheet builds one of these + // internally; the columnar benchmark below builds its own so both cases start from the same + // shape. + struct StringBuffer + { + std::vector offsets{0}; + std::vector data{}; + + void reserve(size_t count) + { + offsets.reserve(count + 1); + } + + void push(std::string_view value) + { + const uint8_t *bytes = reinterpret_cast(value.data()); + data.insert(data.end(), bytes, bytes + value.size()); + offsets.push_back(static_cast(data.size())); + } + }; +} + static void BM_WriteSheet(benchmark::State &state) { const std::vector &rows = fixture_rows(); @@ -101,27 +129,40 @@ static void BM_WriteColumns(benchmark::State &state) // Transposed once, outside the measured region: this case exists to measure the write, not the // transpose BM_WriteSheet already covers. - std::vector region_offsets{0}; - std::vector region_data; + // + // ALL SEVEN columns, the same set BM_WriteSheet writes. An earlier version of this benchmark + // wrote only four, which made it look ~2x faster when a third of that gap was simply three + // fewer columns of work. + StringBuffer region; + StringBuffer country; + StringBuffer item_type; + std::vector order_dates; std::vector order_ids; std::vector units; std::vector revenue; + region.reserve(rows.size()); + country.reserve(rows.size()); + item_type.reserve(rows.size()); + order_dates.reserve(rows.size()); order_ids.reserve(rows.size()); units.reserve(rows.size()); revenue.reserve(rows.size()); - region_offsets.reserve(rows.size() + 1); for (const Row &row : rows) { - const uint8_t *bytes = reinterpret_cast(row.region.data()); - region_data.insert(region_data.end(), bytes, bytes + row.region.size()); - region_offsets.push_back(static_cast(region_data.size())); + region.push(row.region); + country.push(row.country); + item_type.push(row.item_type); + order_dates.push_back(static_cast(row.order_date.time_since_epoch().count())); order_ids.push_back(row.order_id); units.push_back(row.units_sold); revenue.push_back(row.total_revenue); } - const std::array columns{ - xl::string_column("Region", region_offsets, region_data), + const std::array columns{ + xl::string_column("Region", region.offsets, region.data), + xl::string_column("Country", country.offsets, country.data), + xl::string_column("Item Type", item_type.offsets, item_type.data), + xl::date_column("Order Date", order_dates), xl::i64_column("Order ID", order_ids), xl::i64_column("Units Sold", units), xl::f64_column("Total Revenue", revenue)}; diff --git a/cpp/benchmarks/benchmark_write_compare.cpp b/cpp/benchmarks/benchmark_write_compare.cpp new file mode 100644 index 0000000..9e63090 --- /dev/null +++ b/cpp/benchmarks/benchmark_write_compare.cpp @@ -0,0 +1,353 @@ +// Compares ExcelReader against xlnt (https://github.com/tfussell/xlnt) and xlsxio +// (https://github.com/brechtsanders/xlsxio) WRITING the full row shape of +// tests/ExcelReader.Benchmarks/Data/65K_Records_Data.xlsx: all 14 columns, 65,535 data rows plus a +// header row. The rows are read once at startup with ExcelReader and then written back out by each +// library in turn, so all three start from exactly the same in-memory data. +// +// WORK IS NOT MATCHED across all four cases, and the mismatch runs in both directions. Read the +// table with the caveats, not without them: +// +// * BM_ExcelReader_WriteColumns is handed buffers that are already columnar. Nothing is +// transposed. No cell-at-a-time API can reach this shape at all, so it is a ceiling, not a +// competitor's number. Compare it only against BM_ExcelReader_WriteSheet. +// * BM_ExcelReader_WriteSheet starts from a std::vector - the same shape xlnt and +// xlsxio are handed - and pays the row-to-column transpose itself. THIS is the matched-work +// number, and the only one of ours that belongs next to the two competitors. +// * ExcelReader attaches a number format to the two XL_T_DATE columns (so Excel shows a date +// rather than a serial), which the two cases below do NOT do: both write those columns as bare +// numbers, the cheaper option. That difference favours the competitors. +// * xlnt builds a full in-memory document model (styles, formats, formulas) before serializing. +// It is doing more than this library exposes, and its number reflects that. +// * xlsxio streams cells straight to the ZIP, the closest thing here to matched work on the +// competitor side - the same relationship it has to ExcelReader on the reading benchmark. +// +// State the CPU, OS and compiler version alongside any number published from this file. + +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace +{ + // Same 14 columns as benchmark_compare.cpp's read-side FullRow, with owned std::string text so + // the rows survive the TableView they were parsed from. + struct FullRow + { + std::string Region; + std::string Country; + std::string ItemType; + std::string SalesChannel; + std::string OrderPriority; + std::chrono::sys_days OrderDate; + int64_t OrderId; + std::chrono::sys_days ShipDate; + int64_t UnitsSold; + double UnitPrice; + double UnitCost; + double TotalRevenue; + double TotalCost; + double TotalProfit; + }; +} + +template <> +struct xl::ExcelMapper +{ + static constexpr auto get_bindings() + { + return std::make_tuple( + xl::make_field("Region", &FullRow::Region), + xl::make_field("Country", &FullRow::Country), + xl::make_field("Item Type", &FullRow::ItemType), + xl::make_field("Sales Channel", &FullRow::SalesChannel), + xl::make_field("Order Priority", &FullRow::OrderPriority), + xl::make_field("Order Date", &FullRow::OrderDate), + xl::make_field("Order ID", &FullRow::OrderId), + xl::make_field("Ship Date", &FullRow::ShipDate), + xl::make_field("Units Sold", &FullRow::UnitsSold), + xl::make_field("Unit Price", &FullRow::UnitPrice), + xl::make_field("Unit Cost", &FullRow::UnitCost), + xl::make_field("Total Revenue", &FullRow::TotalRevenue), + xl::make_field("Total Cost", &FullRow::TotalCost), + xl::make_field("Total Profit", &FullRow::TotalProfit)); + } +}; + +namespace +{ + constexpr std::array kHeaders{ + "Region", "Country", "Item Type", "Sales Channel", "Order Priority", "Order Date", + "Order ID", "Ship Date", "Units Sold", "Unit Price", "Unit Cost", "Total Revenue", + "Total Cost", "Total Profit"}; + + // Reads the fixture once into row structs. Aborts rather than silently benchmarking an empty + // input - a suite that measures nothing is worse than no suite. + const std::vector &fixture_rows() + { + static const std::vector rows = [] + { + auto workbook = xl::Workbook::open(EXCELREADER_XLSX_FIXTURE_PATH, XL_FORMAT_XLSX); + if (!workbook.has_value()) + { + std::fprintf(stderr, "missing or unreadable fixture %s\n", EXCELREADER_XLSX_FIXTURE_PATH); + std::abort(); + } + auto table = xl::parse_sheet(*workbook); + if (!table.has_value() || table->size() == 0) + { + std::fprintf(stderr, "fixture %s parsed to zero rows\n", EXCELREADER_XLSX_FIXTURE_PATH); + std::abort(); + } + return table->to_vector(); + }(); + return rows; + } + + std::filesystem::path bench_path(std::string_view name) + { + return std::filesystem::temp_directory_path() / + std::filesystem::path(std::string("excelreader-write-compare-") + std::string(name)); + } + + int32_t days(std::chrono::sys_days value) + { + return static_cast(value.time_since_epoch().count()); + } + + // The offsets/blob pair an XL_T_STRING column needs. + struct StringBuffer + { + std::vector offsets{0}; + std::vector data{}; + + void reserve(size_t count) + { + offsets.reserve(count + 1); + } + + void push(std::string_view value) + { + const uint8_t *bytes = reinterpret_cast(value.data()); + data.insert(data.end(), bytes, bytes + value.size()); + offsets.push_back(static_cast(data.size())); + } + }; +} + +static void BM_ExcelReader_WriteSheet(benchmark::State &state) +{ + const std::vector &rows = fixture_rows(); + const std::filesystem::path path = bench_path("sheet.xlsx"); + for (auto _ : state) + { + auto result = xl::write_sheet(path.string(), XL_FORMAT_XLSX, rows); + if (!result.has_value()) + { + state.SkipWithError(result.error().message.c_str()); + return; + } + benchmark::DoNotOptimize(result); + } + state.SetItemsProcessed(static_cast(state.iterations()) * static_cast(rows.size())); + std::filesystem::remove(path); +} +BENCHMARK(BM_ExcelReader_WriteSheet); + +static void BM_ExcelReader_WriteColumns(benchmark::State &state) +{ + const std::vector &rows = fixture_rows(); + + // Transposed once, outside the measured region: this case exists to measure the write, not the + // transpose BM_ExcelReader_WriteSheet already covers. + StringBuffer region; + StringBuffer country; + StringBuffer item_type; + StringBuffer sales_channel; + StringBuffer order_priority; + std::vector order_date; + std::vector order_id; + std::vector ship_date; + std::vector units_sold; + std::vector unit_price; + std::vector unit_cost; + std::vector total_revenue; + std::vector total_cost; + std::vector total_profit; + + const size_t count = rows.size(); + for (StringBuffer *buffer : {®ion, &country, &item_type, &sales_channel, &order_priority}) + { + buffer->reserve(count); + } + order_date.reserve(count); + order_id.reserve(count); + ship_date.reserve(count); + units_sold.reserve(count); + unit_price.reserve(count); + unit_cost.reserve(count); + total_revenue.reserve(count); + total_cost.reserve(count); + total_profit.reserve(count); + + for (const FullRow &row : rows) + { + region.push(row.Region); + country.push(row.Country); + item_type.push(row.ItemType); + sales_channel.push(row.SalesChannel); + order_priority.push(row.OrderPriority); + order_date.push_back(days(row.OrderDate)); + order_id.push_back(row.OrderId); + ship_date.push_back(days(row.ShipDate)); + units_sold.push_back(row.UnitsSold); + unit_price.push_back(row.UnitPrice); + unit_cost.push_back(row.UnitCost); + total_revenue.push_back(row.TotalRevenue); + total_cost.push_back(row.TotalCost); + total_profit.push_back(row.TotalProfit); + } + + const std::array columns{ + xl::string_column(kHeaders[0], region.offsets, region.data), + xl::string_column(kHeaders[1], country.offsets, country.data), + xl::string_column(kHeaders[2], item_type.offsets, item_type.data), + xl::string_column(kHeaders[3], sales_channel.offsets, sales_channel.data), + xl::string_column(kHeaders[4], order_priority.offsets, order_priority.data), + xl::date_column(kHeaders[5], order_date), + xl::i64_column(kHeaders[6], order_id), + xl::date_column(kHeaders[7], ship_date), + xl::i64_column(kHeaders[8], units_sold), + xl::f64_column(kHeaders[9], unit_price), + xl::f64_column(kHeaders[10], unit_cost), + xl::f64_column(kHeaders[11], total_revenue), + xl::f64_column(kHeaders[12], total_cost), + xl::f64_column(kHeaders[13], total_profit)}; + + const std::filesystem::path path = bench_path("columns.xlsx"); + for (auto _ : state) + { + auto result = xl::write_columns(path.string(), XL_FORMAT_XLSX, columns); + if (!result.has_value()) + { + state.SkipWithError(result.error().message.c_str()); + return; + } + benchmark::DoNotOptimize(result); + } + state.SetItemsProcessed(static_cast(state.iterations()) * static_cast(rows.size())); + std::filesystem::remove(path); +} +BENCHMARK(BM_ExcelReader_WriteColumns); + +static void BM_Xlnt_Write(benchmark::State &state) +{ + const std::vector &rows = fixture_rows(); + const std::filesystem::path path = bench_path("xlnt.xlsx"); + for (auto _ : state) + { + xlnt::workbook workbook; + xlnt::worksheet sheet = workbook.active_sheet(); + + // xlnt's cell references are 1-based in both axes, so the header occupies row 1 and data + // starts at row 2 - the same layout ExcelReader's writer produces. + for (uint32_t column = 0; column < kHeaders.size(); ++column) + { + sheet.cell(column + 1, 1).value(kHeaders[column]); + } + + uint32_t row_index = 2; + for (const FullRow &row : rows) + { + sheet.cell(1, row_index).value(row.Region); + sheet.cell(2, row_index).value(row.Country); + sheet.cell(3, row_index).value(row.ItemType); + sheet.cell(4, row_index).value(row.SalesChannel); + sheet.cell(5, row_index).value(row.OrderPriority); + // Written as a bare serial number, not a styled date: attaching a number format here + // would be extra work ExcelReader does and this case deliberately skips. + sheet.cell(6, row_index).value(static_cast(days(row.OrderDate))); + sheet.cell(7, row_index).value(static_cast(row.OrderId)); + sheet.cell(8, row_index).value(static_cast(days(row.ShipDate))); + sheet.cell(9, row_index).value(static_cast(row.UnitsSold)); + sheet.cell(10, row_index).value(row.UnitPrice); + sheet.cell(11, row_index).value(row.UnitCost); + sheet.cell(12, row_index).value(row.TotalRevenue); + sheet.cell(13, row_index).value(row.TotalCost); + sheet.cell(14, row_index).value(row.TotalProfit); + ++row_index; + } + + workbook.save(path.string()); + benchmark::DoNotOptimize(sheet); + } + state.SetItemsProcessed(static_cast(state.iterations()) * static_cast(rows.size())); + std::filesystem::remove(path); +} +BENCHMARK(BM_Xlnt_Write); + +static void BM_Xlsxio_Write(benchmark::State &state) +{ + const std::vector &rows = fixture_rows(); + const std::filesystem::path path = bench_path("xlsxio.xlsx"); + const std::string target = path.string(); + for (auto _ : state) + { + xlsxiowriter handle = xlsxiowrite_open(target.c_str(), "Sheet1"); + if (!handle) + { + state.SkipWithError("xlsxiowrite_open failed"); + return; + } + // xlsxio infers each column's type from the first N rows unless told otherwise. Every cell + // below is written through an explicitly typed accessor, so that detection pass is pure + // overhead here - turning it off keeps this case measuring the write, not the sniffing. + xlsxiowrite_set_detection_rows(handle, 0); + + for (const char *header : kHeaders) + { + xlsxiowrite_add_column(handle, header, 0); + } + + for (const FullRow &row : rows) + { + xlsxiowrite_add_cell_string(handle, row.Region.c_str()); + xlsxiowrite_add_cell_string(handle, row.Country.c_str()); + xlsxiowrite_add_cell_string(handle, row.ItemType.c_str()); + xlsxiowrite_add_cell_string(handle, row.SalesChannel.c_str()); + xlsxiowrite_add_cell_string(handle, row.OrderPriority.c_str()); + // Bare serial numbers, same as the xlnt case above: xlsxiowrite_add_cell_datetime() + // would format them, which neither competitor case is asked to do here. + xlsxiowrite_add_cell_int(handle, days(row.OrderDate)); + xlsxiowrite_add_cell_int(handle, row.OrderId); + xlsxiowrite_add_cell_int(handle, days(row.ShipDate)); + xlsxiowrite_add_cell_int(handle, row.UnitsSold); + xlsxiowrite_add_cell_float(handle, row.UnitPrice); + xlsxiowrite_add_cell_float(handle, row.UnitCost); + xlsxiowrite_add_cell_float(handle, row.TotalRevenue); + xlsxiowrite_add_cell_float(handle, row.TotalCost); + xlsxiowrite_add_cell_float(handle, row.TotalProfit); + xlsxiowrite_next_row(handle); + } + + if (xlsxiowrite_close(handle) != 0) + { + state.SkipWithError("xlsxiowrite_close failed"); + return; + } + } + state.SetItemsProcessed(static_cast(state.iterations()) * static_cast(rows.size())); + std::filesystem::remove(path); +} +BENCHMARK(BM_Xlsxio_Write); diff --git a/rust/excelreader/README.md b/rust/excelreader/README.md index 3af6eac..821f1ac 100644 --- a/rust/excelreader/README.md +++ b/rust/excelreader/README.md @@ -137,7 +137,8 @@ through a layout that may have changed. ## Benchmarks Criterion suite in `benches/`. Measured on Windows 10 (22H2), 16 logical CPUs @ 3.39 GHz, -rustc 1.97.1 (Release), Criterion 0.5, 100 samples per benchmark (medians shown). +rustc 1.97.1 (Release), Criterion 0.5, 100 samples per benchmark (medians shown). `write_bench` +takes 20 samples instead — each of its iterations writes a whole 65,535-row file. `benches/parse_bench.rs` - `open`/`parse_sheet`/`infer_schema`, same methodology as the C++ suite: @@ -166,12 +167,26 @@ calamine is a fast, well-optimized reader in its own right, so the gap is real b of magnitude seen against slower libraries. `benches/write_bench.rs` measures the two write layers and -[rust_xlsxwriter](https://github.com/jmcnamara/rust_xlsxwriter) on the same rows. Work is **not** -matched across all three, by construction: `columns` is handed buffers that are already columnar and -transposes nothing, `sheet` starts from a `Vec` and pays the transpose, and `rust_xlsxwriter` -writes cell by cell through an API that also owns styling this library does not expose. Read `sheet` -against `rust_xlsxwriter`; read `columns` only against `sheet`, as the cost of having row-shaped -data in the first place. +[rust_xlsxwriter](https://github.com/jmcnamara/rust_xlsxwriter) writing the same 7 columns × 65,535 +rows to `.xlsx`, all three starting from the same in-memory `Vec`: + +| Benchmark | Median | +|---|---:| +| `columns` (`write_columns`, pre-transposed) | 62.1 ms | +| `sheet` (`write_sheet`, from `Vec`) | 67.2 ms | +| `rust_xlsxwriter` (cell-at-a-time) | 336.9 ms | + +`sheet` is the matched-work number — it starts from the same shape `rust_xlsxwriter` is handed and +pays the row-to-column transpose itself — and is ~5.0x faster. `columns` is a ceiling no +cell-at-a-time API can reach, since it is handed buffers that are already columnar; read it only +against `sheet`, as the cost of having row-shaped data in the first place. That cost turns out to be +about 8%: the transpose is nearly free next to producing the file. + +Two caveats, both running against the headline number rather than for it. ExcelReader does slightly +*more* work here: it attaches a number format to the date column so Excel shows a date, and writes a +header row, while the `rust_xlsxwriter` case writes that column as a bare serial number and no +header. And `rust_xlsxwriter` carries formatting and formula support this library does not expose at +all, so its number reflects a different feature set, not only a slower path. Run locally: diff --git a/rust/excelreader/benches/write_bench.rs b/rust/excelreader/benches/write_bench.rs index 07e2a2b..b00dad2 100644 --- a/rust/excelreader/benches/write_bench.rs +++ b/rust/excelreader/benches/write_bench.rs @@ -19,7 +19,7 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion}; use excelreader::workbook::{parse_sheet, ExcelMapper, Workbook}; -use excelreader::writer::{write_columns, write_sheet, Column, ColumnData, OwnedColumn}; +use excelreader::writer::{write_columns, write_sheet, Column, OwnedColumn}; use excelreader::{Date, XL_FORMAT_XLSX}; use std::path::{Path, PathBuf}; From 6c3f128a1d5ad094e6bbf4ac445e336ea8806133 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Thu, 20 Aug 2026 16:06:24 -0300 Subject: [PATCH 13/51] docs: publish the Python and C++ write benchmark numbers The Python README had no benchmark section at all; adds read and write tables from benchmarks/bench_read.py and bench_write.py, with the two DataFrame comparisons (pandas to_excel, polars write_excel) labelled as the only matched-work pairs and the Arrow-plus-pylist conversion the DataFrame helpers pay called out explicitly. The C++ standalone write table is the re-run after BM_WriteColumns was corrected to write all 7 columns. --- cpp/README.md | 8 ++++++++ python/README.md | 52 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/cpp/README.md b/cpp/README.md index 7854c25..47a4154 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -190,6 +190,14 @@ columns, so the gap between them is only the cost of starting from row-shaped da is handed buffers that are already columnar, while `BM_WriteSheet` starts from a `std::vector` and pays the row-to-column transpose. +| Benchmark | Time | Rows/s | +|---|---:|---:| +| `BM_WriteColumns` (pre-transposed) | 60.2 ms | 1.07 M/s | +| `BM_WriteSheet` (from `std::vector`) | 67.4 ms | 961 k/s | + +The transpose costs ~12% here. It is not free, but it is far from the dominant cost of producing +the file — see the comparison below, where the same two cases over 14 columns land within ~7%. + `excelreader_cpp_write_compare_benchmarks` (under `-DEXCELREADER_BUILD_BENCHMARKS_COMPARE=ON`) puts that against xlnt and xlsxio's writer, all four writing the full 14-column, 65,535-row shape of `65K_Records_Data.xlsx` from the same in-memory rows: diff --git a/python/README.md b/python/README.md index 109b8a1..7b1a885 100644 --- a/python/README.md +++ b/python/README.md @@ -243,6 +243,58 @@ options = OpenOptions( single-file batch work, not for a server already reading many files in parallel. See the root README for the measured trade. +## Benchmarks + +`benchmarks/bench_read.py` and `benchmarks/bench_write.py` over +`tests/ExcelReader.Benchmarks/Data/65K_Records_Data.xlsb` (65,535 data rows, 14 columns), the same +fixture the .NET, C++ and Rust suites use. Measured on Windows 10 (22H2), 16 logical CPUs +@ 3.39 GHz, CPython 3.14.5, 10 runs each (medians shown; `min` is in the scripts' own output). + +### Reading + +| API | Median | What it produces | +|---|---:|---| +| `parse_typed()` | 54.1 ms | typed columnar buffers, converted natively | +| `to_arrow()` | 54.3 ms | the same parse, handed to pyarrow zero-copy | +| `to_polars()` | 56.1 ms | typed columnar DataFrame, schema inferred | +| `polars.read_excel()` | 137.7 ms | typed columnar DataFrame, types inferred | +| `read_all_columnar()` | 508.5 ms | raw columnar cells, no per-cell Python objects | +| `rows()` | 1,054.6 ms | one `Cell` object per cell, streamed per row | +| `read_all()` | 1,616.3 ms | one `Cell` object per cell, all at once | + +Only the `to_polars()` / `polars.read_excel()` pair is a like-for-like comparison, and even that one +is loose: both produce a typed columnar DataFrame with inferred types, but the inference rules are +not identical. The rows above it produce different things and are listed to show what each API +costs, not to rank them — `read_all()` is 30x slower than `parse_typed()` because it materializes +917,504 Python objects, which is the price of that shape, not a slow parser. + +### Writing + +| API | Median | Output | +|---|---:|---| +| `write_workbook()` → xls | 45.6 ms | 17.7 MB | +| `write_workbook()` → csv | 64.2 ms | 8.2 MB | +| `write_workbook()` → xlsb | 74.1 ms | 5.2 MB | +| `write_workbook()` → xlsx | 129.3 ms | 5.1 MB | +| `write_polars()` → xlsx | 507.5 ms | 5.1 MB | +| `write_pandas()` → xlsx | 514.6 ms | 5.1 MB | +| `polars.DataFrame.write_excel()` | 4,487.8 ms | 5.6 MB | +| `pandas.DataFrame.to_excel()` | 6,972.0 ms | 5.5 MB | + +The two DataFrame comparisons are matched work — same DataFrame in, xlsx out both times: +`write_polars()` is ~8.8x faster than polars' own `write_excel()`, and `write_pandas()` ~13.5x +faster than `to_excel()`. Both of ours pay a conversion the raw path does not: the DataFrame goes +through Arrow and then a Python list before reaching the native columns, which is most of the gap +between the 507 ms row and the 129 ms one. Handing `write_workbook()` buffers that are already +columnar — what `parse_typed()` returns — skips all of it. + +`write_workbook(xlsx)` at 129.3 ms lands within a couple of milliseconds of the C++ binding's +`xl::write_columns` on the same 14 columns, which is the expected result: both are thin wrappers +over the same `xl_write_typed` call, and neither adds work per cell. + +`xls` being the fastest and largest is not a paradox — BIFF8 writes fixed-width records with no +compression, so it trades 3.5x the bytes for less work per cell. + ## Notes - A `Workbook` is **not** thread-safe. Use one per thread. From 5f8f4d51cb0d0222262245c7c28f9eb247ccd63c Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Thu, 20 Aug 2026 16:21:08 -0300 Subject: [PATCH 14/51] bench(cpp): add libxlsxwriter to the write comparison libxlsxwriter is a pure-C streaming writer (same author as rust_xlsxwriter), the closest competitor class to ExcelReader's own native core - worth measuring next to xlnt and xlsxio. Its CMakeLists.txt is FetchContent-friendly except for one hard dependency: find_package(ZLIB REQUIRED), with nothing to redirect it to. madler/zlib has no Config-mode package and OVERRIDE_FIND_PACKAGE only covers Config mode, so this fetches zlib itself and prepends a small generated FindZLIB.cmake to CMAKE_MODULE_PATH that aliases the target already built - a target alias rather than a hardcoded library path, which is what makes it work under a multi-config generator (Visual Studio) where the real .lib only exists once a config is chosen at build time. Verified end-to-end: configures, builds, links, and runs on Windows/MSVC. One xlsxio run in this session logged an internal zip-creation error without failing the benchmark, so that row in the README is marked provisional pending a clean rerun - not something this change caused or can fix. --- cpp/README.md | 45 ++++++++----- cpp/benchmarks/CMakeLists.txt | 66 ++++++++++++++++-- cpp/benchmarks/benchmark_write_compare.cpp | 78 +++++++++++++++++++--- 3 files changed, 158 insertions(+), 31 deletions(-) diff --git a/cpp/README.md b/cpp/README.md index 47a4154..5597aa1 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -199,30 +199,41 @@ The transpose costs ~12% here. It is not free, but it is far from the dominant c the file — see the comparison below, where the same two cases over 14 columns land within ~7%. `excelreader_cpp_write_compare_benchmarks` (under `-DEXCELREADER_BUILD_BENCHMARKS_COMPARE=ON`) puts -that against xlnt and xlsxio's writer, all four writing the full 14-column, 65,535-row shape of -`65K_Records_Data.xlsx` from the same in-memory rows: +that against xlnt, xlsxio and [libxlsxwriter](https://github.com/jmcnamara/libxlsxwriter) — same +author as rust_xlsxwriter, and, like xlsxio, a streaming C writer with no document-model overhead — +all five writing the full 14-column, 65,535-row shape of `65K_Records_Data.xlsx` from the same +in-memory rows: | Library | Wall | CPU | |---|---:|---:| | ExcelReader (`xl::write_columns`, pre-transposed) | 127.8 ms | 127.6 ms | -| ExcelReader (`xl::write_sheet`) | 136.5 ms | 137.5 ms | -| xlsxio (`xlsxiowrite_add_cell_*`) | 2,383.5 ms | 843.8 ms | -| xlnt (`worksheet::cell().value()` + `save()`) | 5,398.0 ms | 5,406.3 ms | +| ExcelReader (`xl::write_sheet`) | 136.5–140.7 ms | 137.5–140.6 ms | +| libxlsxwriter (`worksheet_write_string`/`_number`) | 1,226.9 ms | 1,234.4 ms | +| xlsxio (`xlsxiowrite_add_cell_*`) | 2,383.5–2,453.9 ms | 843.8–906.3 ms | +| xlnt (`worksheet::cell().value()` + `save()`) | 5,398.0–5,471.4 ms | 5,406.3–5,468.8 ms | Same machine as above; Google Benchmark's own iteration counts, no `--benchmark_repetitions` (each -iteration writes a whole 65,535-row file, so the slower cases run once or twice). - -`write_sheet` — the matched-work number, since it starts from the same `std::vector` both -competitors are handed — is ~17.5x faster than xlsxio and ~40x faster than xlnt on wall time. - -Three caveats, none of them optional when quoting these: - -- **xlsxio's wall time is two thirds I/O wait.** Its CPU time is 843.8 ms against 2,383.5 ms wall, - because it streams through a temp file; every other case here is CPU-bound (wall ≈ CPU). Against - CPU time the gap is ~6.1x, not ~17.5x. Which number is the honest one depends on what you are - asking — ~17.5x is what a caller waits, ~6.1x is what the library costs. +iteration writes a whole 65,535-row file, so the slower cases run once or a handful of times — the +ranges above are two separate runs, shown as a range rather than picking one arbitrarily). + +`write_sheet` — the matched-work number, since it starts from the same `std::vector` every +competitor is handed — is ~9.0x faster than libxlsxwriter, ~17.5x faster than xlsxio, and ~40x +faster than xlnt on wall time. + +Caveats, none of them optional when quoting these: + +- **One xlsxio run logged `Error creating file "xl/workbook.xml" inside zip file` mid-benchmark**, + yet `xlsxiowrite_close()` still returned success and the timing came out in the same range as a + clean run. `SkipWithError` only fires on a non-zero return, so this specific case is not proof the + written file was intact — treat the xlsxio row as provisional until a run free of that message + confirms it. +- **xlsxio's wall time is roughly two thirds I/O wait.** Its CPU time is well under half its wall + time, because it streams through a temp file; every other case here is CPU-bound (wall ≈ CPU). + Against CPU time the gap to `write_sheet` is ~6.1–6.6x, not ~17.5x. Which number is the honest one + depends on what you are asking — the wall-time ratio is what a caller waits, the CPU-time ratio is + what the library costs. - **ExcelReader does slightly more work here**, not less: it attaches a number format to the two - date columns so Excel shows a date, while both competitor cases write those as bare serial + date columns so Excel shows a date, while every competitor case writes those as bare serial numbers. That difference favours the competitors. - **xlnt builds a full document model** (styles, formats, formulas) before serializing, which is more than this library exposes at all. Its number reflects a different feature set, not only a diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index e8a6d85..3718f9b 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -45,10 +45,11 @@ if(WIN32) "$") endif() -# --- Comparison benchmark against xlnt (https://github.com/tfussell/xlnt) and xlsxio -# (https://github.com/brechtsanders/xlsxio) - separate opt-in flag because both are much heavier -# FetchContent builds (their own zip/xml stacks) than the suite above. -option(EXCELREADER_BUILD_BENCHMARKS_COMPARE "Build the excelreader-vs-xlnt-vs-xlsxio comparison benchmark" OFF) +# --- Comparison benchmark against xlnt (https://github.com/tfussell/xlnt), xlsxio +# (https://github.com/brechtsanders/xlsxio) and libxlsxwriter +# (https://github.com/jmcnamara/libxlsxwriter) - separate opt-in flag because all three are much +# heavier FetchContent builds (their own zip/xml stacks) than the suite above. +option(EXCELREADER_BUILD_BENCHMARKS_COMPARE "Build the excelreader-vs-xlnt-vs-xlsxio-vs-libxlsxwriter comparison benchmark" OFF) if(EXCELREADER_BUILD_BENCHMARKS_COMPARE) set(STATIC ON CACHE BOOL "" FORCE) set(STATIC_CRT OFF CACHE BOOL "" FORCE) @@ -157,6 +158,61 @@ if(EXCELREADER_BUILD_BENCHMARKS_COMPARE) PUBLIC BUILD_XLSXIO_STATIC) target_link_libraries(xlsxio_write PRIVATE MINIZIP::minizip) + # libxlsxwriter (https://github.com/jmcnamara/libxlsxwriter) - unlike xlsxio, its own + # CMakeLists.txt IS FetchContent-friendly enough to add_subdirectory (no REQUIRED dependency on + # anything already installed except zlib), so it is consumed as-is rather than hand-compiled. + # + # Its one hard requirement is `find_package(ZLIB "1.2.8" REQUIRED)`, called from ITS OWN + # CMakeLists.txt with no FetchContent hook of its own. madler/zlib's CMakeLists.txt has no + # matching Config-mode package (find_package(ZLIB) resolves through CMake's bundled + # Modules/FindZLIB.cmake, a Find-module, not a Config file), so FetchContent's + # OVERRIDE_FIND_PACKAGE keyword - the usual fix for "vendored project calls find_package(X + # REQUIRED) with nothing to redirect to" - does not apply to it either. + # + # The fix: fetch zlib ourselves, then place our OWN FindZLIB.cmake ahead of CMake's built-in one + # on CMAKE_MODULE_PATH. find_package(ZLIB) is a MODULE-mode call, and MODULE mode searches + # CMAKE_MODULE_PATH before it ever reaches CMake's own Modules/ directory - so ours intercepts + # the call and just aliases the target we already built. An ALIAS (not a hardcoded file path) + # is what makes this work under a multi-config generator (Visual Studio): the actual .lib file + # only exists once a config is chosen at BUILD time, so anything resolved at CONFIGURE time + # (which is when find_library would run) cannot know its path - a target alias defers that + # resolution to build time, same as every other target reference in this file. + FetchContent_Declare( + zlib + GIT_REPOSITORY https://github.com/madler/zlib.git + GIT_TAG v1.3.1 + ) + FetchContent_MakeAvailable(zlib) + # madler/zlib's CMakeLists.txt sets its include path via the directory-scoped + # include_directories(), not target_include_directories(PUBLIC/INTERFACE) - so a target built + # OUTSIDE that subdirectory (libxlsxwriter's own sources, added via a separate add_subdirectory + # below) does NOT inherit it just by linking against zlibstatic. Add it back explicitly: zlib.h + # lives in the fetched source tree, zconf.h is generated into the build tree at configure time. + target_include_directories(zlibstatic INTERFACE "${zlib_SOURCE_DIR}" "${zlib_BINARY_DIR}") + + set(_zlib_shim_dir "${CMAKE_CURRENT_BINARY_DIR}/zlib-find-module-shim") + file(MAKE_DIRECTORY "${_zlib_shim_dir}") + file(WRITE "${_zlib_shim_dir}/FindZLIB.cmake" [=[ +# Generated by cpp/benchmarks/CMakeLists.txt - redirects find_package(ZLIB) to the zlibstatic +# target this build already fetched and compiled via FetchContent, instead of searching for an +# installed system zlib. See the comment above FetchContent_Declare(zlib ...) for why this exists. +if(NOT TARGET ZLIB::ZLIB) + add_library(ZLIB::ZLIB ALIAS zlibstatic) +endif() +set(ZLIB_FOUND TRUE) +set(ZLIB_VERSION_STRING "1.3.1") +]=]) + list(PREPEND CMAKE_MODULE_PATH "${_zlib_shim_dir}") + + set(BUILD_TESTS OFF CACHE BOOL "" FORCE) + set(BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) + FetchContent_Declare( + libxlsxwriter + GIT_REPOSITORY https://github.com/jmcnamara/libxlsxwriter.git + GIT_TAG v1.2.4 + ) + FetchContent_MakeAvailable(libxlsxwriter) + set(EXCELREADER_XLSX_FIXTURE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../tests/ExcelReader.Benchmarks/Data/65K_Records_Data.xlsx" CACHE FILEPATH "Path to the 65K-row xlsx fixture the excelreader-vs-xlnt benchmark reads") @@ -172,7 +228,7 @@ if(EXCELREADER_BUILD_BENCHMARKS_COMPARE) # two targets keep that unambiguous. add_executable(excelreader_cpp_write_compare_benchmarks benchmark_write_compare.cpp) target_link_libraries(excelreader_cpp_write_compare_benchmarks PRIVATE - xl::excelreader benchmark::benchmark_main xlnt xlsxio_write) + xl::excelreader benchmark::benchmark_main xlnt xlsxio_write xlsxwriter) target_compile_definitions(excelreader_cpp_write_compare_benchmarks PRIVATE EXCELREADER_XLSX_FIXTURE_PATH="${EXCELREADER_XLSX_FIXTURE_PATH}") diff --git a/cpp/benchmarks/benchmark_write_compare.cpp b/cpp/benchmarks/benchmark_write_compare.cpp index 9e63090..3b9cd0a 100644 --- a/cpp/benchmarks/benchmark_write_compare.cpp +++ b/cpp/benchmarks/benchmark_write_compare.cpp @@ -1,25 +1,30 @@ -// Compares ExcelReader against xlnt (https://github.com/tfussell/xlnt) and xlsxio -// (https://github.com/brechtsanders/xlsxio) WRITING the full row shape of +// Compares ExcelReader against xlnt (https://github.com/tfussell/xlnt), xlsxio +// (https://github.com/brechtsanders/xlsxio) and libxlsxwriter +// (https://github.com/jmcnamara/libxlsxwriter) WRITING the full row shape of // tests/ExcelReader.Benchmarks/Data/65K_Records_Data.xlsx: all 14 columns, 65,535 data rows plus a // header row. The rows are read once at startup with ExcelReader and then written back out by each -// library in turn, so all three start from exactly the same in-memory data. +// library in turn, so all four start from exactly the same in-memory data. // -// WORK IS NOT MATCHED across all four cases, and the mismatch runs in both directions. Read the +// WORK IS NOT MATCHED across all five cases, and the mismatch runs in both directions. Read the // table with the caveats, not without them: // // * BM_ExcelReader_WriteColumns is handed buffers that are already columnar. Nothing is // transposed. No cell-at-a-time API can reach this shape at all, so it is a ceiling, not a // competitor's number. Compare it only against BM_ExcelReader_WriteSheet. -// * BM_ExcelReader_WriteSheet starts from a std::vector - the same shape xlnt and -// xlsxio are handed - and pays the row-to-column transpose itself. THIS is the matched-work -// number, and the only one of ours that belongs next to the two competitors. +// * BM_ExcelReader_WriteSheet starts from a std::vector - the same shape every +// competitor below is handed - and pays the row-to-column transpose itself. THIS is the +// matched-work number, and the only one of ours that belongs next to the competitors. // * ExcelReader attaches a number format to the two XL_T_DATE columns (so Excel shows a date -// rather than a serial), which the two cases below do NOT do: both write those columns as bare -// numbers, the cheaper option. That difference favours the competitors. +// rather than a serial), which every case below does NOT do: all three competitors write those +// columns as bare numbers, the cheaper option. That difference favours the competitors. // * xlnt builds a full in-memory document model (styles, formats, formulas) before serializing. // It is doing more than this library exposes, and its number reflects that. // * xlsxio streams cells straight to the ZIP, the closest thing here to matched work on the // competitor side - the same relationship it has to ExcelReader on the reading benchmark. +// * libxlsxwriter is also a straight streaming writer with no document-model overhead, same +// class of competitor as xlsxio - it is the one most worth comparing BM_ExcelReader_WriteSheet +// against, being C rather than C++ and, like ExcelReader's own core, built for throughput +// rather than a full object model. // // State the CPU, OS and compiler version alongside any number published from this file. @@ -28,6 +33,7 @@ #include #include #include +#include #include #include @@ -351,3 +357,57 @@ static void BM_Xlsxio_Write(benchmark::State &state) std::filesystem::remove(path); } BENCHMARK(BM_Xlsxio_Write); + +static void BM_Libxlsxwriter_Write(benchmark::State &state) +{ + const std::vector &rows = fixture_rows(); + const std::filesystem::path path = bench_path("libxlsxwriter.xlsx"); + const std::string target = path.string(); + for (auto _ : state) + { + lxw_workbook *workbook = workbook_new(target.c_str()); + if (!workbook) + { + state.SkipWithError("workbook_new failed"); + return; + } + lxw_worksheet *sheet = workbook_add_worksheet(workbook, nullptr); + + // Row/column indices are 0-based here, unlike xlnt's cell() above. + for (lxw_col_t column = 0; column < static_cast(kHeaders.size()); ++column) + { + worksheet_write_string(sheet, 0, column, kHeaders[column], nullptr); + } + + lxw_row_t row_index = 1; + for (const FullRow &row : rows) + { + worksheet_write_string(sheet, row_index, 0, row.Region.c_str(), nullptr); + worksheet_write_string(sheet, row_index, 1, row.Country.c_str(), nullptr); + worksheet_write_string(sheet, row_index, 2, row.ItemType.c_str(), nullptr); + worksheet_write_string(sheet, row_index, 3, row.SalesChannel.c_str(), nullptr); + worksheet_write_string(sheet, row_index, 4, row.OrderPriority.c_str(), nullptr); + // Bare serial numbers, same as the xlnt and xlsxio cases above: a formatted date write + // would be extra work neither of those pays either. + worksheet_write_number(sheet, row_index, 5, static_cast(days(row.OrderDate)), nullptr); + worksheet_write_number(sheet, row_index, 6, static_cast(row.OrderId), nullptr); + worksheet_write_number(sheet, row_index, 7, static_cast(days(row.ShipDate)), nullptr); + worksheet_write_number(sheet, row_index, 8, static_cast(row.UnitsSold), nullptr); + worksheet_write_number(sheet, row_index, 9, row.UnitPrice, nullptr); + worksheet_write_number(sheet, row_index, 10, row.UnitCost, nullptr); + worksheet_write_number(sheet, row_index, 11, row.TotalRevenue, nullptr); + worksheet_write_number(sheet, row_index, 12, row.TotalCost, nullptr); + worksheet_write_number(sheet, row_index, 13, row.TotalProfit, nullptr); + ++row_index; + } + + if (workbook_close(workbook) != LXW_NO_ERROR) + { + state.SkipWithError("workbook_close failed"); + return; + } + } + state.SetItemsProcessed(static_cast(state.iterations()) * static_cast(rows.size())); + std::filesystem::remove(path); +} +BENCHMARK(BM_Libxlsxwriter_Write); From 0988ea8d75b3b16b826ce8dd0300b9f7e7b77710 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Thu, 20 Aug 2026 17:20:52 -0300 Subject: [PATCH 15/51] bench(cpp): compare against DuckDB via its excel extension Adds BM_DuckDB_Xlsx_Full to the read comparison and BM_DuckDB_Write to the write comparison, both under the existing EXCELREADER_BUILD_BENCHMARKS_COMPARE flag. DuckDB is fetched as a prebuilt library (libduckdb-{windows,osx,linux}-*. zip from its GitHub Releases) rather than built from source - its amalgamated source is enormous and would dwarf every other FetchContent build in this file, and DuckDB already publishes prebuilt binaries per platform the same way this project's own excelreader-native-*.{dll,so, dylib} assets are consumed (see cmake/FetchNativeLib.cmake). Read side runs a single SQL aggregate over read_xlsx() - the idiomatic way to make a SQL engine touch every cell, not a workaround. Write side loads rows via DuckDB's Appender API before the timed region, then times only COPY ... TO ... WITH (FORMAT xlsx), matching the "transpose outside the loop" treatment BM_ExcelReader_WriteColumns already gets. Every API call (Connection::Query, MaterializedQueryResult::GetValue, Appender::AppendRow, date_t's day-count constructor) was checked against DuckDB's actual headers and test suite before writing this, not recalled from memory - not yet build-verified end to end. --- cpp/benchmarks/CMakeLists.txt | 57 ++++++++++++++- cpp/benchmarks/benchmark_compare.cpp | 70 +++++++++++++++--- cpp/benchmarks/benchmark_write_compare.cpp | 84 ++++++++++++++++++++-- 3 files changed, 192 insertions(+), 19 deletions(-) diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index 3718f9b..a0e14f8 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -49,7 +49,7 @@ endif() # (https://github.com/brechtsanders/xlsxio) and libxlsxwriter # (https://github.com/jmcnamara/libxlsxwriter) - separate opt-in flag because all three are much # heavier FetchContent builds (their own zip/xml stacks) than the suite above. -option(EXCELREADER_BUILD_BENCHMARKS_COMPARE "Build the excelreader-vs-xlnt-vs-xlsxio-vs-libxlsxwriter comparison benchmark" OFF) +option(EXCELREADER_BUILD_BENCHMARKS_COMPARE "Build the excelreader-vs-xlnt-vs-xlsxio-vs-libxlsxwriter-vs-duckdb comparison benchmark" OFF) if(EXCELREADER_BUILD_BENCHMARKS_COMPARE) set(STATIC ON CACHE BOOL "" FORCE) set(STATIC_CRT OFF CACHE BOOL "" FORCE) @@ -213,13 +213,61 @@ set(ZLIB_VERSION_STRING "1.3.1") ) FetchContent_MakeAvailable(libxlsxwriter) + # DuckDB (https://github.com/duckdb/duckdb) - via its "excel" extension, DuckDB reads and + # writes .xlsx through read_xlsx()/COPY ... TO ... WITH (FORMAT xlsx). Unlike xlnt/xlsxio/ + # libxlsxwriter, this is NOT built from source: DuckDB publishes a prebuilt libduckdb.{dll,so, + # dylib} + duckdb.h/duckdb.hpp per platform as GitHub Release assets (same shape as this + # project's own excelreader-native-*.{dll,so,dylib} - see cmake/FetchNativeLib.cmake), so it is + # downloaded the same way rather than compiled - DuckDB's amalgamated source is enormous and + # would dwarf every other FetchContent build in this file. + # + # On Windows the zip already ships a real import library (duckdb.lib) - no lib.exe/dlltool + # workaround needed, unlike this project's own native library, which NativeAOT publishes + # without one. + if(WIN32) + set(_duckdb_asset "libduckdb-windows-amd64.zip") + set(_duckdb_lib_file "duckdb.dll") + set(_duckdb_implib_file "duckdb.lib") + elseif(APPLE) + set(_duckdb_asset "libduckdb-osx-universal.zip") + set(_duckdb_lib_file "libduckdb.dylib") + else() + set(_duckdb_asset "libduckdb-linux-amd64.zip") + set(_duckdb_lib_file "libduckdb.so") + endif() + + set(_duckdb_version "1.5.5") + set(_duckdb_dir "${CMAKE_BINARY_DIR}/duckdb-download") + set(_duckdb_zip "${_duckdb_dir}/${_duckdb_asset}") + if(NOT EXISTS "${_duckdb_dir}/${_duckdb_lib_file}") + file(MAKE_DIRECTORY "${_duckdb_dir}") + set(_duckdb_url "https://github.com/duckdb/duckdb/releases/download/v${_duckdb_version}/${_duckdb_asset}") + message(STATUS "Downloading DuckDB: ${_duckdb_url}") + file(DOWNLOAD "${_duckdb_url}" "${_duckdb_zip}" STATUS _duckdb_status) + list(GET _duckdb_status 0 _duckdb_code) + if(NOT _duckdb_code EQUAL 0) + file(REMOVE "${_duckdb_zip}") + list(GET _duckdb_status 1 _duckdb_message) + message(FATAL_ERROR "Failed to download ${_duckdb_url}: ${_duckdb_message}") + endif() + file(ARCHIVE_EXTRACT INPUT "${_duckdb_zip}" DESTINATION "${_duckdb_dir}") + endif() + + add_library(duckdb SHARED IMPORTED GLOBAL) + set_target_properties(duckdb PROPERTIES + IMPORTED_LOCATION "${_duckdb_dir}/${_duckdb_lib_file}" + INTERFACE_INCLUDE_DIRECTORIES "${_duckdb_dir}") + if(WIN32) + set_target_properties(duckdb PROPERTIES IMPORTED_IMPLIB "${_duckdb_dir}/${_duckdb_implib_file}") + endif() + set(EXCELREADER_XLSX_FIXTURE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../tests/ExcelReader.Benchmarks/Data/65K_Records_Data.xlsx" CACHE FILEPATH "Path to the 65K-row xlsx fixture the excelreader-vs-xlnt benchmark reads") add_executable(excelreader_cpp_compare_benchmarks benchmark_compare.cpp) target_link_libraries(excelreader_cpp_compare_benchmarks PRIVATE - xl::excelreader benchmark::benchmark_main xlnt xlsxio_read) + xl::excelreader benchmark::benchmark_main xlnt xlsxio_read duckdb) target_compile_definitions(excelreader_cpp_compare_benchmarks PRIVATE EXCELREADER_XLSX_FIXTURE_PATH="${EXCELREADER_XLSX_FIXTURE_PATH}") @@ -228,7 +276,7 @@ set(ZLIB_VERSION_STRING "1.3.1") # two targets keep that unambiguous. add_executable(excelreader_cpp_write_compare_benchmarks benchmark_write_compare.cpp) target_link_libraries(excelreader_cpp_write_compare_benchmarks PRIVATE - xl::excelreader benchmark::benchmark_main xlnt xlsxio_write xlsxwriter) + xl::excelreader benchmark::benchmark_main xlnt xlsxio_write xlsxwriter duckdb) target_compile_definitions(excelreader_cpp_write_compare_benchmarks PRIVATE EXCELREADER_XLSX_FIXTURE_PATH="${EXCELREADER_XLSX_FIXTURE_PATH}") @@ -237,6 +285,9 @@ set(ZLIB_VERSION_STRING "1.3.1") add_custom_command(TARGET ${_compare_target} POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different "$" + "$" + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "$" "$") endforeach() endif() diff --git a/cpp/benchmarks/benchmark_compare.cpp b/cpp/benchmarks/benchmark_compare.cpp index c8fd28e..a484688 100644 --- a/cpp/benchmarks/benchmark_compare.cpp +++ b/cpp/benchmarks/benchmark_compare.cpp @@ -1,19 +1,22 @@ -// Compares ExcelReader against xlnt (https://github.com/tfussell/xlnt) and xlsxio -// (https://github.com/brechtsanders/xlsxio) reading the full row shape of -// tests/ExcelReader.Benchmarks/Data/65K_Records_Data.xlsx: all 14 columns, 65,535 data rows. -// Neither competitor reads .xlsb, so this fixture is xlsx-only, unlike the RealExcel.xlsb fixture -// the other benchmarks use. +// Compares ExcelReader against xlnt (https://github.com/tfussell/xlnt), xlsxio +// (https://github.com/brechtsanders/xlsxio) and DuckDB's (https://github.com/duckdb/duckdb) +// "excel" extension reading the full row shape of +// tests/ExcelReader.Benchmarks/Data/65K_Records_Data.xlsx: all 14 columns, 65,535 data rows. None +// of the three competitors reads .xlsb, so this fixture is xlsx-only, unlike the RealExcel.xlsb +// fixture the other benchmarks use. // -// All three sides decode every cell into an owned value (std::string for text, matching xlnt's -// cell::to_string() and xlsxio's xlsxioread_sheet_next_cell_string()) and fold it into one -// accumulator - the ExcelReader side uses std::string bindings rather than the zero-copy -// std::string_view used elsewhere in this suite, so no side gets an allocation-free advantage the -// others can't take. Same methodology as BenchmarkAccumulators.cs (the .NET benchmark suite's -// ExcelReader-vs-Sylvan comparison). +// All four sides decode every cell into an owned value and fold it into one accumulator - the +// ExcelReader side uses std::string bindings rather than the zero-copy std::string_view used +// elsewhere in this suite, so no side gets an allocation-free advantage the others can't take. +// DuckDB's case expresses the same accumulator as a single SQL aggregate query rather than a C++ +// loop, which is the idiomatic way to make a SQL engine touch every cell, not a concession to it. +// Same methodology as BenchmarkAccumulators.cs (the .NET benchmark suite's ExcelReader-vs-Sylvan +// comparison). #include #include +#include #include #include @@ -332,3 +335,48 @@ static void BM_Xlsxio_Xlsx_Full(benchmark::State &state) } } BENCHMARK(BM_Xlsxio_Xlsx_Full); + +// DuckDB (https://github.com/duckdb/duckdb) reads via its "excel" extension's read_xlsx() table +// function, invoked here as a single aggregate query rather than pulled apart row by row in C++: +// DuckDB is a SQL engine, and an aggregate over every column is the idiomatic way to make it +// decode every cell, not an artificial concession to it. The expression matches +// accumulate_full_row() exactly - same columns, same "text length, numeric value" split, same +// epoch-days conversion for the two date columns - so the three sides remain the same "touch every +// cell into one accumulator" methodology, just expressed once in SQL instead of once per row. +static void BM_DuckDB_Xlsx_Full(benchmark::State &state) +{ + duckdb::DuckDB db(nullptr); + duckdb::Connection con(db); + + // INSTALL pulls the extension from DuckDB's extension repository on first use and caches it + // locally afterward; done once per process, outside the timed region. + auto setup = con.Query("INSTALL excel; LOAD excel;"); + if (setup->HasError()) + { + state.SkipWithError(setup->GetError().c_str()); + return; + } + + const std::string query = + "SELECT sum(length(\"Region\")) + sum(length(\"Country\")) + sum(length(\"Item Type\")) + " + "sum(length(\"Sales Channel\")) + sum(length(\"Order Priority\")) + " + "sum(CAST(\"Order Date\" - DATE '1970-01-01' AS BIGINT)) + sum(\"Order ID\") + " + "sum(CAST(\"Ship Date\" - DATE '1970-01-01' AS BIGINT)) + sum(\"Units Sold\") + " + "sum(CAST(\"Unit Price\" AS BIGINT)) + sum(CAST(\"Unit Cost\" AS BIGINT)) + " + "sum(CAST(\"Total Revenue\" AS BIGINT)) + sum(CAST(\"Total Cost\" AS BIGINT)) + " + "sum(CAST(\"Total Profit\" AS BIGINT)) AS acc FROM read_xlsx('" + + std::string(EXCELREADER_XLSX_FIXTURE_PATH) + "', header = true)"; + + for (auto _ : state) + { + auto result = con.Query(query); + if (result->HasError()) + { + state.SkipWithError(result->GetError().c_str()); + return; + } + int64_t acc = result->GetValue(0, 0); + benchmark::DoNotOptimize(acc); + } +} +BENCHMARK(BM_DuckDB_Xlsx_Full); diff --git a/cpp/benchmarks/benchmark_write_compare.cpp b/cpp/benchmarks/benchmark_write_compare.cpp index 3b9cd0a..c2895f8 100644 --- a/cpp/benchmarks/benchmark_write_compare.cpp +++ b/cpp/benchmarks/benchmark_write_compare.cpp @@ -1,11 +1,12 @@ // Compares ExcelReader against xlnt (https://github.com/tfussell/xlnt), xlsxio -// (https://github.com/brechtsanders/xlsxio) and libxlsxwriter -// (https://github.com/jmcnamara/libxlsxwriter) WRITING the full row shape of +// (https://github.com/brechtsanders/xlsxio), libxlsxwriter +// (https://github.com/jmcnamara/libxlsxwriter) and DuckDB's (https://github.com/duckdb/duckdb) +// "excel" extension WRITING the full row shape of // tests/ExcelReader.Benchmarks/Data/65K_Records_Data.xlsx: all 14 columns, 65,535 data rows plus a // header row. The rows are read once at startup with ExcelReader and then written back out by each -// library in turn, so all four start from exactly the same in-memory data. +// library in turn, so all five start from exactly the same in-memory data. // -// WORK IS NOT MATCHED across all five cases, and the mismatch runs in both directions. Read the +// WORK IS NOT MATCHED across all six cases, and the mismatch runs in both directions. Read the // table with the caveats, not without them: // // * BM_ExcelReader_WriteColumns is handed buffers that are already columnar. Nothing is @@ -15,7 +16,7 @@ // competitor below is handed - and pays the row-to-column transpose itself. THIS is the // matched-work number, and the only one of ours that belongs next to the competitors. // * ExcelReader attaches a number format to the two XL_T_DATE columns (so Excel shows a date -// rather than a serial), which every case below does NOT do: all three competitors write those +// rather than a serial), which every case below does NOT do: every competitor writes those // columns as bare numbers, the cheaper option. That difference favours the competitors. // * xlnt builds a full in-memory document model (styles, formats, formulas) before serializing. // It is doing more than this library exposes, and its number reflects that. @@ -25,12 +26,17 @@ // class of competitor as xlsxio - it is the one most worth comparing BM_ExcelReader_WriteSheet // against, being C rather than C++ and, like ExcelReader's own core, built for throughput // rather than a full object model. +// * DuckDB's rows are loaded into an in-memory table via its Appender API BEFORE the timed +// region, so BM_DuckDB_Write measures the COPY TO xlsx step alone - same treatment +// BM_ExcelReader_WriteColumns gets for its transpose. DuckDB is a full analytical query engine +// doing far more than any Excel-writing library here, and this measures one narrow slice of it. // // State the CPU, OS and compiler version alongside any number published from this file. #include #include +#include #include #include #include @@ -411,3 +417,71 @@ static void BM_Libxlsxwriter_Write(benchmark::State &state) std::filesystem::remove(path); } BENCHMARK(BM_Libxlsxwriter_Write); + +// DuckDB (https://github.com/duckdb/duckdb) writes via its "excel" extension's +// `COPY ... TO ... WITH (FORMAT xlsx)`. The rows are loaded into an in-memory DuckDB table via its +// Appender API (DuckDB's own fast bulk-load path, not a parsed INSERT statement) BEFORE the timed +// region starts - matching how BM_ExcelReader_WriteColumns transposes outside the loop - so what's +// measured is the COPY itself, not building the table. +static void BM_DuckDB_Write(benchmark::State &state) +{ + const std::vector &rows = fixture_rows(); + const std::filesystem::path path = bench_path("duckdb.xlsx"); + const std::string target = path.string(); + + duckdb::DuckDB db(nullptr); + duckdb::Connection con(db); + + auto setup = con.Query("INSTALL excel; LOAD excel;"); + if (setup->HasError()) + { + state.SkipWithError(setup->GetError().c_str()); + return; + } + + auto create = con.Query( + "CREATE TABLE fixture (" + "\"Region\" VARCHAR, \"Country\" VARCHAR, \"Item Type\" VARCHAR, " + "\"Sales Channel\" VARCHAR, \"Order Priority\" VARCHAR, \"Order Date\" DATE, " + "\"Order ID\" BIGINT, \"Ship Date\" DATE, \"Units Sold\" BIGINT, " + "\"Unit Price\" DOUBLE, \"Unit Cost\" DOUBLE, \"Total Revenue\" DOUBLE, " + "\"Total Cost\" DOUBLE, \"Total Profit\" DOUBLE)"); + if (create->HasError()) + { + state.SkipWithError(create->GetError().c_str()); + return; + } + + { + duckdb::Appender appender(con, "fixture"); + for (const FullRow &row : rows) + { + // .c_str() rather than the std::string itself: AppendRow deduces one Append + // specialization per argument's exact type, and DuckDB only provides one for + // `const char *` (matching duckdb's own test suite), not for std::string. + appender.AppendRow( + row.Region.c_str(), row.Country.c_str(), row.ItemType.c_str(), + row.SalesChannel.c_str(), row.OrderPriority.c_str(), + duckdb::date_t(days(row.OrderDate)), row.OrderId, duckdb::date_t(days(row.ShipDate)), + row.UnitsSold, row.UnitPrice, row.UnitCost, row.TotalRevenue, row.TotalCost, + row.TotalProfit); + } + appender.Close(); + } + + const std::string copy_query = + "COPY fixture TO '" + target + "' WITH (FORMAT xlsx, HEADER true)"; + for (auto _ : state) + { + auto result = con.Query(copy_query); + if (result->HasError()) + { + state.SkipWithError(result->GetError().c_str()); + return; + } + benchmark::DoNotOptimize(result); + } + state.SetItemsProcessed(static_cast(state.iterations()) * static_cast(rows.size())); + std::filesystem::remove(path); +} +BENCHMARK(BM_DuckDB_Write); From 3c92cadf7410f569749d3eb9903487215a065a6b Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Thu, 20 Aug 2026 17:57:39 -0300 Subject: [PATCH 16/51] Fixing xlsxio benchmark --- cpp/benchmarks/CMakeLists.txt | 21 ++++++++++-- cpp/benchmarks/benchmark_write_compare.cpp | 38 +++++++++++++++++++++- 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index a0e14f8..d5b9487 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -274,14 +274,29 @@ set(ZLIB_VERSION_STRING "1.3.1") # Kept a separate executable from the read comparison rather than a second .cpp in the same # one: both files define their own FullRow plus an xl::ExcelMapper specialization for it, and # two targets keep that unambiguous. + # + # The write comparison is then split AGAIN, into two executables built from the same source: + # xlsxio and libxlsxwriter each vendor an incompatible copy of minizip and export the same C + # symbols (zipOpen, zipOpenNewFileInZip, ...), so linking both into one binary lets calls cross + # between the two implementations - which corrupted xlsxio's output in Release. See the header + # comment in benchmark_write_compare.cpp. Do not merge these back into one target. add_executable(excelreader_cpp_write_compare_benchmarks benchmark_write_compare.cpp) target_link_libraries(excelreader_cpp_write_compare_benchmarks PRIVATE - xl::excelreader benchmark::benchmark_main xlnt xlsxio_write xlsxwriter duckdb) + xl::excelreader benchmark::benchmark_main xlnt xlsxio_write duckdb) target_compile_definitions(excelreader_cpp_write_compare_benchmarks PRIVATE - EXCELREADER_XLSX_FIXTURE_PATH="${EXCELREADER_XLSX_FIXTURE_PATH}") + EXCELREADER_XLSX_FIXTURE_PATH="${EXCELREADER_XLSX_FIXTURE_PATH}" + EXCELREADER_BENCH_XLSXIO) + + add_executable(excelreader_cpp_write_compare_lxw_benchmarks benchmark_write_compare.cpp) + target_link_libraries(excelreader_cpp_write_compare_lxw_benchmarks PRIVATE + xl::excelreader benchmark::benchmark_main xlsxwriter duckdb) + target_compile_definitions(excelreader_cpp_write_compare_lxw_benchmarks PRIVATE + EXCELREADER_XLSX_FIXTURE_PATH="${EXCELREADER_XLSX_FIXTURE_PATH}" + EXCELREADER_BENCH_LIBXLSXWRITER) if(WIN32) - foreach(_compare_target excelreader_cpp_compare_benchmarks excelreader_cpp_write_compare_benchmarks) + foreach(_compare_target excelreader_cpp_compare_benchmarks excelreader_cpp_write_compare_benchmarks + excelreader_cpp_write_compare_lxw_benchmarks) add_custom_command(TARGET ${_compare_target} POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different "$" diff --git a/cpp/benchmarks/benchmark_write_compare.cpp b/cpp/benchmarks/benchmark_write_compare.cpp index c2895f8..a374f64 100644 --- a/cpp/benchmarks/benchmark_write_compare.cpp +++ b/cpp/benchmarks/benchmark_write_compare.cpp @@ -32,14 +32,41 @@ // doing far more than any Excel-writing library here, and this measures one narrow slice of it. // // State the CPU, OS and compiler version alongside any number published from this file. +// +// WHY THIS FILE IS COMPILED TWICE (see cpp/benchmarks/CMakeLists.txt): xlsxio and libxlsxwriter +// each bring their own incompatible copy of minizip, and both export the SAME C symbols +// (zipOpen, zipOpenNewFileInZip, zipWriteInFileInZip, ...) from a static library: +// +// * xlsxio is built against minizip-ng's compat layer, whose zipOpenNewFileInZip takes +// uint16_t extrafield sizes. +// * libxlsxwriter vendors classic minizip (third_party/minizip/zip.c), whose signature takes +// 32-bit uInt sizes and whose body starts with `if (size_extrafield_local > 0xffff) return +// ZIP_PARAMERROR;` - a check that cannot exist in the minizip-ng version. +// +// Linked into one executable, the linker keeps exactly one definition of each name, so calls can +// cross between the two: a zipFile opened as minizip-ng's `mz_zip_compat*` gets read as classic +// minizip's `zip64_internal*`. That is how a Release build produced "Error creating file +// xl/workbook.xml inside zip file" on xlsxio's background thread - garbage read out of the wrong +// struct tripping the 0xffff check - while a Debug build, with different link ordering, did not. +// Neither library's numbers are trustworthy in that state. +// +// So each of the two gets its own executable, and this file's competitor cases are guarded to +// match. Do not merge the targets back together. #include #include #include + +// xlsxio and libxlsxwriter MUST NOT be linked into the same executable - see the note above. This +// file is compiled twice, once with each define, by cpp/benchmarks/CMakeLists.txt. +#ifdef EXCELREADER_BENCH_XLSXIO #include #include +#endif +#ifdef EXCELREADER_BENCH_LIBXLSXWRITER #include +#endif #include #include @@ -127,10 +154,15 @@ namespace return rows; } + // Suffixed with a steady_clock reading so concurrent or back-to-back runs of this executable + // cannot collide on one temp file. steady_clock rather than a process id needs no platform + // header (no , no ) - this file has neither today. std::filesystem::path bench_path(std::string_view name) { + const auto ticks = std::chrono::steady_clock::now().time_since_epoch().count(); return std::filesystem::temp_directory_path() / - std::filesystem::path(std::string("excelreader-write-compare-") + std::string(name)); + std::filesystem::path("excelreader-write-compare-" + std::to_string(ticks) + "-" + + std::string(name)); } int32_t days(std::chrono::sys_days value) @@ -263,6 +295,7 @@ static void BM_ExcelReader_WriteColumns(benchmark::State &state) } BENCHMARK(BM_ExcelReader_WriteColumns); +#ifdef EXCELREADER_BENCH_XLSXIO static void BM_Xlnt_Write(benchmark::State &state) { const std::vector &rows = fixture_rows(); @@ -363,7 +396,9 @@ static void BM_Xlsxio_Write(benchmark::State &state) std::filesystem::remove(path); } BENCHMARK(BM_Xlsxio_Write); +#endif // EXCELREADER_BENCH_XLSXIO +#ifdef EXCELREADER_BENCH_LIBXLSXWRITER static void BM_Libxlsxwriter_Write(benchmark::State &state) { const std::vector &rows = fixture_rows(); @@ -417,6 +452,7 @@ static void BM_Libxlsxwriter_Write(benchmark::State &state) std::filesystem::remove(path); } BENCHMARK(BM_Libxlsxwriter_Write); +#endif // EXCELREADER_BENCH_LIBXLSXWRITER // DuckDB (https://github.com/duckdb/duckdb) writes via its "excel" extension's // `COPY ... TO ... WITH (FORMAT xlsx)`. The rows are loaded into an in-memory DuckDB table via its From e5fa1259cc93257fb1bfdc0ffc42b80829d0cd05 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Thu, 20 Aug 2026 18:53:26 -0300 Subject: [PATCH 17/51] Updating README with duckdb data --- cpp/README.md | 92 +++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 68 insertions(+), 24 deletions(-) diff --git a/cpp/README.md b/cpp/README.md index 5597aa1..ca3ee6b 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -199,46 +199,90 @@ The transpose costs ~12% here. It is not free, but it is far from the dominant c the file — see the comparison below, where the same two cases over 14 columns land within ~7%. `excelreader_cpp_write_compare_benchmarks` (under `-DEXCELREADER_BUILD_BENCHMARKS_COMPARE=ON`) puts -that against xlnt, xlsxio and [libxlsxwriter](https://github.com/jmcnamara/libxlsxwriter) — same -author as rust_xlsxwriter, and, like xlsxio, a streaming C writer with no document-model overhead — -all five writing the full 14-column, 65,535-row shape of `65K_Records_Data.xlsx` from the same -in-memory rows: +that against xlnt, xlsxio and [DuckDB](https://github.com/duckdb/duckdb)'s `excel` extension, all +writing the full 14-column, 65,535-row shape of `65K_Records_Data.xlsx` from the same in-memory +rows. +[libxlsxwriter](https://github.com/jmcnamara/libxlsxwriter) — same author as rust_xlsxwriter, and, +like xlsxio, a streaming C writer with no document-model overhead — is measured separately, by +`excelreader_cpp_write_compare_lxw_benchmarks`: xlsxio and libxlsxwriter each vendor their own +incompatible copy of minizip and export the same C symbols (`zipOpen`, `zipOpenNewFileInZip`, ...), +so linking both into one binary let calls cross between the two implementations and corrupted +xlsxio's output — see [Known issue](#known-issue-xlsxio--libxlsxwriter-cannot-share-a-binary) below. +Building both write-compare targets and running them back to back is required to see every +competitor. | Library | Wall | CPU | |---|---:|---:| -| ExcelReader (`xl::write_columns`, pre-transposed) | 127.8 ms | 127.6 ms | -| ExcelReader (`xl::write_sheet`) | 136.5–140.7 ms | 137.5–140.6 ms | -| libxlsxwriter (`worksheet_write_string`/`_number`) | 1,226.9 ms | 1,234.4 ms | -| xlsxio (`xlsxiowrite_add_cell_*`) | 2,383.5–2,453.9 ms | 843.8–906.3 ms | -| xlnt (`worksheet::cell().value()` + `save()`) | 5,398.0–5,471.4 ms | 5,406.3–5,468.8 ms | +| ExcelReader (`xl::write_columns`, pre-transposed) | 126.3–128.1 ms | 127.6 ms | +| ExcelReader (`xl::write_sheet`) | 135.7–136.6 ms | 137.5 ms | +| DuckDB (`COPY ... TO ... WITH (FORMAT xlsx)`) | 826.3–863.6 ms | 828.1–843.8 ms | +| libxlsxwriter (`worksheet_write_string`/`_number`) | 1,188.0 ms | 1,187.5 ms | +| xlsxio (`xlsxiowrite_add_cell_*`) | 2,450.6 ms | 1,171.9 ms | +| xlnt (`worksheet::cell().value()` + `save()`) | 5,737.9 ms | 5,703.1 ms | Same machine as above; Google Benchmark's own iteration counts, no `--benchmark_repetitions` (each -iteration writes a whole 65,535-row file, so the slower cases run once or a handful of times — the -ranges above are two separate runs, shown as a range rather than picking one arbitrarily). +iteration writes a whole 65,535-row file, so the slower cases run once or a handful of times). xlnt +and xlsxio only build into `excelreader_cpp_write_compare_benchmarks`; libxlsxwriter only into +`excelreader_cpp_write_compare_lxw_benchmarks` (see the known issue below) — the ExcelReader and +DuckDB rows appear in both, and the small ranges above are those two independent runs, not repeated +sampling within one run. `write_sheet` — the matched-work number, since it starts from the same `std::vector` every -competitor is handed — is ~9.0x faster than libxlsxwriter, ~17.5x faster than xlsxio, and ~40x -faster than xlnt on wall time. +competitor is handed — is ~6.1–6.4x faster than DuckDB, ~8.7x faster than libxlsxwriter, ~18.1x +faster than xlsxio, and ~42x faster than xlnt on wall time. + +### Known issue: xlsxio + libxlsxwriter cannot share a binary + +An earlier version of `excelreader_cpp_write_compare_benchmarks` linked xlnt, xlsxio, +libxlsxwriter and DuckDB into one executable. xlsxio (built against minizip-ng's compat layer, +whose `zipOpenNewFileInZip` takes `uint16_t` extrafield sizes) and libxlsxwriter (which vendors +classic minizip, whose same-named function takes 32-bit `uInt` sizes and starts with `if +(size_extrafield_local > 0xffff) return ZIP_PARAMERROR;` — a check that cannot exist in the +minizip-ng version) both export identical C symbol names from a static library linked into that one +binary. The linker kept exactly one definition of each name, so a call could resolve to the wrong +implementation — a `zipFile` opened by one library's `zipOpen` got handed to the other library's +`zipOpenNewFileInZip`, which read it through an incompatible struct layout. That is what produced +`Error creating file "xl/workbook.xml" inside zip file` on xlsxio's background thread in Release +builds (Debug's different link order happened not to trigger it) — and it was silent otherwise: +`xlsxiowrite_close()` still returned success with the workbook.xml entry missing, so the benchmark +published a timing for a file that was skipping work, not a valid xlsx. + +The fix is structural: xlsxio and libxlsxwriter now build into two separate executables +(`excelreader_cpp_write_compare_benchmarks` and `excelreader_cpp_write_compare_lxw_benchmarks`, +both compiled from `benchmark_write_compare.cpp` under an `#ifdef`) that are never linked together. +Every case in both executables also reopens the file it just wrote, outside the timed loop, before +trusting its own timing — a writer that silently drops a required part now fails the benchmark +instead of publishing a number for a broken file. + +**libxlsxwriter's number barely moved after the fix** (1,188.0 ms wall vs. 1,226.9 ms pre-fix, ~3%, +inside normal run-to-run noise) and its CPU stayed essentially equal to wall time both before and +after — consistent with the corruption running through xlsxio's calls resolving into +libxlsxwriter's minizip, not the reverse: libxlsxwriter's own output was apparently never affected. +That is a plausible explanation for the asymmetry, not a second confirmed fact — the collision could +in principle run either direction depending on link order, and this project isn't going to re-derive +MSVC's exact symbol-resolution algorithm to be certain which way it went here. Caveats, none of them optional when quoting these: -- **One xlsxio run logged `Error creating file "xl/workbook.xml" inside zip file` mid-benchmark**, - yet `xlsxiowrite_close()` still returned success and the timing came out in the same range as a - clean run. `SkipWithError` only fires on a non-zero return, so this specific case is not proof the - written file was intact — treat the xlsxio row as provisional until a run free of that message - confirms it. -- **xlsxio's wall time is roughly two thirds I/O wait.** Its CPU time is well under half its wall - time, because it streams through a temp file; every other case here is CPU-bound (wall ≈ CPU). - Against CPU time the gap to `write_sheet` is ~6.1–6.6x, not ~17.5x. Which number is the honest one - depends on what you are asking — the wall-time ratio is what a caller waits, the CPU-time ratio is - what the library costs. +- **xlsxio's CPU time moved after the fix**, from 843.8–906.3 ms (pre-fix, some runs missing the + workbook.xml entry) to a confirmed 1,171.9 ms — about 30% more real work, consistent with the + entry no longer being silently dropped. Wall time barely moved (2,450.6 ms vs. 2,383.5–2,453.9 ms), + because xlsxio's wall time is dominated by I/O wait either way: CPU is now ~48% of wall, + against every other case here being CPU-bound (wall ≈ CPU). Against CPU time the gap from + `write_sheet` to xlsxio is ~8.5x, not ~18.1x — which number is the honest one depends on what you + are asking: the wall-time ratio is what a caller waits, the CPU-time ratio is what the library + costs. - **ExcelReader does slightly more work here**, not less: it attaches a number format to the two date columns so Excel shows a date, while every competitor case writes those as bare serial numbers. That difference favours the competitors. - **xlnt builds a full document model** (styles, formats, formulas) before serializing, which is more than this library exposes at all. Its number reflects a different feature set, not only a slower path. +- **DuckDB's rows are loaded via its Appender API before the timed region**, so its number measures + `COPY ... TO ... xlsx` alone — the same treatment `write_columns` gets for its transpose. DuckDB is + a full analytical query engine doing far more than any Excel-writing library here; this measures + one narrow slice of it, not "DuckDB" as a whole. -The two ExcelReader cases land within ~7% of each other, which is the interesting internal result: +The two ExcelReader cases land within ~6% of each other, which is the interesting internal result: the row-to-column transpose is nearly free next to the cost of producing the file. Reach for `write_columns` when your data is already columnar, but `write_sheet` is not the slow path. From e645d4b49506396d5333f20ad35ff1fe0ce9e43c Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Thu, 20 Aug 2026 19:33:30 -0300 Subject: [PATCH 18/51] chore: gitignore .superpowers/ (subagent-driven-development scratch) Co-Authored-By: Claude Opus 5 (1M context) --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index f0ec5d4..87734d6 100644 --- a/.gitignore +++ b/.gitignore @@ -503,4 +503,6 @@ python/src/excelreader/_lib/* # CMake build output (tests/ExcelReader.NativeSmoke and any other CMake-configured project). Never # built to a path inside the source tree other than this, so a plain top-level rule is enough. -build/ \ No newline at end of file +build/ +# Subagent-driven-development scratch (per-plan ledgers, briefs, review packages) +.superpowers/ From d5f12280c0975c9e6ff160d36d82bbb18d6358db Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Thu, 20 Aug 2026 23:52:18 -0300 Subject: [PATCH 19/51] refactor(native): share the byte/tri-state option decoders between both option structs --- src/ExcelReader.Native/NativeOpenOptions.cs | 97 +++++++++++--------- src/ExcelReader.Native/NativeWriteOptions.cs | 50 ++-------- 2 files changed, 66 insertions(+), 81 deletions(-) diff --git a/src/ExcelReader.Native/NativeOpenOptions.cs b/src/ExcelReader.Native/NativeOpenOptions.cs index 6b971d3..b93ddc5 100644 --- a/src/ExcelReader.Native/NativeOpenOptions.cs +++ b/src/ExcelReader.Native/NativeOpenOptions.cs @@ -14,6 +14,51 @@ internal static class NativeOptionState internal const int True = 2; } + /// + /// Field-level decoding shared by and + /// . Both option structs carry the same two shapes of field — a + /// byte-valued one and a tri-state — with the same rules and the + /// same messages; only the struct's own name differs, so it arrives as an argument. + /// + internal static class NativeOptionDecode + { + /// A byte-valued field: 0 means "use the library default", 1-255 is a real byte. + internal static bool TryByte(int value, string structName, string fieldName, out byte? decoded, out string? error) + { + decoded = null; + error = null; + if (value == 0) + { + return true; + } + if (value is < 1 or > 255) + { + error = $"{structName}.{fieldName} must be 0 (default) or a byte value 1-255; got {value}."; + return false; + } + decoded = (byte)value; + return true; + } + + /// A boolean-shaped field, encoded as rather than a plain + /// 0/1 because several of these default to true. + internal static bool TryState(int value, string structName, string fieldName, out bool? decoded, out string? error) + { + decoded = null; + error = null; + if (value is not (NativeOptionState.Default or NativeOptionState.False or NativeOptionState.True)) + { + error = $"{structName}.{fieldName} must be XL_OPT_DEFAULT/FALSE/TRUE (0/1/2); got {value}."; + return false; + } + if (value != NativeOptionState.Default) + { + decoded = value == NativeOptionState.True; + } + return true; + } + } + /// /// Flat C ABI representation of xl_open_options. Every numeric field is 0 for "use the /// library default"; every boolean-shaped field uses instead of a @@ -47,6 +92,9 @@ internal struct NativeOpenOptionsRaw /// internal readonly struct NativeOpenOptions { + /// The C struct's name, as it appears in every message this type produces. + private const string OptionsName = "xl_open_options"; + internal bool CsvSniffDialect { get; init; } internal byte? CsvDelimiter { get; init; } internal byte? CsvQuote { get; init; } @@ -129,12 +177,12 @@ internal static bool TryDecode(NativeOpenOptionsRaw raw, out NativeOpenOptions o int expectedSize = Marshal.SizeOf(); if (raw.StructSize != expectedSize) { - error = $"xl_open_options.struct_size is {raw.StructSize}, but this library expects {expectedSize}."; + error = $"{OptionsName}.struct_size is {raw.StructSize}, but this library expects {expectedSize}."; return false; } - if (!TryDecodeByte(raw.CsvDelimiter, "csv_delimiter", out byte? delimiter, out error) - || !TryDecodeByte(raw.CsvQuote, "csv_quote", out byte? quote, out error) + if (!NativeOptionDecode.TryByte(raw.CsvDelimiter, OptionsName, "csv_delimiter", out byte? delimiter, out error) + || !NativeOptionDecode.TryByte(raw.CsvQuote, OptionsName, "csv_quote", out byte? quote, out error) || !TryDecodeNonNegative(raw.CsvMaxCellBytes, "csv_max_cell_bytes", out int? csvMaxCellBytes, out error) || !TryDecodeNonNegative(raw.MaxCellBytes, "max_cell_bytes", out int? maxCellBytes, out error) || !TryDecodeNonNegative(raw.MaxZipEntries, "max_zip_entries", out int? maxZipEntries, out error) @@ -144,11 +192,11 @@ internal static bool TryDecode(NativeOpenOptionsRaw raw, out NativeOpenOptions o return false; } - if (!TryDecodeState(raw.CsvSniffDialect, "csv_sniff_dialect", out bool? sniffDialect, out error) - || !TryDecodeState(raw.CsvDetectBom, "csv_detect_bom", out bool? detectBom, out error) - || !TryDecodeState(raw.CsvInternStrings, "csv_intern_strings", out bool? csvInternStrings, out error) - || !TryDecodeState(raw.PrefetchDecompression, "prefetch_decompression", out bool? prefetch, out error) - || !TryDecodeState(raw.InternStrings, "intern_strings", out bool? internStrings, out error)) + if (!NativeOptionDecode.TryState(raw.CsvSniffDialect, OptionsName, "csv_sniff_dialect", out bool? sniffDialect, out error) + || !NativeOptionDecode.TryState(raw.CsvDetectBom, OptionsName, "csv_detect_bom", out bool? detectBom, out error) + || !NativeOptionDecode.TryState(raw.CsvInternStrings, OptionsName, "csv_intern_strings", out bool? csvInternStrings, out error) + || !NativeOptionDecode.TryState(raw.PrefetchDecompression, OptionsName, "prefetch_decompression", out bool? prefetch, out error) + || !NativeOptionDecode.TryState(raw.InternStrings, OptionsName, "intern_strings", out bool? internStrings, out error)) { return false; } @@ -171,23 +219,6 @@ internal static bool TryDecode(NativeOpenOptionsRaw raw, out NativeOpenOptions o return true; } - private static bool TryDecodeByte(int value, string fieldName, out byte? decoded, out string? error) - { - decoded = null; - error = null; - if (value == 0) - { - return true; - } - if (value is < 1 or > 255) - { - error = $"xl_open_options.{fieldName} must be 0 (default) or a byte value 1-255; got {value}."; - return false; - } - decoded = (byte)value; - return true; - } - // Serves both the int and long fields: the rule ("0 means default, negative is a caller error") // and its message are identical, and only the width differed. private static bool TryDecodeNonNegative(T value, string fieldName, out T? decoded, out string? error) @@ -207,21 +238,5 @@ private static bool TryDecodeNonNegative(T value, string fieldName, out T? de decoded = value; return true; } - - private static bool TryDecodeState(int value, string fieldName, out bool? decoded, out string? error) - { - decoded = null; - error = null; - if (value is not (NativeOptionState.Default or NativeOptionState.False or NativeOptionState.True)) - { - error = $"xl_open_options.{fieldName} must be XL_OPT_DEFAULT/FALSE/TRUE (0/1/2); got {value}."; - return false; - } - if (value != NativeOptionState.Default) - { - decoded = value == NativeOptionState.True; - } - return true; - } } } diff --git a/src/ExcelReader.Native/NativeWriteOptions.cs b/src/ExcelReader.Native/NativeWriteOptions.cs index 6f6624d..286242b 100644 --- a/src/ExcelReader.Native/NativeWriteOptions.cs +++ b/src/ExcelReader.Native/NativeWriteOptions.cs @@ -29,6 +29,9 @@ internal unsafe struct NativeWriteOptionsRaw /// internal readonly struct NativeWriteOptions { + /// The C struct's name, as it appears in every message this type produces. + private const string OptionsName = "xl_write_options"; + /// Excel's own limit; a longer name is rejected here rather than by the writer, so the /// caller gets XL_INVALID_ARGUMENT before a file is created instead of XL_ERROR after. private const int MaxSheetNameLength = 31; @@ -71,10 +74,10 @@ internal static bool TryDecode(NativeWriteOptionsRaw raw, string? sheetName, out } if (!TryValidateSheetName(sheetName, out error) - || !TryDecodeByte(raw.CsvDelimiter, "csv_delimiter", out byte? delimiter, out error) - || !TryDecodeByte(raw.CsvQuote, "csv_quote", out byte? quote, out error) - || !TryDecodeState(raw.Date1904, "date1904", out bool? date1904, out error) - || !TryDecodeState(raw.UseSharedStrings, "use_shared_strings", out bool? sharedStrings, out error)) + || !NativeOptionDecode.TryByte(raw.CsvDelimiter, OptionsName, "csv_delimiter", out byte? delimiter, out error) + || !NativeOptionDecode.TryByte(raw.CsvQuote, OptionsName, "csv_quote", out byte? quote, out error) + || !NativeOptionDecode.TryState(raw.Date1904, OptionsName, "date1904", out bool? date1904, out error) + || !NativeOptionDecode.TryState(raw.UseSharedStrings, OptionsName, "use_shared_strings", out bool? sharedStrings, out error)) { return false; } @@ -106,7 +109,7 @@ internal static bool TryValidateStructSize(NativeWriteOptionsRaw raw, [NotNullWh int expectedSize = Marshal.SizeOf(); if (raw.StructSize != expectedSize) { - error = $"xl_write_options.struct_size is {raw.StructSize}, but this library expects {expectedSize}."; + error = $"{OptionsName}.struct_size is {raw.StructSize}, but this library expects {expectedSize}."; return false; } return true; @@ -121,47 +124,14 @@ private static bool TryValidateSheetName(string? sheetName, out string? error) } if (sheetName.Length is 0 or > MaxSheetNameLength) { - error = $"xl_write_options.sheet_name must be 1-{MaxSheetNameLength} characters; got {sheetName.Length}."; + error = $"{OptionsName}.sheet_name must be 1-{MaxSheetNameLength} characters; got {sheetName.Length}."; return false; } if (sheetName.AsSpan().IndexOfAny(ForbiddenSheetNameCharactersSearchValues) >= 0) { - error = $@"xl_write_options.sheet_name must not contain any of : \ / ? * [ ] ; got ""{sheetName}""."; - return false; - } - return true; - } - - private static bool TryDecodeByte(int value, string fieldName, out byte? decoded, out string? error) - { - decoded = null; - error = null; - if (value == 0) - { - return true; - } - if (value is < 1 or > 255) - { - error = $"xl_write_options.{fieldName} must be 0 (default) or a byte value 1-255; got {value}."; + error = $@"{OptionsName}.sheet_name must not contain any of : \ / ? * [ ] ; got ""{sheetName}""."; return false; } - decoded = (byte)value; - return true; - } - - private static bool TryDecodeState(int value, string fieldName, out bool? decoded, out string? error) - { - decoded = null; - error = null; - if (value is not (NativeOptionState.Default or NativeOptionState.False or NativeOptionState.True)) - { - error = $"xl_write_options.{fieldName} must be XL_OPT_DEFAULT/FALSE/TRUE (0/1/2); got {value}."; - return false; - } - if (value != NativeOptionState.Default) - { - decoded = value == NativeOptionState.True; - } return true; } } From 2d7ebb7b2e6b30d02f98f7f29575b8dd74ca4ffa Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Fri, 21 Aug 2026 00:01:16 -0300 Subject: [PATCH 20/51] refactor(native): share the skip-to-header-row loop between parse_typed and infer_schema --- src/ExcelReader.Native/NativeApi.Schema.cs | 17 +++++---------- src/ExcelReader.Native/NativeApi.Typed.cs | 25 ++++++++++++++++------ 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/src/ExcelReader.Native/NativeApi.Schema.cs b/src/ExcelReader.Native/NativeApi.Schema.cs index b274fcc..40c26cd 100644 --- a/src/ExcelReader.Native/NativeApi.Schema.cs +++ b/src/ExcelReader.Native/NativeApi.Schema.cs @@ -102,14 +102,9 @@ internal static void FreeSchema(ref NativeInferredSchema schema) // empty string — an empty name would fail xl_parse_typed's own "blank name" validation later. private static bool TryReadHeader(IExcelRowEnumerator rows, int headerRow, List names, List stats, out string? error) { - error = null; - for (int rowNumber = 1; rowNumber <= headerRow; rowNumber++) + if (!TrySkipToHeaderRow(rows, headerRow, out error)) { - if (!rows.MoveNext()) - { - error = $"sheet has fewer than {headerRow} row(s); cannot resolve header_row."; - return false; - } + return false; } Row header = rows.Current; foreach (RowCell cell in header.Cells) @@ -261,11 +256,9 @@ internal void Observe(in Cell cell, bool isDate1904) internal readonly int InferType() { int kinds = (SawString ? 1 : 0) + (SawNumber ? 1 : 0) + (SawDate ? 1 : 0) + (SawBool ? 1 : 0); - if (SawFormulaOrError || kinds != 1) - { - return NativeColumnType.String; - } - if (SawString) + // A mix of kinds, a formula/error result, nothing sampled at all, or plain text — all + // four fall back to the string type, the only one able to represent them verbatim. + if (SawFormulaOrError || kinds != 1 || SawString) { return NativeColumnType.String; } diff --git a/src/ExcelReader.Native/NativeApi.Typed.cs b/src/ExcelReader.Native/NativeApi.Typed.cs index 3588ddd..cd6c465 100644 --- a/src/ExcelReader.Native/NativeApi.Typed.cs +++ b/src/ExcelReader.Native/NativeApi.Typed.cs @@ -193,6 +193,23 @@ private static bool TryValidateArguments(NativeColumnSpec[] specs, int headerRow return true; } + // Advances `rows` so that `rows.Current` is the header row itself. Shared with + // NativeApi.Schema.cs's TryReadHeader so the two cannot drift on the row arithmetic or on the + // message a too-short sheet produces. + private static bool TrySkipToHeaderRow(IExcelRowEnumerator rows, int headerRow, out string? error) + { + error = null; + for (int rowNumber = 1; rowNumber <= headerRow; rowNumber++) + { + if (!rows.MoveNext()) + { + error = $"sheet has fewer than {headerRow} row(s); cannot resolve header_row."; + return false; + } + } + return true; + } + // Advances `rows` past any skipped rows and the header row itself (headerRow > 0), or leaves it // untouched at the sheet's first row (headerRow == 0, index-only specs). Either way, `rows` is // positioned so the next MoveNext() yields the first DATA row. @@ -208,13 +225,9 @@ private static bool TryResolveColumns(IExcelRowEnumerator rows, NativeColumnSpec return true; } - for (int rowNumber = 1; rowNumber <= headerRow; rowNumber++) + if (!TrySkipToHeaderRow(rows, headerRow, out error)) { - if (!rows.MoveNext()) - { - error = $"sheet has fewer than {headerRow} row(s); cannot resolve header_row."; - return false; - } + return false; } Row header = rows.Current; From 4c784bad78a8140f59320b1ebe2171eb5841b7ca Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Fri, 21 Aug 2026 00:07:06 -0300 Subject: [PATCH 21/51] refactor(cpp): collapse the six scalar column factories onto one constexpr template --- .../include/excelreader.hpp | 40 ++++++++++--------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/src/ExcelReader.Native/include/excelreader.hpp b/src/ExcelReader.Native/include/excelreader.hpp index 6838813..35a3569 100644 --- a/src/ExcelReader.Native/include/excelreader.hpp +++ b/src/ExcelReader.Native/include/excelreader.hpp @@ -309,58 +309,60 @@ namespace xl return validity.empty() ? nullptr : validity.data(); } + // Every non-string column lowers identically: the values span supplies both the pointer and + // the row count, the validity span both the pointer and its length, and the only thing that + // varies per column type is the XL_T_* tag. The named factories below are one line each on + // top of this, so the wire layout lives in exactly one place. + template + inline constexpr ColumnRef scalar_column(std::string_view name, std::span values, + std::span validity) noexcept + { + return ColumnRef{name, Tag, static_cast(values.size()), values.data(), + validity_pointer(validity), static_cast(validity.size()), + nullptr, 0}; + } + } // namespace detail // One constructor per column type rather than an overload set: XL_T_BOOL's buffer and a string // blob are both std::span, and XL_T_I64/TIME/TIMESTAMP are all - // std::span, so overload resolution could not tell them apart. + // std::span, so overload resolution could not tell them apart. Each is one line + // over detail::scalar_column, which holds the shared lowering. inline constexpr ColumnRef i64_column(std::string_view name, std::span values, std::span validity = {}) noexcept { - return ColumnRef{name, XL_T_I64, static_cast(values.size()), values.data(), - detail::validity_pointer(validity), static_cast(validity.size()), - nullptr, 0}; + return detail::scalar_column(name, values, validity); } inline constexpr ColumnRef f64_column(std::string_view name, std::span values, std::span validity = {}) noexcept { - return ColumnRef{name, XL_T_F64, static_cast(values.size()), values.data(), - detail::validity_pointer(validity), static_cast(validity.size()), - nullptr, 0}; + return detail::scalar_column(name, values, validity); } // `values` is one byte per row, 0 or 1 - NOT a bit-packed bitmap. inline constexpr ColumnRef bool_column(std::string_view name, std::span values, std::span validity = {}) noexcept { - return ColumnRef{name, XL_T_BOOL, static_cast(values.size()), values.data(), - detail::validity_pointer(validity), static_cast(validity.size()), - nullptr, 0}; + return detail::scalar_column(name, values, validity); } inline constexpr ColumnRef date_column(std::string_view name, std::span days_since_epoch, std::span validity = {}) noexcept { - return ColumnRef{name, XL_T_DATE, static_cast(days_since_epoch.size()), - days_since_epoch.data(), detail::validity_pointer(validity), - static_cast(validity.size()), nullptr, 0}; + return detail::scalar_column(name, days_since_epoch, validity); } inline constexpr ColumnRef time_column(std::string_view name, std::span micros_since_midnight, std::span validity = {}) noexcept { - return ColumnRef{name, XL_T_TIME, static_cast(micros_since_midnight.size()), - micros_since_midnight.data(), detail::validity_pointer(validity), - static_cast(validity.size()), nullptr, 0}; + return detail::scalar_column(name, micros_since_midnight, validity); } inline constexpr ColumnRef timestamp_column(std::string_view name, std::span micros_since_epoch, std::span validity = {}) noexcept { - return ColumnRef{name, XL_T_TIMESTAMP, static_cast(micros_since_epoch.size()), - micros_since_epoch.data(), detail::validity_pointer(validity), - static_cast(validity.size()), nullptr, 0}; + return detail::scalar_column(name, micros_since_epoch, validity); } // `offsets` has length + 1 entries; `data` is every row's UTF-8 bytes concatenated. Unlike the From 039a5d4b76bcc6d2f5264eadfc7623f075739cc6 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Fri, 21 Aug 2026 00:10:50 -0300 Subject: [PATCH 22/51] refactor(cpp): share the options-lowering block across open/open_memory/write_columns --- .../include/excelreader.hpp | 35 +++++++++---------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/src/ExcelReader.Native/include/excelreader.hpp b/src/ExcelReader.Native/include/excelreader.hpp index 35a3569..063088f 100644 --- a/src/ExcelReader.Native/include/excelreader.hpp +++ b/src/ExcelReader.Native/include/excelreader.hpp @@ -227,6 +227,20 @@ namespace xl namespace detail { + // A NULL options pointer is the ABI's "every default", and is NOT the same as a zeroed + // struct (whose struct_size of 0 is rejected). `storage` is the caller's own local, which + // must outlive the FFI call the returned pointer is handed to. + template + inline const Raw *lower_options(const Opts *options, Raw &storage) noexcept + { + if (options == nullptr) + { + return nullptr; + } + storage = options->to_c(); + return &storage; + } + // Case-insensitive suffix match over ASCII, which is all a file extension can be here. // constexpr and allocation-free so format_from_path stays usable in a constant expression. constexpr bool ends_with_ci(std::string_view text, std::string_view suffix) noexcept @@ -483,12 +497,7 @@ namespace xl } xl_workbook *handle = nullptr; xl_open_options c_options{}; - const xl_open_options *c_options_ptr = nullptr; - if (options != nullptr) - { - c_options = options->to_c(); - c_options_ptr = &c_options; - } + const xl_open_options *c_options_ptr = detail::lower_options(options, c_options); int32_t status = xl_open_file_ex(reinterpret_cast(path.data()), static_cast(path.size()), format, c_options_ptr, &handle); @@ -510,12 +519,7 @@ namespace xl } xl_workbook *handle = nullptr; xl_open_options c_options{}; - const xl_open_options *c_options_ptr = nullptr; - if (options != nullptr) - { - c_options = options->to_c(); - c_options_ptr = &c_options; - } + const xl_open_options *c_options_ptr = detail::lower_options(options, c_options); int32_t status = xl_open_memory_ex(data.data(), static_cast(data.size()), format, c_options_ptr, &handle); if (status != XL_OK) @@ -1160,12 +1164,7 @@ namespace xl // A zeroed xl_write_options is NOT the same as no options: its struct_size of 0 is rejected. // NULL is what means "every default". xl_write_options raw_options{}; - const xl_write_options *options_pointer = nullptr; - if (options != nullptr) - { - raw_options = options->to_c(); - options_pointer = &raw_options; - } + const xl_write_options *options_pointer = detail::lower_options(options, raw_options); const int32_t status = xl_write_typed(reinterpret_cast(path.data()), static_cast(path.size()), format, specs.data(), From bdbfdb1750347d8920da893f048826a885b10435 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Fri, 21 Aug 2026 00:19:23 -0300 Subject: [PATCH 23/51] refactor(cpp): route TableView's row materialization through one detail::row_at --- .../include/excelreader.hpp | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/src/ExcelReader.Native/include/excelreader.hpp b/src/ExcelReader.Native/include/excelreader.hpp index 063088f..09f66b2 100644 --- a/src/ExcelReader.Native/include/excelreader.hpp +++ b/src/ExcelReader.Native/include/excelreader.hpp @@ -912,11 +912,24 @@ namespace xl } template - void populate_instance(T &instance, const xl_table &table, int64_t row, const Tuple &bindings, std::index_sequence) + inline void populate_instance(T &instance, const xl_table &table, int64_t row, const Tuple &bindings, std::index_sequence) { (..., assign_field(instance, table.columns[Is], row, std::get(bindings))); } + // The one place a T is built from a row of the columnar buffers. Both TableView::operator[] + // and its iterator's operator*() go through this, so the bindings lookup and the index + // sequence are written once. + template + inline T row_at(const xl_table &table, int64_t row) + { + T instance{}; + static constexpr auto bindings = ExcelMapper::get_bindings(); + static constexpr size_t num_fields = std::tuple_size_v; + populate_instance(instance, table, row, bindings, std::make_index_sequence{}); + return instance; + } + } // namespace detail // ---- TableView: a lazy, non-owning-of-T view over a parsed xl_table ------------------------ @@ -958,14 +971,7 @@ namespace xl iterator() = default; - T operator*() const - { - T instance{}; - static constexpr auto bindings = ExcelMapper::get_bindings(); - static constexpr size_t num_fields = std::tuple_size_v; - detail::populate_instance(instance, *table_, row_, bindings, std::make_index_sequence{}); - return instance; - } + T operator*() const { return detail::row_at(*table_, row_); } T operator[](difference_type n) const { return *(*this + n); } @@ -1051,14 +1057,7 @@ namespace xl // Unchecked, exactly like std::vector::operator[]: a `row` outside [0, size()) reads past // the columnar buffers and returns whatever sits after the allocation. Use at() below unless // the caller has already established the bound. - T operator[](int64_t row) const - { - T instance{}; - static constexpr auto bindings = ExcelMapper::get_bindings(); - static constexpr size_t num_fields = std::tuple_size_v; - detail::populate_instance(instance, table_, row, bindings, std::make_index_sequence{}); - return instance; - } + T operator[](int64_t row) const { return detail::row_at(table_, row); } // Bounds-checked counterpart to operator[]. Returns nullopt rather than throwing, since this // header is exception-free by design (std::vector::at's out_of_range is not an option here). From dcd0a67c6077ac99b5e158a076f2eefa96f4baf2 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Fri, 21 Aug 2026 00:19:36 -0300 Subject: [PATCH 24/51] Adding env variable for rust --- .vscode/settings.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index e738329..f101367 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -11,5 +11,8 @@ "-DEXCELREADER_BUILD_BENCHMARKS_COMPARE=ON", "-DEXCELREADER_BUILD_TESTS=ON", "-DCMAKE_POLICY_VERSION_MINIMUM=3.5" - ] + ], + "rust-analyzer.server.extraEnv": { + "EXCELREADER_NATIVE_LIB_DIR": "${workspaceFolder}/python/src/excelreader/_lib" + } } \ No newline at end of file From 30713df110064924b5e0bad0e43c077383fad0ab Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Fri, 21 Aug 2026 00:23:16 -0300 Subject: [PATCH 25/51] refactor(python): inherit the inferred-spec layout and derive the format tables from one source --- python/src/excelreader/_native.py | 31 +++++++++++++++++++------------ python/src/excelreader/reader.py | 8 +------- python/src/excelreader/writer.py | 17 ++++------------- 3 files changed, 24 insertions(+), 32 deletions(-) diff --git a/python/src/excelreader/_native.py b/python/src/excelreader/_native.py index af29ebb..faa4764 100644 --- a/python/src/excelreader/_native.py +++ b/python/src/excelreader/_native.py @@ -5,10 +5,10 @@ from __future__ import annotations -from collections.abc import Sequence import ctypes import os import platform +from collections.abc import Sequence from functools import lru_cache from pathlib import Path from typing import TYPE_CHECKING @@ -34,6 +34,18 @@ XL_FORMAT_XLSX = 2 XL_FORMAT_XLSB = 3 XL_FORMAT_CSV = 4 +# The one mapping from a public format name to its XL_FORMAT_* value. reader.py and writer.py both +# derive their tables from these rather than restating them. +FORMATS = { + "auto": XL_FORMAT_AUTO, + "xls": XL_FORMAT_XLS, + "xlsx": XL_FORMAT_XLSX, + "xlsb": XL_FORMAT_XLSB, + "csv": XL_FORMAT_CSV, +} +# xl_write_typed rejects XL_FORMAT_AUTO - a file being created has no signature bytes to sniff - so +# the write side gets the same table minus that entry. +WRITE_FORMATS = {name: value for name, value in FORMATS.items() if name != "auto"} # Every boolean-shaped NativeOpenOptions field uses one of these three states, never a plain 0/1 - # several of them default to true, so a bare 0 would be ambiguous between "off" and "use the library @@ -117,23 +129,18 @@ def column_spec_by_index(index: int, type_: int, *, nullable: bool = False) -> N ) -class NativeInferredColumnSpec(ctypes.Structure): - """Mirrors xl_column_spec's layout exactly, for the OUTPUT direction (xl_infer_schema). +class NativeInferredColumnSpec(NativeColumnSpec): + """The OUTPUT direction of the same xl_column_spec (xl_infer_schema fills these in). + + Layout is inherited, not restated: it is literally the same C struct, and a second copy of the + field list is a second place for it to drift from the header. Only the ownership rules differ, + which is why this carries its own name. Always carries `name_count` 0 or 1 — inference never guesses more than one candidate name per column. Unlike `column_spec_by_name`'s buffers, `names[0]` here is a raw pointer with no guaranteed NUL terminator, so decode it with `ctypes.string_at(names[0], name_lens[0])`. """ - _fields_ = [ - ("names", ctypes.POINTER(ctypes.POINTER(ctypes.c_uint8))), - ("name_lens", ctypes.POINTER(ctypes.c_int32)), - ("name_count", ctypes.c_int32), - ("index", ctypes.c_int32), - ("type", ctypes.c_int32), - ("nullable", ctypes.c_int32), - ] - class NativeInferredSchema(ctypes.Structure): """Mirrors xl_inferred_schema. `columns` is a native-owned array of `column_count` values, freed diff --git a/python/src/excelreader/reader.py b/python/src/excelreader/reader.py index 567ff51..9464c59 100644 --- a/python/src/excelreader/reader.py +++ b/python/src/excelreader/reader.py @@ -29,13 +29,7 @@ _numpy = None # NumPy is an optional extra (pip install excelreader-native[numpy]) — see # _buffer_to_array()/_to_columnar_array() below, the only two places that consult it. -_FORMATS = { - "auto": _native.XL_FORMAT_AUTO, - "xls": _native.XL_FORMAT_XLS, - "xlsx": _native.XL_FORMAT_XLSX, - "xlsb": _native.XL_FORMAT_XLSB, - "csv": _native.XL_FORMAT_CSV, -} +_FORMATS = _native.FORMATS _CELL_HEADER = struct.Struct(" int: From bdf1e1aa360b36883283592b785c9c74c2276b0b Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Fri, 21 Aug 2026 00:26:03 -0300 Subject: [PATCH 26/51] refactor(rust): share the raw-options pointer helper across open/open_memory/write_columns --- rust/excelreader/src/options.rs | 8 ++++++++ rust/excelreader/src/workbook.rs | 10 +++------- rust/excelreader/src/writer.rs | 6 ++---- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/rust/excelreader/src/options.rs b/rust/excelreader/src/options.rs index 3192bc9..902d9c3 100644 --- a/rust/excelreader/src/options.rs +++ b/rust/excelreader/src/options.rs @@ -200,6 +200,14 @@ impl WriteOptions { } } +/// The pointer an FFI options argument takes: a borrowed raw struct, or NULL for "every library +/// default". `raw` must outlive the call the returned pointer is handed to - taking it by reference +/// rather than by value is what makes the compiler check that. +#[inline] +pub(crate) fn ptr_or_null(raw: &Option) -> *const T { + raw.as_ref().map_or(std::ptr::null(), |value| value as *const T) +} + #[cfg(test)] mod tests { use super::*; diff --git a/rust/excelreader/src/workbook.rs b/rust/excelreader/src/workbook.rs index 1a09a97..8d121b3 100644 --- a/rust/excelreader/src/workbook.rs +++ b/rust/excelreader/src/workbook.rs @@ -1,6 +1,6 @@ use crate::{ Date, Error, OpenOptions, Time, Timestamp, XlColumn, XlColumnSpec, XlInferredSchema, - XlOpenOptions, XlTable, XlWorkbook, XL_BUFFER_TOO_SMALL, XL_ERROR, XL_FORMAT_AUTO, XL_OK, + XlTable, XlWorkbook, XL_BUFFER_TOO_SMALL, XL_ERROR, XL_FORMAT_AUTO, XL_OK, XL_T_BOOL, XL_T_DATE, XL_T_F64, XL_T_I64, XL_T_STRING, XL_T_TIME, XL_T_TIMESTAMP, }; use std::marker::PhantomData; @@ -83,9 +83,7 @@ impl Workbook { ) -> Result { check_abi_version()?; let raw = options.map(OpenOptions::to_raw); - let raw_ptr = raw - .as_ref() - .map_or(std::ptr::null(), |o| o as *const XlOpenOptions); + let raw_ptr = crate::options::ptr_or_null(&raw); let mut handle: *mut XlWorkbook = std::ptr::null_mut(); // `raw` outlives the call below, and the native side copies the path before returning. let status = unsafe { @@ -110,9 +108,7 @@ impl Workbook { ) -> Result { check_abi_version()?; let raw = options.map(OpenOptions::to_raw); - let raw_ptr = raw - .as_ref() - .map_or(std::ptr::null(), |o| o as *const XlOpenOptions); + let raw_ptr = crate::options::ptr_or_null(&raw); let mut handle: *mut XlWorkbook = std::ptr::null_mut(); let status = unsafe { crate::xl_open_memory_ex( diff --git a/rust/excelreader/src/writer.rs b/rust/excelreader/src/writer.rs index ab7dcaf..9bf5900 100644 --- a/rust/excelreader/src/writer.rs +++ b/rust/excelreader/src/writer.rs @@ -8,7 +8,7 @@ use crate::workbook::{check, check_abi_version}; use crate::{ - Error, WriteOptions, XlColumn, XlColumnSpec, XlTable, XlWriteOptions, XL_FORMAT_AUTO, + Error, WriteOptions, XlColumn, XlColumnSpec, XlTable, XL_FORMAT_AUTO, XL_FORMAT_CSV, XL_FORMAT_XLS, XL_FORMAT_XLSB, XL_FORMAT_XLSX, XL_INVALID_ARGUMENT, XL_T_BOOL, XL_T_DATE, XL_T_F64, XL_T_I64, XL_T_STRING, XL_T_TIME, XL_T_TIMESTAMP, }; @@ -248,9 +248,7 @@ pub fn write_columns( // Lowered here rather than by the caller: the raw struct holds a pointer into // `options.sheet_name`, and this borrow provably covers the FFI call below. let raw_options = options.map(WriteOptions::to_raw); - let options_ptr = raw_options - .as_ref() - .map_or(std::ptr::null(), |o| o as *const XlWriteOptions); + let options_ptr = crate::options::ptr_or_null(&raw_options); let status = unsafe { crate::xl_write_typed( From 1d421800babe0be1de19dbfab5a44db34c0514b7 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Fri, 21 Aug 2026 00:28:22 -0300 Subject: [PATCH 27/51] refactor(rust): collapse the duplicated rejection arms in the ExcelMapper derive --- rust/excelreader-derive/src/lib.rs | 33 +++++++++++++----------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/rust/excelreader-derive/src/lib.rs b/rust/excelreader-derive/src/lib.rs index 521ada1..e0b335f 100644 --- a/rust/excelreader-derive/src/lib.rs +++ b/rust/excelreader-derive/src/lib.rs @@ -72,19 +72,19 @@ fn expand(input: DeriveInput) -> syn::Result { fn named_fields( input: &DeriveInput, ) -> syn::Result<&syn::punctuated::Punctuated> { - match &input.data { - Data::Struct(data) => match &data.fields { - Fields::Named(fields) => Ok(&fields.named), - _ => Err(syn::Error::new_spanned( - input, - "ExcelMapper can only be derived for structs with named fields", - )), - }, - _ => Err(syn::Error::new_spanned( - input, - "ExcelMapper can only be derived for structs with named fields", - )), + // One pattern, one message: a tuple struct, a unit struct and an enum are all the same rejection, + // and writing it twice is how the two copies eventually stop matching. + if let Data::Struct(syn::DataStruct { + fields: Fields::Named(fields), + .. + }) = &input.data + { + return Ok(&fields.named); } + Err(syn::Error::new_spanned( + input, + "ExcelMapper can only be derived for structs with named fields", + )) } fn field_binding(field: &Field) -> syn::Result { @@ -358,13 +358,8 @@ impl FieldKind { .map(|segment| segment.ident.to_string()), _ => None, }; - let name = name.as_deref(); - if let Some(name) = name { - if INT_TYPES.contains(&name) { - return Ok(FieldKind::Int); - } - } - match name { + match name.as_deref() { + Some(int) if INT_TYPES.contains(&int) => Ok(FieldKind::Int), Some("String") => Ok(FieldKind::Str), Some("f32" | "f64") => Ok(FieldKind::Float), Some("bool") => Ok(FieldKind::Bool), From 8d8ace86b7fc1a47a36fdbb05f67efff21c21633 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Fri, 21 Aug 2026 01:44:37 -0300 Subject: [PATCH 28/51] Parsing fixes --- rust/excelreader-derive/src/lib.rs | 77 ++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 25 deletions(-) diff --git a/rust/excelreader-derive/src/lib.rs b/rust/excelreader-derive/src/lib.rs index e0b335f..d52618a 100644 --- a/rust/excelreader-derive/src/lib.rs +++ b/rust/excelreader-derive/src/lib.rs @@ -23,19 +23,24 @@ fn expand(input: DeriveInput) -> syn::Result { let struct_name = &input.ident; let fields = named_fields(&input)?; - let bindings = fields + // Parsed ONCE, up front: both halves below read the same ParsedField, so they cannot disagree + // about a field's kind, its primary name, or whether it is optional. Every attribute and type + // error surfaces here, before either half is generated. + let parsed = fields .iter() - .map(field_binding) + .map(parse_field) .collect::>>()?; + let bindings = parsed.iter().map(field_binding).collect::>(); + // The write half needs one set of buffer identifiers per field, so it is generated by index // rather than field by field: the declarations, the per-row pushes, and the OwnedColumn // construction all have to agree on the same names. let mut declarations = Vec::new(); let mut pushes = Vec::new(); let mut columns = Vec::new(); - for (index, field) in fields.iter().enumerate() { - let plan = WriteField::new(index, field)?; + for (index, field) in parsed.iter().enumerate() { + let plan = WriteField::new(index, field); declarations.push(plan.declarations()); pushes.push(plan.push()); columns.push(plan.column()); @@ -87,26 +92,48 @@ fn named_fields( )) } -fn field_binding(field: &Field) -> syn::Result { - let field_ident = field.ident.as_ref().expect("named_fields guarantees Some"); - let name = excel_names(field)?; +/// Everything both halves of the derive need from one field, resolved once. +/// +/// The read half (`field_binding`) and the write half (`WriteField`) used to each re-run +/// `excel_names`/`unwrap_option`/`FieldKind::from_type` over the same field. Parsing once is not +/// just less work: it is what guarantees the two halves cannot disagree about a field's kind or +/// about which of its names is the primary one. +struct ParsedField<'a> { + ident: &'a syn::Ident, + /// Primary `#[excel(name = "...")]` first, then any aliases in declared order. + names: Vec, + kind: FieldKind, + is_option: bool, +} + +fn parse_field(field: &Field) -> syn::Result> { let (inner_ty, is_option) = unwrap_option(&field.ty); - let kind = FieldKind::from_type(inner_ty)?; - let xl_type = kind.xl_type_tokens(); - let value = kind.value_tokens(); - let assign_value = if is_option { + Ok(ParsedField { + ident: field.ident.as_ref().expect("named_fields guarantees Some"), + names: excel_names(field)?, + kind: FieldKind::from_type(inner_ty)?, + is_option, + }) +} + +fn field_binding(parsed: &ParsedField<'_>) -> proc_macro2::TokenStream { + let field_ident = parsed.ident; + let name = &parsed.names; + let xl_type = parsed.kind.xl_type_tokens(); + let value = parsed.kind.value_tokens(); + let assign_value = if parsed.is_option { quote! { ::std::option::Option::Some(#value) } } else { value }; - Ok(quote! { + quote! { ::excelreader::workbook::ColumnBinding { names: &[#(#name),*], xl_type: #xl_type, assign: |r, col, row| r.#field_ident = #assign_value, } - }) + } } fn excel_names(field: &Field) -> syn::Result> { @@ -159,6 +186,7 @@ fn unwrap_option(ty: &Type) -> (&Type, bool) { (ty, false) } +#[derive(Clone, Copy)] enum FieldKind { Str, /// `i64` itself, and every other integer width. The column is always `XL_T_I64` on the wire; a @@ -186,22 +214,21 @@ struct WriteField { } impl WriteField { - fn new(index: usize, field: &Field) -> syn::Result { - let names = excel_names(field)?; - let (inner_ty, is_option) = unwrap_option(&field.ty); - Ok(WriteField { - ident: field.ident.clone().expect("named_fields guarantees Some"), - name: names - .into_iter() - .next() - .expect("excel_names always returns the primary name first"), - kind: FieldKind::from_type(inner_ty)?, - is_option, + fn new(index: usize, parsed: &ParsedField<'_>) -> WriteField { + WriteField { + ident: parsed.ident.clone(), + name: parsed + .names + .first() + .expect("excel_names always returns the primary name first") + .clone(), + kind: parsed.kind, + is_option: parsed.is_option, values: quote::format_ident!("values_{index}"), offsets: quote::format_ident!("offsets_{index}"), data: quote::format_ident!("data_{index}"), validity: quote::format_ident!("validity_{index}"), - }) + } } fn declarations(&self) -> proc_macro2::TokenStream { From b783d284a6473c92a7601ae31850fe4944004c3c Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Fri, 21 Aug 2026 14:53:03 -0300 Subject: [PATCH 29/51] Fixing dependencies --- python/pyproject.toml | 9 +++++++-- python/src/excelreader/reader.py | 7 ++++++- rust/excelreader/Cargo.toml | 3 ++- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index f585010..39c3d57 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -14,10 +14,15 @@ classifiers = [ "Programming Language :: Python :: 3", "License :: OSI Approved :: MIT License", ] -dependencies = ["typing_extensions>=4.0"] +dependencies = [] [project.optional-dependencies] -dev = ["pytest>=7.0"] +# typing_extensions is a type-checking-time-only import (see reader.py's TYPE_CHECKING guard for +# typing.Self, 3.11+ in the stdlib) — never evaluated at runtime, so it belongs in dev, not +# dependencies. Keeping it out of the wheel's runtime deps is the point: this package's whole +# selling point is a ctypes-only, interpreter-independent wheel with no compiled Python extension +# and (now) no runtime dependency at all. +dev = ["pytest>=7.0", "typing_extensions>=4.0"] # Optional: Workbook.read_all_columnar() returns NumPy int32 arrays when this is installed, and # array('i') (stdlib, always available) otherwise — see reader.py's _to_columnar_array(). Kept # optional on purpose: the package's whole selling point is a ctypes-only, interpreter-independent diff --git a/python/src/excelreader/reader.py b/python/src/excelreader/reader.py index 9464c59..6cb000f 100644 --- a/python/src/excelreader/reader.py +++ b/python/src/excelreader/reader.py @@ -7,8 +7,13 @@ from array import array from collections.abc import Iterator, Sequence from pathlib import Path +from typing import TYPE_CHECKING -from typing_extensions import Self +if TYPE_CHECKING: + # typing.Self is 3.11+; typing_extensions backports it for 3.9/3.10. Only needed for the + # annotation below, which `from __future__ import annotations` keeps unevaluated at runtime — + # so the wheel stays free of a runtime dependency on typing_extensions. + from typing_extensions import Self from excelreader import _native from excelreader.types import ( diff --git a/rust/excelreader/Cargo.toml b/rust/excelreader/Cargo.toml index 50a470b..3fd3d99 100644 --- a/rust/excelreader/Cargo.toml +++ b/rust/excelreader/Cargo.toml @@ -20,12 +20,13 @@ excelreader-derive = { path = "../excelreader-derive", version = "0.0.0" } # `default-features = false` drops chrono's `oldtime`/`clock` stack: this crate only converts # between integer counts and calendar types, and never reads the system clock or a timezone database. chrono = { version = "0.4", optional = true, default-features = false, features = ["std"] } -rust_xlsxwriter = "0.98.2" [dev-dependencies] trybuild = "1" criterion = "0.5" calamine = { version = "0.36.1", features = ["chrono"] } +# Comparison baseline for write_bench only — this crate's own write path never calls it. +rust_xlsxwriter = "0.98.2" [[bench]] name = "parse_bench" From 58e85dc53c411a1531ddea06afc8f83b6c4b0677 Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Fri, 21 Aug 2026 17:51:09 -0300 Subject: [PATCH 30/51] fix: export xl_parse_arrow from the Windows .def files The symbol was already exported from the NativeAOT library via [UnmanagedCallersOnly], but absent from the three hand-maintained .def copies that lib.exe/dlltool turn into the Windows import library, so C++ and Rust could not link against it. Adds a test asserting the three copies stay identical. --- rust/excelreader/excelreader.def | 1 + .../include/excelreader.def | 1 + tests/ExcelReader.Tests/DefFileSyncTests.cs | 51 +++++++++++++++++++ 3 files changed, 53 insertions(+) create mode 100644 tests/ExcelReader.Tests/DefFileSyncTests.cs diff --git a/rust/excelreader/excelreader.def b/rust/excelreader/excelreader.def index d0b5386..c0b90bb 100644 --- a/rust/excelreader/excelreader.def +++ b/rust/excelreader/excelreader.def @@ -12,6 +12,7 @@ EXPORTS xl_is_date1904 xl_parse_typed xl_free_table + xl_parse_arrow xl_write_typed xl_infer_schema xl_free_schema diff --git a/src/ExcelReader.Native/include/excelreader.def b/src/ExcelReader.Native/include/excelreader.def index d0b5386..c0b90bb 100644 --- a/src/ExcelReader.Native/include/excelreader.def +++ b/src/ExcelReader.Native/include/excelreader.def @@ -12,6 +12,7 @@ EXPORTS xl_is_date1904 xl_parse_typed xl_free_table + xl_parse_arrow xl_write_typed xl_infer_schema xl_free_schema diff --git a/tests/ExcelReader.Tests/DefFileSyncTests.cs b/tests/ExcelReader.Tests/DefFileSyncTests.cs new file mode 100644 index 0000000..8d5905c --- /dev/null +++ b/tests/ExcelReader.Tests/DefFileSyncTests.cs @@ -0,0 +1,51 @@ +using System.Reflection; + +namespace ExcelReader.Tests +{ + public sealed class DefFileSyncTests + { + // The repo root, found by walking up from the test binary until the solution file appears. + private static string RepoRoot() + { + DirectoryInfo? dir = new(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!); + while (dir is not null && !File.Exists(Path.Combine(dir.FullName, "ExcelReader.slnx"))) + { + dir = dir.Parent; + } + Assert.NotNull(dir); + return dir.FullName; + } + + private static readonly string[] DefPaths = + [ + Path.Combine("src", "ExcelReader.Native", "include", "excelreader.def"), + Path.Combine("rust", "excelreader", "excelreader.def"), + Path.Combine("cpp", "include", "xl", "excelreader.def"), + ]; + + private static string[] ReadExports(string root, string relative) + { + string[] lines = File.ReadAllLines(Path.Combine(root, relative)); + Assert.Equal("EXPORTS", lines[0].Trim()); + return [.. lines.Skip(1).Select(l => l.Trim()).Where(l => l.Length > 0)]; + } + + [Fact] + public void Should_ListIdenticalExports_When_ComparingTheThreeDefCopies() + { + string root = RepoRoot(); + string[] canonical = ReadExports(root, DefPaths[0]); + + foreach (string copy in DefPaths.Skip(1)) + { + Assert.Equal(canonical, ReadExports(root, copy)); + } + } + + [Fact] + public void Should_ExportParseArrow_When_ReadingTheCanonicalDef() + { + Assert.Contains("xl_parse_arrow", ReadExports(RepoRoot(), DefPaths[0]), StringComparer.Ordinal); + } + } +} From dc9afa4c225aa531d3b97a47578d9672b87adc00 Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Fri, 21 Aug 2026 18:11:27 -0300 Subject: [PATCH 31/51] feat(cpp): add xl::parse_arrow and the ArrowTable RAII owner Wraps the existing xl_parse_arrow export. Kept in its own header so a caller who does not want the Arrow C Data Interface declarations never includes them, and deliberately free of any Apache Arrow C++ dependency: the C Data Interface is the interop currency, so the caller feeds the pair into whichever Arrow implementation they already link. --- cpp/tests/CMakeLists.txt | 15 +++ cpp/tests/arrow.cpp | 63 +++++++++++ .../include/excelreader_arrow.hpp | 100 ++++++++++++++++++ 3 files changed, 178 insertions(+) create mode 100644 cpp/tests/arrow.cpp create mode 100644 src/ExcelReader.Native/include/excelreader_arrow.hpp diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 056fcf9..a8eefdf 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -27,4 +27,19 @@ if(WIN32) COMMAND ${CMAKE_COMMAND} -E copy_if_different "$" "$") +endif() + +add_executable(excelreader_cpp_arrow arrow.cpp) +target_link_libraries(excelreader_cpp_arrow PRIVATE xl::excelreader) +target_compile_definitions(excelreader_cpp_arrow PRIVATE + EXCELREADER_FIXTURE_PATH="${EXCELREADER_FIXTURE_PATH}") + +add_test(NAME excelreader_cpp_arrow COMMAND excelreader_cpp_arrow) +if(WIN32) + # Same reason as the smoke test: the native DLL has no import-time PATH entry, so copy it next + # to the test executable for the loader to find. + add_custom_command(TARGET excelreader_cpp_arrow POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "$" + "$") endif() \ No newline at end of file diff --git a/cpp/tests/arrow.cpp b/cpp/tests/arrow.cpp new file mode 100644 index 0000000..b3237f9 --- /dev/null +++ b/cpp/tests/arrow.cpp @@ -0,0 +1,63 @@ +#include + +#include +#include +#include +#include + +struct Record +{ + std::string_view Coluna1; + int64_t Coluna3; +}; + +template <> +struct xl::ExcelMapper +{ + static constexpr auto get_bindings() + { + return std::make_tuple( + xl::make_field("Coluna1", &Record::Coluna1), + xl::make_field("Coluna3", &Record::Coluna3)); + } +}; + +#define CHECK(cond, msg) \ + do \ + { \ + if (!(cond)) \ + { \ + std::fprintf(stderr, "FAIL: %s (%s:%d)\n", msg, __FILE__, __LINE__); \ + return 1; \ + } \ + } while (0) + +int main() +{ + xl::OpenOptions options{.prefetch_decompression = 1}; + auto workbook = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, XL_FORMAT_XLSB, &options); + CHECK(workbook.has_value(), "xl::Workbook::open must succeed on the RealExcel.xlsb fixture"); + + auto table = xl::parse_arrow(*workbook); + CHECK(table.has_value(), "xl::parse_arrow must succeed"); + + // The export hands back ONE top-level struct array whose children are the columns. + CHECK(std::strcmp(table->schema.format, "+s") == 0, "top level must be a struct array"); + CHECK(table->schema.n_children == 2, "must have two child columns"); + CHECK(table->array.n_children == 2, "must have two child arrays"); + CHECK(std::strcmp(table->schema.children[0]->name, "Coluna1") == 0, "first column must be named Coluna1"); + CHECK(std::strcmp(table->schema.children[0]->format, "u") == 0, "Coluna1 must be utf8"); + CHECK(std::strcmp(table->schema.children[1]->format, "l") == 0, "Coluna3 must be int64"); + CHECK(table->array.length == 100, "RealExcel.xlsb has 100 data rows"); + + // Destructor must release both; running under a leak checker in CI is what proves it, but a + // move-then-destroy here at least exercises the moved-from path being inert. + { + xl::ArrowTable moved = std::move(*table); + CHECK(moved.array.release != nullptr, "moved-to table must still own a release callback"); + CHECK(table->array.release == nullptr, "moved-from table must be released/inert"); + } + + std::printf("OK: C++ arrow test passed\n"); + return 0; +} diff --git a/src/ExcelReader.Native/include/excelreader_arrow.hpp b/src/ExcelReader.Native/include/excelreader_arrow.hpp new file mode 100644 index 0000000..b73d2f0 --- /dev/null +++ b/src/ExcelReader.Native/include/excelreader_arrow.hpp @@ -0,0 +1,100 @@ +/* Optional Arrow C Data Interface export for the C++ binding. Deliberately a separate header from + * excelreader.hpp (mirroring the excelreader.h / excelreader_arrow.h split): a caller who never + * wants Arrow never pulls these declarations in. + * + * This header does NOT depend on the Apache Arrow C++ library. The Arrow C Data Interface is a + * fixed, versioned ABI - handing back the raw ArrowArray/ArrowSchema pair lets the caller feed it + * into whichever Arrow implementation they already link, instead of forcing one on them. */ +#ifndef XL_EXCELREADER_ARROW_HPP +#define XL_EXCELREADER_ARROW_HPP + +#include "excelreader.hpp" +#include "excelreader_arrow.h" + +#include +#include + +namespace xl +{ + // Owns one ArrowArray/ArrowSchema pair produced by parse_arrow, releasing both on destruction. + // + // Ownership note: xl_parse_arrow's results are released through their OWN release callbacks, + // never through xl_free_table - the native side already freed its intermediate table before + // returning. Releasing the schema and array is independent; both are done here. + struct ArrowTable + { + ArrowArray array{}; + ArrowSchema schema{}; + + ArrowTable() = default; + + ArrowTable(const ArrowTable &) = delete; + ArrowTable &operator=(const ArrowTable &) = delete; + + // A released-or-moved-from ArrowArray/ArrowSchema is defined by the Arrow spec as one whose + // `release` member is null, so zeroing the source is exactly what "moved-from" means here. + ArrowTable(ArrowTable &&other) noexcept + : array(std::exchange(other.array, ArrowArray{})), + schema(std::exchange(other.schema, ArrowSchema{})) + { + } + + ArrowTable &operator=(ArrowTable &&other) noexcept + { + if (this != &other) + { + release(); + array = std::exchange(other.array, ArrowArray{}); + schema = std::exchange(other.schema, ArrowSchema{}); + } + return *this; + } + + ~ArrowTable() { release(); } + + // Releases both structures early. Idempotent: the Arrow spec requires a release callback to + // null its own struct's `release` member, so a second call is a no-op. + void release() noexcept + { + if (array.release != nullptr) + { + array.release(&array); + } + if (schema.release != nullptr) + { + schema.release(&schema); + } + } + }; + + // Same schema-driven parse as xl::parse_sheet, returned as one top-level Arrow struct + // array/schema instead of a TableView. `header_row` has the same meaning as in parse_sheet + // (0 = no header). Consumes the workbook's shared row cursor, hence Workbook&. + template + std::expected parse_arrow(Workbook &workbook, int32_t header_row = 1) + { + // The next four lines are xl::parse_sheet's own spec-building block + // (cpp/include/xl/excelreader.hpp:1101-1106), copied verbatim: both entry points take an + // identical xl_column_spec array, built from the same ExcelMapper::get_bindings(). + static constexpr auto bindings = ExcelMapper::get_bindings(); + static constexpr size_t num_fields = std::tuple_size_v; + std::array, num_fields> name_lens_storage{}; + std::array specs_array = + detail::build_specs(bindings, std::make_index_sequence{}, name_lens_storage); + std::span specs(specs_array); + + ArrowTable result; + int32_t status = xl_parse_arrow(workbook.handle(), specs.data(), + static_cast(specs.size()), header_row, + &result.array, &result.schema); + if (status != XL_OK) + { + // On failure the ABI leaves both outputs untouched (still zeroed), so ~ArrowTable is a + // no-op and there is nothing to release here. + return std::unexpected(detail::make_error(status)); + } + return result; + } +} + +#endif /* XL_EXCELREADER_ARROW_HPP */ From 83a30efd089456692a7306035f300808f691a7bc Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Sat, 22 Aug 2026 10:28:33 -0300 Subject: [PATCH 32/51] refactor(rust): extract build_specs, shared by parse_typed and parse_arrow Both native entry points take an identical xl_column_spec array. Pulls the pointer bookkeeping (and the keepalive vectors the specs point into) into one SpecArena so the upcoming Arrow binding does not copy it. Co-Authored-By: Claude Opus 5 --- rust/excelreader/src/workbook.rs | 39 ++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/rust/excelreader/src/workbook.rs b/rust/excelreader/src/workbook.rs index 8d121b3..a3269bb 100644 --- a/rust/excelreader/src/workbook.rs +++ b/rust/excelreader/src/workbook.rs @@ -496,13 +496,18 @@ impl Iterator for TableViewIter<'_, T> { impl ExactSizeIterator for TableViewIter<'_, T> {} -/// Schema-driven columnar parse of the current sheet, matching C++'s `xl::parse_sheet`. -/// -/// Takes `&mut Workbook` because the parse consumes the workbook's shared row cursor. -pub fn parse_sheet( - workbook: &mut Workbook, - header_row: i32, -) -> Result, Error> { +/// Keeps the per-column name pointer/length vectors alive for as long as the `XlColumnSpec` array +/// that points into them. The two `_name_*` fields are never read - dropping them early would +/// leave `specs` holding dangling pointers, which is the whole reason they are stored here. +pub(crate) struct SpecArena { + pub(crate) specs: Vec, + _name_ptrs: Vec>, + _name_lens: Vec>, +} + +/// Lowers `T`'s ExcelMapper bindings into the flat `xl_column_spec` array both `xl_parse_typed` and +/// `xl_parse_arrow` take - their column-spec input is identical. +pub(crate) fn build_specs() -> SpecArena { let bindings = T::bindings(); let name_ptrs: Vec> = bindings .iter() @@ -524,6 +529,22 @@ pub fn parse_sheet( nullable: 1, }) .collect(); + SpecArena { + specs, + _name_ptrs: name_ptrs, + _name_lens: name_lens, + } +} + +/// Schema-driven columnar parse of the current sheet, matching C++'s `xl::parse_sheet`. +/// +/// Takes `&mut Workbook` because the parse consumes the workbook's shared row cursor. +pub fn parse_sheet( + workbook: &mut Workbook, + header_row: i32, +) -> Result, Error> { + let bindings = T::bindings(); + let arena = build_specs::(); let mut table = XlTable { column_count: 0, row_count: 0, @@ -532,8 +553,8 @@ pub fn parse_sheet( unsafe { let status = crate::xl_parse_typed( workbook.handle, - specs.as_ptr(), - specs.len() as i32, + arena.specs.as_ptr(), + arena.specs.len() as i32, header_row, &mut table, ); From d964b0f18cdc3cd820746094b6257887866963f5 Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Sat, 22 Aug 2026 11:26:00 -0300 Subject: [PATCH 33/51] feat(rust): add parse_arrow behind the arrow feature Imports the native xl_parse_arrow export through the Arrow C Data Interface and hands back an arrow::array::RecordBatch. Off by default, matching the chrono feature: arrow-rs is a large dependency the typed parse path needs none of. Co-Authored-By: Claude Sonnet 5 --- rust/excelreader/Cargo.toml | 4 +++ rust/excelreader/src/arrow.rs | 52 +++++++++++++++++++++++++++ rust/excelreader/src/lib.rs | 20 +++++++++++ rust/excelreader/src/workbook.rs | 7 ++++ rust/excelreader/tests/parse_arrow.rs | 44 +++++++++++++++++++++++ 5 files changed, 127 insertions(+) create mode 100644 rust/excelreader/src/arrow.rs create mode 100644 rust/excelreader/tests/parse_arrow.rs diff --git a/rust/excelreader/Cargo.toml b/rust/excelreader/Cargo.toml index 3fd3d99..acede2d 100644 --- a/rust/excelreader/Cargo.toml +++ b/rust/excelreader/Cargo.toml @@ -14,12 +14,16 @@ default = [] # NaiveDate/NaiveTime/NaiveDateTime, so `#[derive(ExcelMapper)]` can populate chrono fields # directly. Off by default: the newtypes alone need no dependency at all. chrono = ["dep:chrono"] +# Converts the native Arrow C Data Interface export into an `arrow::array::RecordBatch`. Off by +# default: arrow-rs is a large dependency and the crate's typed-parse path needs none of it. +arrow = ["dep:arrow"] [dependencies] excelreader-derive = { path = "../excelreader-derive", version = "0.0.0" } # `default-features = false` drops chrono's `oldtime`/`clock` stack: this crate only converts # between integer counts and calendar types, and never reads the system clock or a timezone database. chrono = { version = "0.4", optional = true, default-features = false, features = ["std"] } +arrow = { version = "56", optional = true, default-features = false, features = ["ffi"] } [dev-dependencies] trybuild = "1" diff --git a/rust/excelreader/src/arrow.rs b/rust/excelreader/src/arrow.rs new file mode 100644 index 0000000..736f525 --- /dev/null +++ b/rust/excelreader/src/arrow.rs @@ -0,0 +1,52 @@ +//! Arrow C Data Interface import, behind the `arrow` cargo feature. +//! +//! The native side already produces one top-level Arrow struct array whose children are the +//! columns, so the only work here is handing arrow-rs the FFI pair and letting it take ownership. + +use std::ffi::c_void; + +use arrow::array::{RecordBatch, StructArray}; +use arrow::ffi::{from_ffi, FFI_ArrowArray, FFI_ArrowSchema}; + +// `check` is `pub(crate)` in workbook.rs (not error.rs) - visible from this sibling module because +// pub(crate) means "crate-wide", not "same file". +use crate::workbook::{build_specs, check, ExcelMapper, Workbook}; +use crate::{Error, XL_ERROR}; + +/// Schema-driven parse of the current sheet into an Arrow [`RecordBatch`], using the same +/// `#[derive(ExcelMapper)]` mapping as [`crate::workbook::parse_sheet`]. +/// +/// `header_row` has the same meaning as in `parse_sheet` (0 = no header). Takes `&mut Workbook` +/// because the parse consumes the workbook's shared row cursor. +pub fn parse_arrow( + workbook: &mut Workbook, + header_row: i32, +) -> Result { + let arena = build_specs::(); + + // Both start with a null `release`, which the Arrow spec defines as "owns nothing" - so if the + // call below fails and leaves them untouched, dropping them is a no-op and nothing leaks. + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + + check(unsafe { + crate::xl_parse_arrow( + workbook.handle(), + arena.specs.as_ptr(), + arena.specs.len() as i32, + header_row, + &mut array as *mut FFI_ArrowArray as *mut c_void, + &mut schema as *mut FFI_ArrowSchema as *mut c_void, + ) + })?; + + // from_ffi consumes `array` by value: arrow-rs now owns it and will invoke its release callback + // when the resulting ArrayData is dropped. `schema` stays owned here and releases on drop at + // the end of this function, which is correct - the two are released independently. + let data = unsafe { from_ffi(array, &schema) }.map_err(|e| Error { + code: XL_ERROR, + message: format!("importing the native Arrow array failed: {e}"), + })?; + + Ok(RecordBatch::from(StructArray::from(data))) +} diff --git a/rust/excelreader/src/lib.rs b/rust/excelreader/src/lib.rs index 6b5785a..4686e74 100644 --- a/rust/excelreader/src/lib.rs +++ b/rust/excelreader/src/lib.rs @@ -8,6 +8,9 @@ mod temporal; pub mod workbook; pub mod writer; +#[cfg(feature = "arrow")] +pub mod arrow; + pub use error::Error; pub use options::{OpenOptions, WriteOptions}; pub use temporal::{Date, Time, Timestamp}; @@ -175,6 +178,23 @@ extern "C" { out_table: *mut XlTable, ) -> c_int; + /// Same schema-driven parse as `xl_parse_typed`, exported as one top-level Arrow struct + /// array/schema. `out_array`/`out_schema` are `struct ArrowArray*`/`struct ArrowSchema*` from + /// the Arrow C Data Interface, typed here as `c_void` because arrow-rs's own `#[repr(C)]` + /// `FFI_ArrowArray`/`FFI_ArrowSchema` are ABI-identical to them - redeclaring the spec structs + /// would be a second source of truth for a fixed, versioned ABI. + /// + /// On `XL_OK` the caller owns both and releases each through its OWN `release` callback, never + /// through `xl_free_table`. On any other status both outputs are left untouched. + pub fn xl_parse_arrow( + handle: *mut XlWorkbook, + specs: *const XlColumnSpec, + spec_count: i32, + header_row: i32, + out_array: *mut c_void, + out_schema: *mut c_void, + ) -> c_int; + pub fn xl_free_table(table: *mut XlTable); pub fn xl_infer_schema( diff --git a/rust/excelreader/src/workbook.rs b/rust/excelreader/src/workbook.rs index a3269bb..d31e6cc 100644 --- a/rust/excelreader/src/workbook.rs +++ b/rust/excelreader/src/workbook.rs @@ -193,6 +193,13 @@ impl Workbook { Ok(columns) } + /// The raw handle, for sibling modules (e.g. `arrow::parse_arrow`) that need to call an + /// `xl_*` function this struct has no wrapper for yet. Not part of the crate's public surface - + /// `pub(crate)`, not `pub`. + pub(crate) fn handle(&self) -> *mut XlWorkbook { + self.handle + } + /// Shared two-pass buffer dance for the `xl_*` functions that write a UTF-8 name into a caller /// buffer and report the required capacity through `XL_BUFFER_TOO_SMALL`. fn fill_string( diff --git a/rust/excelreader/tests/parse_arrow.rs b/rust/excelreader/tests/parse_arrow.rs new file mode 100644 index 0000000..3f07f3e --- /dev/null +++ b/rust/excelreader/tests/parse_arrow.rs @@ -0,0 +1,44 @@ +#![cfg(feature = "arrow")] + +use arrow::array::{Array, Int64Array, StringArray}; +use excelreader::arrow::parse_arrow; +use excelreader::workbook::{ExcelMapper, Workbook}; + +#[derive(Default, ExcelMapper)] +struct Record { + #[excel(name = "Coluna1")] + coluna1: String, + #[excel(name = "Coluna3")] + coluna3: i64, +} + +fn fixture_path() -> String { + concat!(env!("CARGO_MANIFEST_DIR"), "/../../RealExcel.xlsb").to_string() +} + +#[test] +fn parse_arrow_returns_a_record_batch_with_one_column_per_field() { + let mut workbook = Workbook::open(&fixture_path()).expect("open must succeed"); + let batch = parse_arrow::(&mut workbook, 1).expect("parse_arrow must succeed"); + + assert_eq!(batch.num_columns(), 2); + assert_eq!(batch.num_rows(), 100); // RealExcel.xlsb has 100 data rows. + // Field names come from the column spec's source name (the `#[excel(name = "...")]` value), + // not the Rust struct field identifier - the native side names each Arrow child field from + // `spec.Names[0]` (see ExcelReader.Native/NativeApi.Arrow.cs's BuildChildSchema). + assert_eq!(batch.schema().field(0).name(), "Coluna1"); + assert_eq!(batch.schema().field(1).name(), "Coluna3"); + + // Downcasting proves the XL_T_* -> Arrow format-code mapping actually landed, not just that a + // batch of the right shape came back. + assert!(batch.column(0).as_any().downcast_ref::().is_some()); + assert!(batch.column(1).as_any().downcast_ref::().is_some()); +} + +#[test] +fn parse_arrow_reports_an_error_without_leaving_a_half_built_batch() { + let mut workbook = Workbook::open(&fixture_path()).expect("open must succeed"); + // header_row is 1-based; a row number past the end of the sheet cannot resolve any column name. + let result = parse_arrow::(&mut workbook, 1_000_000); + assert!(result.is_err()); +} From b3ca97ec2a6908a51d6ad5de81030b33cf4c0990 Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Sat, 22 Aug 2026 11:56:05 -0300 Subject: [PATCH 34/51] docs: the Arrow export is no longer Python-only Co-Authored-By: Claude Opus 5 --- README.md | 5 ++++- cpp/README.md | 15 +++++++++++++++ rust/excelreader/README.md | 17 +++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3f48034..7a50547 100644 --- a/README.md +++ b/README.md @@ -897,7 +897,10 @@ write_sheet("out.xlsx", XL_FORMAT_XLSX, &rows, None)?; auto written = xl::write_sheet("out.xlsx", rows); // format inferred from the extension ``` -Row-by-row decoded reads and the Arrow export remain Python-only. +Row-by-row decoded reads remain Python-only. The Arrow export is available from Python +(`to_arrow`/`to_record_batch`), C++ (`xl::parse_arrow`, in the separate `` +header — no Apache Arrow C++ dependency, you get the raw C Data Interface pair), and Rust +(`excelreader::arrow::parse_arrow`, behind the `arrow` cargo feature, returning an `arrow::array::RecordBatch`). ## Contributing diff --git a/cpp/README.md b/cpp/README.md index ca3ee6b..5fb1aa1 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -78,6 +78,21 @@ for (const auto& column : *workbook->infer_schema(1, 100)) { Every entry point returns `std::expected` — this header throws nothing. +### Arrow export + +`` is a separate header — including `` never pulls the +Arrow C Data Interface declarations in. It does not depend on the Apache Arrow C++ library: you get +the raw `ArrowArray`/`ArrowSchema` pair, owned by an RAII `xl::ArrowTable`, to hand to whichever +Arrow implementation you already link. + +```cpp +#include + +auto workbook = xl::Workbook::open("book.xlsx").value(); +auto table = xl::parse_arrow(workbook).value(); +// table.array / table.schema are a top-level struct array; both release in ~ArrowTable. +``` + ## Writing Two layers, mirroring the two on the reading side. diff --git a/rust/excelreader/README.md b/rust/excelreader/README.md index 821f1ac..97c4291 100644 --- a/rust/excelreader/README.md +++ b/rust/excelreader/README.md @@ -71,6 +71,23 @@ for column in workbook.infer_schema(1, 100)? { `Workbook::open_with` takes an explicit format and `OpenOptions`; `Workbook::open_memory` reads from a byte slice. Note that format sniffing does not detect CSV - pass `XL_FORMAT_CSV` explicitly. +### Arrow export (`arrow` feature) + +```toml +excelreader = { version = "0.0.0", features = ["arrow"] } +``` + +```rust +use excelreader::arrow::parse_arrow; +use excelreader::workbook::Workbook; + +let mut workbook = Workbook::open("book.xlsx")?; +let batch = parse_arrow::(&mut workbook, 1)?; +println!("{} rows x {} columns", batch.num_rows(), batch.num_columns()); +``` + +Off by default — arrow-rs is a large dependency and the typed-parse path needs none of it. + ## Writing `#[derive(ExcelMapper)]` generates both halves, so the same struct reads and writes - the field From dea92cae4931a02e4996863512ffe762c28bf375 Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Sat, 22 Aug 2026 12:00:52 -0300 Subject: [PATCH 35/51] docs(fix): use Row struct name instead of undefined Record in Arrow export examples --- cpp/README.md | 6 +++--- rust/excelreader/README.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/README.md b/cpp/README.md index 5fb1aa1..bd0053d 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -88,9 +88,9 @@ Arrow implementation you already link. ```cpp #include -auto workbook = xl::Workbook::open("book.xlsx").value(); -auto table = xl::parse_arrow(workbook).value(); -// table.array / table.schema are a top-level struct array; both release in ~ArrowTable. +auto workbook = xl::Workbook::open("book.xlsx"); +auto table = xl::parse_arrow(*workbook); +// table->array / table->schema are a top-level struct array; both release in ~ArrowTable. ``` ## Writing diff --git a/rust/excelreader/README.md b/rust/excelreader/README.md index 97c4291..231ec19 100644 --- a/rust/excelreader/README.md +++ b/rust/excelreader/README.md @@ -82,7 +82,7 @@ use excelreader::arrow::parse_arrow; use excelreader::workbook::Workbook; let mut workbook = Workbook::open("book.xlsx")?; -let batch = parse_arrow::(&mut workbook, 1)?; +let batch = parse_arrow::(&mut workbook, 1)?; println!("{} rows x {} columns", batch.num_rows(), batch.num_columns()); ``` From 7fc494b7b3bb039786bb3521e96e6bcd08cbb174 Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Sat, 22 Aug 2026 18:41:02 -0300 Subject: [PATCH 36/51] fix: final review fixes for Arrow C++/Rust bindings branch - DefFileSyncTests: drop the cpp/include/xl symlink arm from the .def comparison (it resolves to the same file as the canonical copy, so it compared a file to itself and would break on Windows CI runners that don't enable core.symlinks). Only two .def copies are physically distinct; the remaining test still catches real drift. - workbook.rs: make SpecArena generic and carry T::bindings() alongside the flat spec array, so build_specs::() computes bindings exactly once instead of parse_sheet calling T::bindings() a second time. arrow.rs is unaffected and compiles unmodified. - Add C++ coverage for parse_arrow failing cleanly on an out-of-range header_row (cpp/tests/arrow.cpp), and strengthen the matching Rust test to prove the workbook is still usable after the failed call. --- cpp/tests/arrow.cpp | 7 +++++++ rust/excelreader/src/workbook.rs | 12 +++++++++--- rust/excelreader/tests/parse_arrow.rs | 6 ++++++ tests/ExcelReader.Tests/DefFileSyncTests.cs | 7 +++++-- 4 files changed, 27 insertions(+), 5 deletions(-) diff --git a/cpp/tests/arrow.cpp b/cpp/tests/arrow.cpp index b3237f9..40f667e 100644 --- a/cpp/tests/arrow.cpp +++ b/cpp/tests/arrow.cpp @@ -58,6 +58,13 @@ int main() CHECK(table->array.release == nullptr, "moved-from table must be released/inert"); } + // An out-of-range header_row must fail cleanly, leaving no half-built ArrowTable behind - this + // matters here more than on the happy path because ~ArrowTable calls through the release + // function pointers it holds, so a half-initialized table on the failure path would mean the + // destructor walks into garbage. + auto failed = xl::parse_arrow(*workbook, 1'000'000); + CHECK(!failed.has_value(), "xl::parse_arrow must fail for an out-of-range header_row"); + std::printf("OK: C++ arrow test passed\n"); return 0; } diff --git a/rust/excelreader/src/workbook.rs b/rust/excelreader/src/workbook.rs index d31e6cc..43a5c82 100644 --- a/rust/excelreader/src/workbook.rs +++ b/rust/excelreader/src/workbook.rs @@ -506,15 +506,20 @@ impl ExactSizeIterator for TableViewIter<'_, T> {} /// Keeps the per-column name pointer/length vectors alive for as long as the `XlColumnSpec` array /// that points into them. The two `_name_*` fields are never read - dropping them early would /// leave `specs` holding dangling pointers, which is the whole reason they are stored here. -pub(crate) struct SpecArena { +/// +/// Also carries the `T::bindings()` this arena was built from, so callers that need both the flat +/// spec array (for the FFI call) and the typed bindings (for result-column lookups) can get both +/// from a single `T::bindings()` call instead of computing it twice. +pub(crate) struct SpecArena { pub(crate) specs: Vec, + pub(crate) bindings: Vec>, _name_ptrs: Vec>, _name_lens: Vec>, } /// Lowers `T`'s ExcelMapper bindings into the flat `xl_column_spec` array both `xl_parse_typed` and /// `xl_parse_arrow` take - their column-spec input is identical. -pub(crate) fn build_specs() -> SpecArena { +pub(crate) fn build_specs() -> SpecArena { let bindings = T::bindings(); let name_ptrs: Vec> = bindings .iter() @@ -538,6 +543,7 @@ pub(crate) fn build_specs() -> SpecArena { .collect(); SpecArena { specs, + bindings, _name_ptrs: name_ptrs, _name_lens: name_lens, } @@ -550,8 +556,8 @@ pub fn parse_sheet( workbook: &mut Workbook, header_row: i32, ) -> Result, Error> { - let bindings = T::bindings(); let arena = build_specs::(); + let bindings = arena.bindings; let mut table = XlTable { column_count: 0, row_count: 0, diff --git a/rust/excelreader/tests/parse_arrow.rs b/rust/excelreader/tests/parse_arrow.rs index 3f07f3e..1de532b 100644 --- a/rust/excelreader/tests/parse_arrow.rs +++ b/rust/excelreader/tests/parse_arrow.rs @@ -41,4 +41,10 @@ fn parse_arrow_reports_an_error_without_leaving_a_half_built_batch() { // header_row is 1-based; a row number past the end of the sheet cannot resolve any column name. let result = parse_arrow::(&mut workbook, 1_000_000); assert!(result.is_err()); + + // The failed call must not have left the workbook or its shared row cursor in a broken state - + // a normal, valid parse right after the failure should succeed exactly as if the failed call + // had never happened. + let batch = parse_arrow::(&mut workbook, 1).expect("parse_arrow must succeed"); + assert_eq!(batch.num_rows(), 100); // RealExcel.xlsb has 100 data rows. } diff --git a/tests/ExcelReader.Tests/DefFileSyncTests.cs b/tests/ExcelReader.Tests/DefFileSyncTests.cs index 8d5905c..2a01fba 100644 --- a/tests/ExcelReader.Tests/DefFileSyncTests.cs +++ b/tests/ExcelReader.Tests/DefFileSyncTests.cs @@ -16,11 +16,14 @@ private static string RepoRoot() return dir.FullName; } + // There are only two physically distinct .def files in this repo. `cpp/include/xl` is a git + // symlink to `src/ExcelReader.Native/include`, so the C++ package reaches the canonical + // file through that symlink rather than through a maintained copy of its own - comparing it + // here would just compare the canonical file to itself and could never catch real drift. private static readonly string[] DefPaths = [ Path.Combine("src", "ExcelReader.Native", "include", "excelreader.def"), Path.Combine("rust", "excelreader", "excelreader.def"), - Path.Combine("cpp", "include", "xl", "excelreader.def"), ]; private static string[] ReadExports(string root, string relative) @@ -31,7 +34,7 @@ private static string[] ReadExports(string root, string relative) } [Fact] - public void Should_ListIdenticalExports_When_ComparingTheThreeDefCopies() + public void Should_ListIdenticalExports_When_ComparingTheCanonicalAndRustDefCopies() { string root = RepoRoot(); string[] canonical = ReadExports(root, DefPaths[0]); From 7fc8f800ec5a4d01400b1f032b730e661be68947 Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Sat, 22 Aug 2026 19:47:56 -0300 Subject: [PATCH 37/51] Optimizing string allocation --- rust/excelreader/src/workbook.rs | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/rust/excelreader/src/workbook.rs b/rust/excelreader/src/workbook.rs index 43a5c82..c23ff37 100644 --- a/rust/excelreader/src/workbook.rs +++ b/rust/excelreader/src/workbook.rs @@ -208,7 +208,7 @@ impl Workbook { ) -> Result { // One sized attempt first: Excel caps sheet names at 31 characters, so 128 bytes clears // even the 4-byte-per-character worst case and the retry never runs in practice. - let mut buffer = vec![0u8; 128]; + let mut buffer = [0u8; 128]; let mut len: i32 = 0; let mut status = call( self.handle, @@ -216,18 +216,26 @@ impl Workbook { buffer.len() as i32, &mut len, ); - if status == XL_BUFFER_TOO_SMALL { - buffer = vec![0u8; len.max(0) as usize]; - status = call( - self.handle, - buffer.as_mut_ptr(), - buffer.len() as i32, - &mut len, - ); + if status != XL_BUFFER_TOO_SMALL { + check(status)?; + let buffer_slice = &buffer[..len.max(0) as usize]; + return str::from_utf8(buffer_slice) + .map(|s| s.to_string()) + .map_err(|e| Error { + code: XL_ERROR, + message: format!("native library returned a non-UTF-8 name: {e}"), + }); } + let mut vec_buffer = vec![0u8; len.max(0) as usize]; + status = call( + self.handle, + vec_buffer.as_mut_ptr(), + vec_buffer.len() as i32, + &mut len, + ); check(status)?; - buffer.truncate(len.max(0) as usize); - String::from_utf8(buffer).map_err(|e| Error { + vec_buffer.truncate(len.max(0) as usize); + String::from_utf8(vec_buffer).map_err(|e| Error { code: XL_ERROR, message: format!("native library returned a non-UTF-8 name: {e}"), }) From c155935d1cb1d5fa50a67488969fa0a8eefcb3cd Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Mon, 24 Aug 2026 14:47:07 -0300 Subject: [PATCH 38/51] Implementing schema inference for core C# --- cpp/benchmarks/benchmark_compare.cpp | 20 +- rust/excelreader/tests/parse_arrow.rs | 2 +- src/ExcelReader.Core/Enums/ExcelColumnType.cs | 29 +++ src/ExcelReader.Core/ExcelReader.Core.csproj | 3 + .../PublicAPI/net10.0/PublicAPI.Unshipped.txt | 25 +++ .../PublicAPI/net8.0/PublicAPI.Unshipped.txt | 25 +++ src/ExcelReader.Core/Reader/Excel.cs | 30 +++ .../Reader/ExcelColumnSchema.cs | 31 +++ .../Reader/SchemaInference.cs | 204 ++++++++++++++++++ src/ExcelReader.Native/NativeApi.Arrow.cs | 7 +- src/ExcelReader.Native/NativeApi.Schema.cs | 181 ++-------------- src/ExcelReader.Native/NativeApi.Typed.cs | 28 +-- .../ExcelReader.Tests/SchemaInferenceTests.cs | 179 +++++++++++++++ 13 files changed, 572 insertions(+), 192 deletions(-) create mode 100644 src/ExcelReader.Core/Enums/ExcelColumnType.cs create mode 100644 src/ExcelReader.Core/Reader/ExcelColumnSchema.cs create mode 100644 src/ExcelReader.Core/Reader/SchemaInference.cs create mode 100644 tests/ExcelReader.Tests/SchemaInferenceTests.cs diff --git a/cpp/benchmarks/benchmark_compare.cpp b/cpp/benchmarks/benchmark_compare.cpp index a484688..50107c4 100644 --- a/cpp/benchmarks/benchmark_compare.cpp +++ b/cpp/benchmarks/benchmark_compare.cpp @@ -112,7 +112,7 @@ static void BM_ExcelReader_Xlsx_Full(benchmark::State &state) auto buffer_result = read_file_to_buffer(EXCELREADER_XLSX_FIXTURE_PATH); if (!buffer_result.has_value()) { - state.SkipWithError(buffer_result.error().c_str()); + state.SkipWithError(buffer_result.error()); return; } for (auto _ : state) @@ -120,13 +120,13 @@ static void BM_ExcelReader_Xlsx_Full(benchmark::State &state) auto workbook = xl::Workbook::open_memory(buffer_result.value(), XL_FORMAT_XLSX); if (!workbook.has_value()) { - state.SkipWithError(workbook.error().message.c_str()); + state.SkipWithError(workbook.error().message); return; } auto table = xl::parse_sheet(*workbook); if (!table.has_value()) { - state.SkipWithError(table.error().message.c_str()); + state.SkipWithError(table.error().message); return; } int64_t acc = 0; @@ -149,7 +149,7 @@ static void BM_ExcelReader_Xlsx_ParseOnly(benchmark::State &state) auto buffer_result = read_file_to_buffer(EXCELREADER_XLSX_FIXTURE_PATH); if (!buffer_result.has_value()) { - state.SkipWithError(buffer_result.error().c_str()); + state.SkipWithError(buffer_result.error()); return; } for (auto _ : state) @@ -157,13 +157,13 @@ static void BM_ExcelReader_Xlsx_ParseOnly(benchmark::State &state) auto workbook = xl::Workbook::open_memory(buffer_result.value(), XL_FORMAT_XLSX); if (!workbook.has_value()) { - state.SkipWithError(workbook.error().message.c_str()); + state.SkipWithError(workbook.error().message); return; } auto table = xl::parse_sheet(*workbook); if (!table.has_value()) { - state.SkipWithError(table.error().message.c_str()); + state.SkipWithError(table.error().message); return; } benchmark::DoNotOptimize(table); @@ -176,7 +176,7 @@ static void BM_Xlnt_Xlsx_Full(benchmark::State &state) auto buffer_result = read_file_to_buffer(EXCELREADER_XLSX_FIXTURE_PATH); if (!buffer_result.has_value()) { - state.SkipWithError(buffer_result.error().c_str()); + state.SkipWithError(buffer_result.error()); return; } for (auto _ : state) @@ -293,7 +293,7 @@ static void BM_Xlsxio_Xlsx_Full(benchmark::State &state) auto buffer_result = read_file_to_buffer(EXCELREADER_XLSX_FIXTURE_PATH); if (!buffer_result.has_value()) { - state.SkipWithError(buffer_result.error().c_str()); + state.SkipWithError(buffer_result.error()); return; } auto &buffer = buffer_result.value(); @@ -353,7 +353,7 @@ static void BM_DuckDB_Xlsx_Full(benchmark::State &state) auto setup = con.Query("INSTALL excel; LOAD excel;"); if (setup->HasError()) { - state.SkipWithError(setup->GetError().c_str()); + state.SkipWithError(setup->GetError()); return; } @@ -372,7 +372,7 @@ static void BM_DuckDB_Xlsx_Full(benchmark::State &state) auto result = con.Query(query); if (result->HasError()) { - state.SkipWithError(result->GetError().c_str()); + state.SkipWithError(result->GetError()); return; } int64_t acc = result->GetValue(0, 0); diff --git a/rust/excelreader/tests/parse_arrow.rs b/rust/excelreader/tests/parse_arrow.rs index 1de532b..47534a4 100644 --- a/rust/excelreader/tests/parse_arrow.rs +++ b/rust/excelreader/tests/parse_arrow.rs @@ -21,7 +21,7 @@ fn parse_arrow_returns_a_record_batch_with_one_column_per_field() { let mut workbook = Workbook::open(&fixture_path()).expect("open must succeed"); let batch = parse_arrow::(&mut workbook, 1).expect("parse_arrow must succeed"); - assert_eq!(batch.num_columns(), 2); + assert_eq!(batch.num_columns(), 2); assert_eq!(batch.num_rows(), 100); // RealExcel.xlsb has 100 data rows. // Field names come from the column spec's source name (the `#[excel(name = "...")]` value), // not the Rust struct field identifier - the native side names each Arrow child field from diff --git a/src/ExcelReader.Core/Enums/ExcelColumnType.cs b/src/ExcelReader.Core/Enums/ExcelColumnType.cs new file mode 100644 index 0000000..fc14952 --- /dev/null +++ b/src/ExcelReader.Core/Enums/ExcelColumnType.cs @@ -0,0 +1,29 @@ +namespace ExcelReader.Core.Enums +{ + /// + /// A column's inferred or declared value type, as produced by + /// . + /// + /// + /// The underlying values are fixed: they are the XL_T_* constants of the native C ABI + /// (see src/ExcelReader.Native/include/excelreader.h), which marshals this enum with a + /// plain integer cast rather than a translation table. Renumbering a member is a silent ABI break. + /// + public enum ExcelColumnType + { + /// UTF-8 text. Also the fallback for any column whose sampled cells disagreed. + StringColumn = 0, + /// A 64-bit signed integer. + Int64Column = 1, + /// A 64-bit IEEE 754 floating-point number. + Float64Column = 2, + /// A boolean. + BoolColumn = 3, + /// A whole date, counted in days since 1970-01-01. + DateColumn = 4, + /// A time of day, counted in microseconds since midnight. + TimeColumn = 5, + /// A date and time, counted in microseconds since 1970-01-01T00:00:00Z. + TimestampColumn = 6, + } +} \ No newline at end of file diff --git a/src/ExcelReader.Core/ExcelReader.Core.csproj b/src/ExcelReader.Core/ExcelReader.Core.csproj index dc32be1..9480f19 100644 --- a/src/ExcelReader.Core/ExcelReader.Core.csproj +++ b/src/ExcelReader.Core/ExcelReader.Core.csproj @@ -41,6 +41,9 @@ + + + net10.0 + ExcelReader.Cli + ExcelReader.Cli + true + excelreader + ExcelReader.NET.Cli + 1.0.0 + Command-line workbook inspection and CSV conversion, built on ExcelReader.NET. + true + true + + + + $(NoWarn);RS0016;RS0026;RS0037;IDISP001;CA1822 + + + + + + + + + + + + + + all + runtime + compile; build; native; contentfiles; analyzers; buildtransitive + + + + diff --git a/src/ExcelReader.Cli/Program.cs b/src/ExcelReader.Cli/Program.cs new file mode 100644 index 0000000..c3013cf --- /dev/null +++ b/src/ExcelReader.Cli/Program.cs @@ -0,0 +1,18 @@ +using ConsoleAppFramework; + +namespace ExcelReader.Cli +{ + // An explicit entry-point class rather than top-level statements: ExcelReader.Tests is itself an + // Exe, and a generated global-namespace Program would collide with its test host's entry point + // once this project is referenced. + internal static class Program + { + internal static int Main(string[] args) + { + ConsoleApp.ConsoleAppBuilder app = ConsoleApp.Create(); + app.Add(); + app.Run(args); + return Environment.ExitCode; + } + } +} diff --git a/tests/ExcelReader.Tests/CliTests.cs b/tests/ExcelReader.Tests/CliTests.cs new file mode 100644 index 0000000..60a34c4 --- /dev/null +++ b/tests/ExcelReader.Tests/CliTests.cs @@ -0,0 +1,189 @@ +using ExcelReader.Cli; + +namespace ExcelReader.Tests +{ + public sealed class CliTests + { + private static string Fixture(string name) => Path.Combine("data", name); + + private static (int Code, string Out, string Err) Sheets(string path) + { + StringWriter stdout = new(); + StringWriter stderr = new(); + int code = CliCommands.Sheets(path, stdout, stderr); + return (code, stdout.ToString(), stderr.ToString()); + } + + [Fact] + public void Should_ListEverySheet_When_RunningSheets() + { + (int code, string output, string error) = Sheets(Fixture("RealExcel.xlsb")); + + Assert.Equal(0, code); + Assert.Empty(error); + + // One line per sheet, "\t". + string[] lines = output.Split('\n', StringSplitOptions.RemoveEmptyEntries); + Assert.NotEmpty(lines); + Assert.StartsWith("0\t", lines[0], StringComparison.Ordinal); + } + + [Fact] + public void Should_ReturnOneAndWriteToStderr_When_TheFileDoesNotExist() + { + (int code, string output, string error) = Sheets( + Path.Combine(Path.GetTempPath(), "no-such-workbook.xlsx")); + + Assert.Equal(1, code); + Assert.Empty(output); + Assert.NotEmpty(error); + // A message the user can act on, not a stack trace. + Assert.DoesNotContain(" at ", error, StringComparison.Ordinal); + } + + [Fact] + public void Should_SelectBySheetName_When_SheetIsNotNumeric() + { + using ExcelReader.Core.Reader.IExcelRowReader byIndex = CliCommands.Open(Fixture("RealExcel.xlsb"), "0"); + string firstSheetName = byIndex.SheetName; + + using ExcelReader.Core.Reader.IExcelRowReader byName = CliCommands.Open(Fixture("RealExcel.xlsb"), firstSheetName); + Assert.Equal(firstSheetName, byName.SheetName); + } + + [Fact] + public void Should_Throw_When_TheNamedSheetIsAbsent() + { + ArgumentException error = Assert.Throws( + () => CliCommands.Open(Fixture("RealExcel.xlsb"), "NoSuchSheet")); + + Assert.Contains("NoSuchSheet", error.Message, StringComparison.Ordinal); + } + + [Fact] + public void Should_DefaultToTheFirstSheet_When_SheetIsNull() + { + using ExcelReader.Core.Reader.IExcelRowReader reader = CliCommands.Open(Fixture("RealExcel.xlsb"), null); + + Assert.Equal(reader.SheetNameAt(0), reader.SheetName); + } + + private static (int Code, string Out, string Err) Convert( + string path, string? sheet = null, string? output = null, char delimiter = ',') + { + using MemoryStream stdout = new(); + StringWriter stderr = new(); + int code = CliCommands.Convert(path, sheet, output, delimiter, stdout, stderr); + return (code, System.Text.Encoding.UTF8.GetString(stdout.ToArray()), stderr.ToString()); + } + + [Fact] + public void Should_WriteCsvToTheOutputStream_When_NoOutputFileIsGiven() + { + (int code, string output, string error) = Convert(Fixture("RealExcel.xlsb")); + + Assert.Equal(0, code); + Assert.Empty(error); + Assert.NotEmpty(output); + Assert.Contains(",", output, StringComparison.Ordinal); + } + + [Fact] + public void Should_WriteCsvToAFile_When_OutputIsGiven() + { + string target = Path.Combine(Path.GetTempPath(), $"excelreader-cli-{Guid.NewGuid():N}.csv"); + try + { + (int code, string output, string error) = Convert(Fixture("RealExcel.xlsb"), output: target); + + Assert.Equal(0, code); + Assert.Empty(error); + // Nothing on the stdout stream: the CSV went to the file. + Assert.Empty(output); + Assert.True(File.Exists(target)); + Assert.NotEmpty(File.ReadAllText(target)); + } + finally + { + File.Delete(target); + } + } + + [Fact] + public void Should_UseTheGivenDelimiter_When_DelimiterIsOverridden() + { + (int code, string output, _) = Convert(Fixture("RealExcel.xlsb"), delimiter: ';'); + + Assert.Equal(0, code); + Assert.Contains(";", output, StringComparison.Ordinal); + } + + [Fact] + public void Should_ReturnOne_When_TheNamedSheetIsAbsent() + { + (int code, _, string error) = Convert(Fixture("RealExcel.xlsb"), sheet: "NoSuchSheet"); + + Assert.Equal(1, code); + Assert.Contains("NoSuchSheet", error, StringComparison.Ordinal); + } + + private static (int Code, string Out, string Err) Schema( + string path, string? sheet = null, int headerRow = 1, int sampleSize = 100) + { + StringWriter stdout = new(); + StringWriter stderr = new(); + int code = CliCommands.Schema(path, sheet, headerRow, sampleSize, stdout, stderr); + return (code, stdout.ToString(), stderr.ToString()); + } + + [Fact] + public void Should_PrintOneLinePerColumn_When_RunningSchema() + { + (int code, string output, string error) = Schema(Fixture("RealExcel.xlsb")); + + Assert.Equal(0, code); + Assert.Empty(error); + + string[] lines = output.Split('\n', StringSplitOptions.RemoveEmptyEntries); + Assert.NotEmpty(lines); + // "\t\t[?]" + foreach (string line in lines) + { + string[] parts = line.TrimEnd('\r').Split('\t'); + Assert.Equal(3, parts.Length); + Assert.True(int.TryParse(parts[0], System.Globalization.CultureInfo.InvariantCulture, out _)); + Assert.NotEmpty(parts[2]); + } + } + + [Fact] + public void Should_PrintAnEmptyNameField_When_HeaderRowIsZero() + { + (int code, string output, _) = Schema(Fixture("RealExcel.xlsb"), headerRow: 0); + + Assert.Equal(0, code); + // A null name renders as an empty middle field, never as the literal "null". + string firstLine = output.Split('\n', StringSplitOptions.RemoveEmptyEntries)[0].TrimEnd('\r'); + Assert.Equal(string.Empty, firstLine.Split('\t')[1]); + } + + [Fact] + public void Should_MarkNullableColumnsWithAQuestionMark_When_PrintingTheSchema() + { + // A CSV with a gap guarantees at least one nullable column, independent of the fixture. + string target = Path.Combine(Path.GetTempPath(), $"excelreader-cli-{Guid.NewGuid():N}.csv"); + File.WriteAllText(target, "Id,Note\n1,here\n2,\n"); + try + { + (int code, string output, _) = Schema(target); + + Assert.Equal(0, code); + Assert.Contains("?", output, StringComparison.Ordinal); + } + finally + { + File.Delete(target); + } + } + } +} diff --git a/tests/ExcelReader.Tests/ExcelReader.Tests.csproj b/tests/ExcelReader.Tests/ExcelReader.Tests.csproj index 916a831..09b17f9 100644 --- a/tests/ExcelReader.Tests/ExcelReader.Tests.csproj +++ b/tests/ExcelReader.Tests/ExcelReader.Tests.csproj @@ -49,6 +49,11 @@ + + +