diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 43797b6..07643f2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -56,6 +56,19 @@ jobs: -p:Version=${{ steps.version.outputs.value }} --output ./artifacts + # Portable IL tool package (see the csproj: "a dotnet tool ships as IL and is launched by + # whatever SDK the user has") - not the self-contained AOT binaries publish-cli attaches to + # the GitHub Release below, which need no .NET runtime installed but must be built per-RID. + # `dotnet tool install --global ExcelReader.NET.Cli` (README's documented install command) + # resolves this package. + - name: Pack CLI + run: >- + dotnet pack src/ExcelReader.Cli/ExcelReader.Cli.csproj + --configuration Release + --no-restore + -p:Version=${{ steps.version.outputs.value }} + --output ./artifacts + - name: Generate SBOM uses: anchore/sbom-action@v0 with: @@ -127,6 +140,76 @@ jobs: files: ${{ steps.asset.outputs.name }} fail_on_unmatched_files: true + publish-cli: + name: Publish excelreader CLI (${{ matrix.os }}) + needs: publish + runs-on: ${{ matrix.os }} + permissions: + contents: write + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Setup .NET + uses: actions/setup-dotnet@v6 + with: + global-json-file: global.json + + - name: Derive version from tag + id: version + shell: bash + run: echo "value=${GITHUB_REF_NAME#v}" >> "$GITHUB_OUTPUT" + + - name: Determine RID + id: rid + shell: bash + run: | + arch=$([ "$(uname -m 2>/dev/null || echo x64)" = "arm64" ] && echo arm64 || echo x64) + case "${{ runner.os }}" in + Windows) os=win ;; + macOS) os=osx ;; + *) os=linux ;; + esac + echo "value=${os}-${arch}" >> "$GITHUB_OUTPUT" + + # PublishAot/PackAsTool=false override the project's normal dotnet-tool packaging (a + # dotnet tool ships as IL, launched by the caller's own SDK) with a single self-contained + # native executable instead - no .NET runtime required on the machine that runs it. + # ConsoleAppFramework is already compile-time only and Spectre.Console is AOT-compatible, + # so this publishes clean with zero IL2xxx/IL3xxx trim warnings. + - name: Publish native AOT binary + shell: bash + run: >- + dotnet publish src/ExcelReader.Cli/ExcelReader.Cli.csproj + --configuration Release + -r ${{ steps.rid.outputs.value }} + -p:PublishAot=true + -p:PackAsTool=false + -p:Version=${{ steps.version.outputs.value }} + --output ./cli-publish + + - name: Rename asset for this platform + id: asset + shell: bash + run: | + case "${{ runner.os }}" in + Windows) src=ExcelReader.Cli.exe; ext=.exe ;; + *) src=ExcelReader.Cli; ext= ;; + esac + name="excelreader-${{ steps.rid.outputs.value }}${ext}" + cp "./cli-publish/${src}" "${name}" + echo "name=${name}" >> "$GITHUB_OUTPUT" + + - name: Upload to the GitHub Release + uses: softprops/action-gh-release@v3 + with: + files: ${{ steps.asset.outputs.name }} + fail_on_unmatched_files: true + publish-rust: name: Publish Rust crates needs: publish-native-assets diff --git a/.gitignore b/.gitignore index f0ec5d4..87734d6 100644 --- a/.gitignore +++ b/.gitignore @@ -503,4 +503,6 @@ python/src/excelreader/_lib/* # CMake build output (tests/ExcelReader.NativeSmoke and any other CMake-configured project). Never # built to a path inside the source tree other than this, so a plain top-level rule is enough. -build/ \ No newline at end of file +build/ +# Subagent-driven-development scratch (per-plan ledgers, briefs, review packages) +.superpowers/ diff --git a/.vscode/settings.json b/.vscode/settings.json index e738329..f101367 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -11,5 +11,8 @@ "-DEXCELREADER_BUILD_BENCHMARKS_COMPARE=ON", "-DEXCELREADER_BUILD_TESTS=ON", "-DCMAKE_POLICY_VERSION_MINIMUM=3.5" - ] + ], + "rust-analyzer.server.extraEnv": { + "EXCELREADER_NATIVE_LIB_DIR": "${workspaceFolder}/python/src/excelreader/_lib" + } } \ No newline at end of file diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 37c9c24..da79838 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -48,6 +48,28 @@ Reader internals that would otherwise be duplicated four times over live in one the single buffer-growth-cap function (`NextBufferSize`) every pooled buffer in the stack grows through, so one limit policy governs all of them consistently. +## The `excelreader` CLI + +`src/ExcelReader.Cli/` is a thin `dotnet tool` shell (`excelreader`) over Core's public API — it +parses no spreadsheet bytes of its own. It splits in two on purpose: `Commands` is a one-line-per- +command adapter whose XML doc comments ConsoleAppFramework's source generator turns into argument +parsing, routing and `--help`, while `CliCommands` holds the bodies as plain functions over explicit +writers. The split keeps the tested surface free of the framework's static output hooks, so the CLI +tests run in parallel like every other test class. ConsoleAppFramework is compile-time only +(`PrivateAssets`), so the published tool's only *runtime* dependency besides ExcelReader.Core is +Spectre.Console, used for `sheets`/`schema`'s tables and `convert`'s progress spinner on a real +terminal. + +Rendering follows `Commands`/`CliCommands`'s own split, one level further: `Commands` picks plain vs. +interactive per call (`Console.IsOutputRedirected`/`IsErrorRedirected`), so a script gets the exact +same tab-separated text and stderr line the tool always wrote, unchanged. Two small always-stderr +helpers back that split - `ErrorConsole` (a Spectre `IAnsiConsole` pinned to `Console.Error`, since +Spectre's own default instance targets stdout, which `convert` may be using for the converted bytes +themselves) and `ColorizingErrorWriter` (a `TextWriter` that renders `CliCommands.Execute`'s one-line +failure in red through `ErrorConsole` on a terminal, or passes it through byte-for-byte otherwise). +Both live in `ExcelReader.Cli`, not `CliCommands.cs` - the interactive/plain decision is +`Console`-shaped state, exactly what that file's tests are built to never touch. + ## Why readers are split into partial classes `XlsxReader` and `XlsbReader` are large enough that one file would be unwieldy, so each is split by diff --git a/ExcelReader.slnx b/ExcelReader.slnx index aa5b4a8..4660d92 100644 --- a/ExcelReader.slnx +++ b/ExcelReader.slnx @@ -1,5 +1,6 @@ + diff --git a/README.md b/README.md index ad5f04c..899565e 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,32 @@ ExcelReader is built for streaming spreadsheet workloads where low allocations m dotnet add package ExcelReader.NET ``` +## Command line + +```bash +dotnet tool install --global ExcelReader.NET.Cli +``` + +```bash +excelreader sheets book.xlsb # 0Sheet1 +excelreader schema book.xlsb --sample-size 500 # 0IdInt64 +excelreader convert book.xlsb --output data.csv # convert to another format, by extension +excelreader convert book.xlsb --output data.xlsx # .xlsx, .xlsb, .xls and .csv all work +excelreader convert book.xlsb --format xlsx | head -c 4 # to stdout, --format picks it instead +``` + +Flags: `--sheet|-s `, `--header-row N` (0 = no header), `--sample-size N`, +`--output|-o `, `--format|-f ` (defaults to `--output`'s extension, or csv +for stdout), `--delimiter|-d ` (csv only). Run `excelreader --help` for the full list. + +On a real terminal, `sheets`/`schema` render as a [Spectre.Console](https://spectreconsole.net) table +and `convert` shows a progress spinner on stderr; piped or redirected (a script, `| head`, `2>file`), +every command falls back to the same plain tab-separated text and stderr line it always wrote, so +nothing here changes for anything already parsing this tool's output. + +Exit codes are `0` ok and `1` failure; results go to stdout and errors to stderr, so `convert` is +safe to pipe. + ## Read rows ```csharp @@ -863,13 +889,13 @@ dotnet test --project tests/ExcelReader.Tests/ExcelReader.Tests.csproj --configu ## Other languages -ExcelReader ships a NativeAOT shared library with a C ABI, so non-.NET languages can read XLSX, -XLSB, XLS and CSV without a .NET runtime installed. +ExcelReader ships a NativeAOT shared library with a C ABI, so non-.NET languages can read and write +XLSX, XLSB, XLS and CSV without a .NET runtime installed. - C ABI header: [`src/ExcelReader.Native/include/excelreader.h`](src/ExcelReader.Native/include/excelreader.h) - Python package: [`python/`](python/README.md) -- C++ package: header-only CMake wrapper, `xl::Workbook`/`xl::parse_sheet` over the same ABI — see [`cpp/README.md`](cpp/README.md). -- Rust crate: safe `Workbook`/`parse_sheet` bindings, downloadable via `cargo add excelreader` — see [`rust/excelreader/README.md`](rust/excelreader/README.md). +- C++ package: header-only CMake wrapper, `xl::Workbook`/`xl::parse_sheet`/`xl::write_sheet` over the same ABI — see [`cpp/README.md`](cpp/README.md). +- Rust crate: safe `Workbook`/`parse_sheet`/`write_sheet` bindings, downloadable via `cargo add excelreader` — see [`rust/excelreader/README.md`](rust/excelreader/README.md). ```python from excelreader import open_workbook @@ -879,7 +905,28 @@ with open_workbook("book.xlsx") as workbook: print([cell.value for cell in row]) ``` -Reading only — the writers are not exposed across the ABI yet. +Writing goes through one export, `xl_write_typed`: a whole sheet in a single call, from columnar +buffers the ABI borrows rather than copies. All three bindings expose it — Python as +`write_workbook`/`write_arrow`/`write_pandas`/`write_polars`, C++ as `xl::write_columns` and +`xl::write_sheet`, Rust as `writer::write_columns` and `writer::write_sheet`. In C++ and Rust the +same struct mapping drives both directions, so reading a sheet and writing it back needs one +mapping, not two: + +```rust +use excelreader::writer::write_sheet; +use excelreader::XL_FORMAT_XLSX; + +write_sheet("out.xlsx", XL_FORMAT_XLSX, &rows, None)?; +``` + +```cpp +auto written = xl::write_sheet("out.xlsx", rows); // format inferred from the extension +``` + +Row-by-row decoded reads remain Python-only. The Arrow export is available from Python +(`to_arrow`/`to_record_batch`), C++ (`xl::parse_arrow`, in the separate `` +header — no Apache Arrow C++ dependency, you get the raw C Data Interface pair), and Rust +(`excelreader::arrow::parse_arrow`, behind the `arrow` cargo feature, returning an `arrow::array::RecordBatch`). ## Contributing diff --git a/cpp/README.md b/cpp/README.md index 947a423..bd0053d 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -1,9 +1,9 @@ # excelreader (C++) Header-only C++23 wrapper around ExcelReader's native C ABI: opening a workbook (from a path or -memory, with the full open-options surface), sheet navigation, schema inference, and schema-driven -typed table parsing. No writing, no Arrow, no row-by-row decode yet — see the root README's Python -section for what those look like. +memory, with the full open-options surface), sheet navigation, schema inference, schema-driven +typed table parsing, and schema-driven writing. No Arrow, no row-by-row decode yet — see the root +README's Python section for what those look like. ## Requirements @@ -78,6 +78,68 @@ for (const auto& column : *workbook->infer_schema(1, 100)) { Every entry point returns `std::expected` — this header throws nothing. +### Arrow export + +`` is a separate header — including `` never pulls the +Arrow C Data Interface declarations in. It does not depend on the Apache Arrow C++ library: you get +the raw `ArrowArray`/`ArrowSchema` pair, owned by an RAII `xl::ArrowTable`, to hand to whichever +Arrow implementation you already link. + +```cpp +#include + +auto workbook = xl::Workbook::open("book.xlsx"); +auto table = xl::parse_arrow(*workbook); +// table->array / table->schema are a top-level struct array; both release in ~ArrowTable. +``` + +## Writing + +Two layers, mirroring the two on the reading side. + +`xl::write_sheet` uses the same `xl::ExcelMapper` specialization `xl::parse_sheet` reads +with, so a round trip needs one mapping, not two: + +```cpp +std::vector rows = /* ... */; +auto written = xl::write_sheet("out.xlsx", rows); // format inferred from the extension +if (!written) { + std::fprintf(stderr, "%s\n", written.error().message.c_str()); +} +``` + +If you already hold columnar buffers, `xl::write_columns` hands them to the ABI with **no copy** — +they are borrowed for the duration of the call and must outlive it: + +```cpp +std::vector ids{1, 2, 3}; +std::vector values{0.5, 1.5, 2.5}; +std::array columns{ + xl::i64_column("id", ids), + xl::f64_column("value", values)}; + +auto written = xl::write_columns("out.xlsx", XL_FORMAT_XLSX, columns); +``` + +One constructor per column type: `i64_column`, `f64_column`, `bool_column`, `date_column`, +`time_column`, `timestamp_column`, and `string_column` (which takes an `int32` offsets span of +`rows + 1` entries plus the UTF-8 blob). + +A nullable column is a values buffer plus an LSB-first validity bitmap — bit set means the row is +valid — passed as the last argument to any of those constructors. `write_columns` checks the bitmap +is long enough for the row count before calling: the ABI takes it without a length and reads +`(rows + 7) / 8` bytes on trust, so a short one would be a buffer overrun. On the struct side, +declare the field `std::optional` and `write_sheet` builds the bitmap for you. + +`xl::WriteOptions` sets the sheet name, the CSV dialect, and the XLS/XLSB and XLSX/XLSB toggles. +`XL_FORMAT_AUTO` is rejected — a file being created has no signature bytes to sniff — so +`xl::format_from_path` returning `XL_FORMAT_AUTO` for an unrecognized extension surfaces as a failed +write rather than a silently chosen format. + +`write_sheet` walks the range once and appends each field to its own column buffer, with the +per-field dispatch resolved at compile time. That transpose is the only copy it makes, and it is +what the ABI's columnar shape costs a row-shaped caller; `write_columns` pays nothing. + ## Bounds and ABI `TableView::operator[]` is unchecked, like `std::vector`'s. Use `TableView::at(row)`, which returns @@ -134,3 +196,108 @@ cmake --build build --config Release --target excelreader_cpp_benchmarks Add `-DEXCELREADER_BUILD_BENCHMARKS_COMPARE=ON -DCMAKE_POLICY_VERSION_MINIMUM=3.5` (xlnt's own `CMakeLists.txt` predates CMake's minimum-version floor) and build/run `excelreader_cpp_compare_benchmarks` for the xlnt/xlsxio comparison. + +### Writing + +`excelreader_cpp_write_benchmarks` (same `-DEXCELREADER_BUILD_BENCHMARKS=ON` flag) measures the two +write layers against each other over 7 columns of the same fixture. Both cases write the same +columns, so the gap between them is only the cost of starting from row-shaped data: `BM_WriteColumns` +is handed buffers that are already columnar, while `BM_WriteSheet` starts from a +`std::vector` and pays the row-to-column transpose. + +| Benchmark | Time | Rows/s | +|---|---:|---:| +| `BM_WriteColumns` (pre-transposed) | 60.2 ms | 1.07 M/s | +| `BM_WriteSheet` (from `std::vector`) | 67.4 ms | 961 k/s | + +The transpose costs ~12% here. It is not free, but it is far from the dominant cost of producing +the file — see the comparison below, where the same two cases over 14 columns land within ~7%. + +`excelreader_cpp_write_compare_benchmarks` (under `-DEXCELREADER_BUILD_BENCHMARKS_COMPARE=ON`) puts +that against xlnt, xlsxio and [DuckDB](https://github.com/duckdb/duckdb)'s `excel` extension, all +writing the full 14-column, 65,535-row shape of `65K_Records_Data.xlsx` from the same in-memory +rows. +[libxlsxwriter](https://github.com/jmcnamara/libxlsxwriter) — same author as rust_xlsxwriter, and, +like xlsxio, a streaming C writer with no document-model overhead — is measured separately, by +`excelreader_cpp_write_compare_lxw_benchmarks`: xlsxio and libxlsxwriter each vendor their own +incompatible copy of minizip and export the same C symbols (`zipOpen`, `zipOpenNewFileInZip`, ...), +so linking both into one binary let calls cross between the two implementations and corrupted +xlsxio's output — see [Known issue](#known-issue-xlsxio--libxlsxwriter-cannot-share-a-binary) below. +Building both write-compare targets and running them back to back is required to see every +competitor. + +| Library | Wall | CPU | +|---|---:|---:| +| ExcelReader (`xl::write_columns`, pre-transposed) | 126.3–128.1 ms | 127.6 ms | +| ExcelReader (`xl::write_sheet`) | 135.7–136.6 ms | 137.5 ms | +| DuckDB (`COPY ... TO ... WITH (FORMAT xlsx)`) | 826.3–863.6 ms | 828.1–843.8 ms | +| libxlsxwriter (`worksheet_write_string`/`_number`) | 1,188.0 ms | 1,187.5 ms | +| xlsxio (`xlsxiowrite_add_cell_*`) | 2,450.6 ms | 1,171.9 ms | +| xlnt (`worksheet::cell().value()` + `save()`) | 5,737.9 ms | 5,703.1 ms | + +Same machine as above; Google Benchmark's own iteration counts, no `--benchmark_repetitions` (each +iteration writes a whole 65,535-row file, so the slower cases run once or a handful of times). xlnt +and xlsxio only build into `excelreader_cpp_write_compare_benchmarks`; libxlsxwriter only into +`excelreader_cpp_write_compare_lxw_benchmarks` (see the known issue below) — the ExcelReader and +DuckDB rows appear in both, and the small ranges above are those two independent runs, not repeated +sampling within one run. + +`write_sheet` — the matched-work number, since it starts from the same `std::vector` every +competitor is handed — is ~6.1–6.4x faster than DuckDB, ~8.7x faster than libxlsxwriter, ~18.1x +faster than xlsxio, and ~42x faster than xlnt on wall time. + +### Known issue: xlsxio + libxlsxwriter cannot share a binary + +An earlier version of `excelreader_cpp_write_compare_benchmarks` linked xlnt, xlsxio, +libxlsxwriter and DuckDB into one executable. xlsxio (built against minizip-ng's compat layer, +whose `zipOpenNewFileInZip` takes `uint16_t` extrafield sizes) and libxlsxwriter (which vendors +classic minizip, whose same-named function takes 32-bit `uInt` sizes and starts with `if +(size_extrafield_local > 0xffff) return ZIP_PARAMERROR;` — a check that cannot exist in the +minizip-ng version) both export identical C symbol names from a static library linked into that one +binary. The linker kept exactly one definition of each name, so a call could resolve to the wrong +implementation — a `zipFile` opened by one library's `zipOpen` got handed to the other library's +`zipOpenNewFileInZip`, which read it through an incompatible struct layout. That is what produced +`Error creating file "xl/workbook.xml" inside zip file` on xlsxio's background thread in Release +builds (Debug's different link order happened not to trigger it) — and it was silent otherwise: +`xlsxiowrite_close()` still returned success with the workbook.xml entry missing, so the benchmark +published a timing for a file that was skipping work, not a valid xlsx. + +The fix is structural: xlsxio and libxlsxwriter now build into two separate executables +(`excelreader_cpp_write_compare_benchmarks` and `excelreader_cpp_write_compare_lxw_benchmarks`, +both compiled from `benchmark_write_compare.cpp` under an `#ifdef`) that are never linked together. +Every case in both executables also reopens the file it just wrote, outside the timed loop, before +trusting its own timing — a writer that silently drops a required part now fails the benchmark +instead of publishing a number for a broken file. + +**libxlsxwriter's number barely moved after the fix** (1,188.0 ms wall vs. 1,226.9 ms pre-fix, ~3%, +inside normal run-to-run noise) and its CPU stayed essentially equal to wall time both before and +after — consistent with the corruption running through xlsxio's calls resolving into +libxlsxwriter's minizip, not the reverse: libxlsxwriter's own output was apparently never affected. +That is a plausible explanation for the asymmetry, not a second confirmed fact — the collision could +in principle run either direction depending on link order, and this project isn't going to re-derive +MSVC's exact symbol-resolution algorithm to be certain which way it went here. + +Caveats, none of them optional when quoting these: + +- **xlsxio's CPU time moved after the fix**, from 843.8–906.3 ms (pre-fix, some runs missing the + workbook.xml entry) to a confirmed 1,171.9 ms — about 30% more real work, consistent with the + entry no longer being silently dropped. Wall time barely moved (2,450.6 ms vs. 2,383.5–2,453.9 ms), + because xlsxio's wall time is dominated by I/O wait either way: CPU is now ~48% of wall, + against every other case here being CPU-bound (wall ≈ CPU). Against CPU time the gap from + `write_sheet` to xlsxio is ~8.5x, not ~18.1x — which number is the honest one depends on what you + are asking: the wall-time ratio is what a caller waits, the CPU-time ratio is what the library + costs. +- **ExcelReader does slightly more work here**, not less: it attaches a number format to the two + date columns so Excel shows a date, while every competitor case writes those as bare serial + numbers. That difference favours the competitors. +- **xlnt builds a full document model** (styles, formats, formulas) before serializing, which is + more than this library exposes at all. Its number reflects a different feature set, not only a + slower path. +- **DuckDB's rows are loaded via its Appender API before the timed region**, so its number measures + `COPY ... TO ... xlsx` alone — the same treatment `write_columns` gets for its transpose. DuckDB is + a full analytical query engine doing far more than any Excel-writing library here; this measures + one narrow slice of it, not "DuckDB" as a whole. + +The two ExcelReader cases land within ~6% of each other, which is the interesting internal result: +the row-to-column transpose is nearly free next to the cost of producing the file. Reach for +`write_columns` when your data is already columnar, but `write_sheet` is not the slow path. diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index e5f160e..d5b9487 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -33,10 +33,23 @@ if(WIN32) "$") endif() -# --- Comparison benchmark against xlnt (https://github.com/tfussell/xlnt) and xlsxio -# (https://github.com/brechtsanders/xlsxio) - separate opt-in flag because both are much heavier -# FetchContent builds (their own zip/xml stacks) than the suite above. -option(EXCELREADER_BUILD_BENCHMARKS_COMPARE "Build the excelreader-vs-xlnt-vs-xlsxio comparison benchmark" OFF) +add_executable(excelreader_cpp_write_benchmarks benchmark_write.cpp) +target_link_libraries(excelreader_cpp_write_benchmarks PRIVATE xl::excelreader benchmark::benchmark_main) +target_compile_definitions(excelreader_cpp_write_benchmarks PRIVATE + EXCELREADER_LARGE_FIXTURE_PATH="${EXCELREADER_LARGE_FIXTURE_PATH}") + +if(WIN32) + add_custom_command(TARGET excelreader_cpp_write_benchmarks POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "$" + "$") +endif() + +# --- Comparison benchmark against xlnt (https://github.com/tfussell/xlnt), xlsxio +# (https://github.com/brechtsanders/xlsxio) and libxlsxwriter +# (https://github.com/jmcnamara/libxlsxwriter) - separate opt-in flag because all three are much +# heavier FetchContent builds (their own zip/xml stacks) than the suite above. +option(EXCELREADER_BUILD_BENCHMARKS_COMPARE "Build the excelreader-vs-xlnt-vs-xlsxio-vs-libxlsxwriter-vs-duckdb comparison benchmark" OFF) if(EXCELREADER_BUILD_BENCHMARKS_COMPARE) set(STATIC ON CACHE BOOL "" FORCE) set(STATIC_CRT OFF CACHE BOOL "" FORCE) @@ -133,20 +146,164 @@ if(EXCELREADER_BUILD_BENCHMARKS_COMPARE) PUBLIC BUILD_XLSXIO_STATIC) target_link_libraries(xlsxio_read PRIVATE MINIZIP::minizip expat::expat) - add_executable(excelreader_cpp_compare_benchmarks benchmark_compare.cpp) - target_link_libraries(excelreader_cpp_compare_benchmarks PRIVATE - xl::excelreader benchmark::benchmark_main xlnt xlsxio_read) + # xlsxio's write side is a separate translation unit from its read side, with no shared state - + # it needs minizip's zip.h (same compat shim as above) but not expat, since writing parses no + # XML. + add_library(xlsxio_write STATIC "${xlsxio_SOURCE_DIR}/lib/xlsxio_write.c") + target_include_directories(xlsxio_write + PUBLIC "${xlsxio_SOURCE_DIR}/include" + PRIVATE "${xlsxio_SOURCE_DIR}/lib" "${_xlsxio_minizip_shim}") + target_compile_definitions(xlsxio_write + PRIVATE BUILD_XLSXIO USE_MINIZIP + PUBLIC BUILD_XLSXIO_STATIC) + target_link_libraries(xlsxio_write PRIVATE MINIZIP::minizip) + + # libxlsxwriter (https://github.com/jmcnamara/libxlsxwriter) - unlike xlsxio, its own + # CMakeLists.txt IS FetchContent-friendly enough to add_subdirectory (no REQUIRED dependency on + # anything already installed except zlib), so it is consumed as-is rather than hand-compiled. + # + # Its one hard requirement is `find_package(ZLIB "1.2.8" REQUIRED)`, called from ITS OWN + # CMakeLists.txt with no FetchContent hook of its own. madler/zlib's CMakeLists.txt has no + # matching Config-mode package (find_package(ZLIB) resolves through CMake's bundled + # Modules/FindZLIB.cmake, a Find-module, not a Config file), so FetchContent's + # OVERRIDE_FIND_PACKAGE keyword - the usual fix for "vendored project calls find_package(X + # REQUIRED) with nothing to redirect to" - does not apply to it either. + # + # The fix: fetch zlib ourselves, then place our OWN FindZLIB.cmake ahead of CMake's built-in one + # on CMAKE_MODULE_PATH. find_package(ZLIB) is a MODULE-mode call, and MODULE mode searches + # CMAKE_MODULE_PATH before it ever reaches CMake's own Modules/ directory - so ours intercepts + # the call and just aliases the target we already built. An ALIAS (not a hardcoded file path) + # is what makes this work under a multi-config generator (Visual Studio): the actual .lib file + # only exists once a config is chosen at BUILD time, so anything resolved at CONFIGURE time + # (which is when find_library would run) cannot know its path - a target alias defers that + # resolution to build time, same as every other target reference in this file. + FetchContent_Declare( + zlib + GIT_REPOSITORY https://github.com/madler/zlib.git + GIT_TAG v1.3.1 + ) + FetchContent_MakeAvailable(zlib) + # madler/zlib's CMakeLists.txt sets its include path via the directory-scoped + # include_directories(), not target_include_directories(PUBLIC/INTERFACE) - so a target built + # OUTSIDE that subdirectory (libxlsxwriter's own sources, added via a separate add_subdirectory + # below) does NOT inherit it just by linking against zlibstatic. Add it back explicitly: zlib.h + # lives in the fetched source tree, zconf.h is generated into the build tree at configure time. + target_include_directories(zlibstatic INTERFACE "${zlib_SOURCE_DIR}" "${zlib_BINARY_DIR}") + + set(_zlib_shim_dir "${CMAKE_CURRENT_BINARY_DIR}/zlib-find-module-shim") + file(MAKE_DIRECTORY "${_zlib_shim_dir}") + file(WRITE "${_zlib_shim_dir}/FindZLIB.cmake" [=[ +# Generated by cpp/benchmarks/CMakeLists.txt - redirects find_package(ZLIB) to the zlibstatic +# target this build already fetched and compiled via FetchContent, instead of searching for an +# installed system zlib. See the comment above FetchContent_Declare(zlib ...) for why this exists. +if(NOT TARGET ZLIB::ZLIB) + add_library(ZLIB::ZLIB ALIAS zlibstatic) +endif() +set(ZLIB_FOUND TRUE) +set(ZLIB_VERSION_STRING "1.3.1") +]=]) + list(PREPEND CMAKE_MODULE_PATH "${_zlib_shim_dir}") + + set(BUILD_TESTS OFF CACHE BOOL "" FORCE) + set(BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) + FetchContent_Declare( + libxlsxwriter + GIT_REPOSITORY https://github.com/jmcnamara/libxlsxwriter.git + GIT_TAG v1.2.4 + ) + FetchContent_MakeAvailable(libxlsxwriter) + + # DuckDB (https://github.com/duckdb/duckdb) - via its "excel" extension, DuckDB reads and + # writes .xlsx through read_xlsx()/COPY ... TO ... WITH (FORMAT xlsx). Unlike xlnt/xlsxio/ + # libxlsxwriter, this is NOT built from source: DuckDB publishes a prebuilt libduckdb.{dll,so, + # dylib} + duckdb.h/duckdb.hpp per platform as GitHub Release assets (same shape as this + # project's own excelreader-native-*.{dll,so,dylib} - see cmake/FetchNativeLib.cmake), so it is + # downloaded the same way rather than compiled - DuckDB's amalgamated source is enormous and + # would dwarf every other FetchContent build in this file. + # + # On Windows the zip already ships a real import library (duckdb.lib) - no lib.exe/dlltool + # workaround needed, unlike this project's own native library, which NativeAOT publishes + # without one. + if(WIN32) + set(_duckdb_asset "libduckdb-windows-amd64.zip") + set(_duckdb_lib_file "duckdb.dll") + set(_duckdb_implib_file "duckdb.lib") + elseif(APPLE) + set(_duckdb_asset "libduckdb-osx-universal.zip") + set(_duckdb_lib_file "libduckdb.dylib") + else() + set(_duckdb_asset "libduckdb-linux-amd64.zip") + set(_duckdb_lib_file "libduckdb.so") + endif() + + set(_duckdb_version "1.5.5") + set(_duckdb_dir "${CMAKE_BINARY_DIR}/duckdb-download") + set(_duckdb_zip "${_duckdb_dir}/${_duckdb_asset}") + if(NOT EXISTS "${_duckdb_dir}/${_duckdb_lib_file}") + file(MAKE_DIRECTORY "${_duckdb_dir}") + set(_duckdb_url "https://github.com/duckdb/duckdb/releases/download/v${_duckdb_version}/${_duckdb_asset}") + message(STATUS "Downloading DuckDB: ${_duckdb_url}") + file(DOWNLOAD "${_duckdb_url}" "${_duckdb_zip}" STATUS _duckdb_status) + list(GET _duckdb_status 0 _duckdb_code) + if(NOT _duckdb_code EQUAL 0) + file(REMOVE "${_duckdb_zip}") + list(GET _duckdb_status 1 _duckdb_message) + message(FATAL_ERROR "Failed to download ${_duckdb_url}: ${_duckdb_message}") + endif() + file(ARCHIVE_EXTRACT INPUT "${_duckdb_zip}" DESTINATION "${_duckdb_dir}") + endif() + + add_library(duckdb SHARED IMPORTED GLOBAL) + set_target_properties(duckdb PROPERTIES + IMPORTED_LOCATION "${_duckdb_dir}/${_duckdb_lib_file}" + INTERFACE_INCLUDE_DIRECTORIES "${_duckdb_dir}") + if(WIN32) + set_target_properties(duckdb PROPERTIES IMPORTED_IMPLIB "${_duckdb_dir}/${_duckdb_implib_file}") + endif() set(EXCELREADER_XLSX_FIXTURE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../tests/ExcelReader.Benchmarks/Data/65K_Records_Data.xlsx" CACHE FILEPATH "Path to the 65K-row xlsx fixture the excelreader-vs-xlnt benchmark reads") + + add_executable(excelreader_cpp_compare_benchmarks benchmark_compare.cpp) + target_link_libraries(excelreader_cpp_compare_benchmarks PRIVATE + xl::excelreader benchmark::benchmark_main xlnt xlsxio_read duckdb) target_compile_definitions(excelreader_cpp_compare_benchmarks PRIVATE EXCELREADER_XLSX_FIXTURE_PATH="${EXCELREADER_XLSX_FIXTURE_PATH}") + # Kept a separate executable from the read comparison rather than a second .cpp in the same + # one: both files define their own FullRow plus an xl::ExcelMapper specialization for it, and + # two targets keep that unambiguous. + # + # The write comparison is then split AGAIN, into two executables built from the same source: + # xlsxio and libxlsxwriter each vendor an incompatible copy of minizip and export the same C + # symbols (zipOpen, zipOpenNewFileInZip, ...), so linking both into one binary lets calls cross + # between the two implementations - which corrupted xlsxio's output in Release. See the header + # comment in benchmark_write_compare.cpp. Do not merge these back into one target. + add_executable(excelreader_cpp_write_compare_benchmarks benchmark_write_compare.cpp) + target_link_libraries(excelreader_cpp_write_compare_benchmarks PRIVATE + xl::excelreader benchmark::benchmark_main xlnt xlsxio_write duckdb) + target_compile_definitions(excelreader_cpp_write_compare_benchmarks PRIVATE + EXCELREADER_XLSX_FIXTURE_PATH="${EXCELREADER_XLSX_FIXTURE_PATH}" + EXCELREADER_BENCH_XLSXIO) + + add_executable(excelreader_cpp_write_compare_lxw_benchmarks benchmark_write_compare.cpp) + target_link_libraries(excelreader_cpp_write_compare_lxw_benchmarks PRIVATE + xl::excelreader benchmark::benchmark_main xlsxwriter duckdb) + target_compile_definitions(excelreader_cpp_write_compare_lxw_benchmarks PRIVATE + EXCELREADER_XLSX_FIXTURE_PATH="${EXCELREADER_XLSX_FIXTURE_PATH}" + EXCELREADER_BENCH_LIBXLSXWRITER) + if(WIN32) - add_custom_command(TARGET excelreader_cpp_compare_benchmarks POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different - "$" - "$") + foreach(_compare_target excelreader_cpp_compare_benchmarks excelreader_cpp_write_compare_benchmarks + excelreader_cpp_write_compare_lxw_benchmarks) + add_custom_command(TARGET ${_compare_target} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "$" + "$" + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "$" + "$") + endforeach() endif() endif() diff --git a/cpp/benchmarks/benchmark_compare.cpp b/cpp/benchmarks/benchmark_compare.cpp index c8fd28e..50107c4 100644 --- a/cpp/benchmarks/benchmark_compare.cpp +++ b/cpp/benchmarks/benchmark_compare.cpp @@ -1,19 +1,22 @@ -// Compares ExcelReader against xlnt (https://github.com/tfussell/xlnt) and xlsxio -// (https://github.com/brechtsanders/xlsxio) reading the full row shape of -// tests/ExcelReader.Benchmarks/Data/65K_Records_Data.xlsx: all 14 columns, 65,535 data rows. -// Neither competitor reads .xlsb, so this fixture is xlsx-only, unlike the RealExcel.xlsb fixture -// the other benchmarks use. +// Compares ExcelReader against xlnt (https://github.com/tfussell/xlnt), xlsxio +// (https://github.com/brechtsanders/xlsxio) and DuckDB's (https://github.com/duckdb/duckdb) +// "excel" extension reading the full row shape of +// tests/ExcelReader.Benchmarks/Data/65K_Records_Data.xlsx: all 14 columns, 65,535 data rows. None +// of the three competitors reads .xlsb, so this fixture is xlsx-only, unlike the RealExcel.xlsb +// fixture the other benchmarks use. // -// All three sides decode every cell into an owned value (std::string for text, matching xlnt's -// cell::to_string() and xlsxio's xlsxioread_sheet_next_cell_string()) and fold it into one -// accumulator - the ExcelReader side uses std::string bindings rather than the zero-copy -// std::string_view used elsewhere in this suite, so no side gets an allocation-free advantage the -// others can't take. Same methodology as BenchmarkAccumulators.cs (the .NET benchmark suite's -// ExcelReader-vs-Sylvan comparison). +// All four sides decode every cell into an owned value and fold it into one accumulator - the +// ExcelReader side uses std::string bindings rather than the zero-copy std::string_view used +// elsewhere in this suite, so no side gets an allocation-free advantage the others can't take. +// DuckDB's case expresses the same accumulator as a single SQL aggregate query rather than a C++ +// loop, which is the idiomatic way to make a SQL engine touch every cell, not a concession to it. +// Same methodology as BenchmarkAccumulators.cs (the .NET benchmark suite's ExcelReader-vs-Sylvan +// comparison). #include #include +#include #include #include @@ -109,7 +112,7 @@ static void BM_ExcelReader_Xlsx_Full(benchmark::State &state) auto buffer_result = read_file_to_buffer(EXCELREADER_XLSX_FIXTURE_PATH); if (!buffer_result.has_value()) { - state.SkipWithError(buffer_result.error().c_str()); + state.SkipWithError(buffer_result.error()); return; } for (auto _ : state) @@ -117,13 +120,13 @@ static void BM_ExcelReader_Xlsx_Full(benchmark::State &state) auto workbook = xl::Workbook::open_memory(buffer_result.value(), XL_FORMAT_XLSX); if (!workbook.has_value()) { - state.SkipWithError(workbook.error().message.c_str()); + state.SkipWithError(workbook.error().message); return; } auto table = xl::parse_sheet(*workbook); if (!table.has_value()) { - state.SkipWithError(table.error().message.c_str()); + state.SkipWithError(table.error().message); return; } int64_t acc = 0; @@ -146,7 +149,7 @@ static void BM_ExcelReader_Xlsx_ParseOnly(benchmark::State &state) auto buffer_result = read_file_to_buffer(EXCELREADER_XLSX_FIXTURE_PATH); if (!buffer_result.has_value()) { - state.SkipWithError(buffer_result.error().c_str()); + state.SkipWithError(buffer_result.error()); return; } for (auto _ : state) @@ -154,13 +157,13 @@ static void BM_ExcelReader_Xlsx_ParseOnly(benchmark::State &state) auto workbook = xl::Workbook::open_memory(buffer_result.value(), XL_FORMAT_XLSX); if (!workbook.has_value()) { - state.SkipWithError(workbook.error().message.c_str()); + state.SkipWithError(workbook.error().message); return; } auto table = xl::parse_sheet(*workbook); if (!table.has_value()) { - state.SkipWithError(table.error().message.c_str()); + state.SkipWithError(table.error().message); return; } benchmark::DoNotOptimize(table); @@ -173,7 +176,7 @@ static void BM_Xlnt_Xlsx_Full(benchmark::State &state) auto buffer_result = read_file_to_buffer(EXCELREADER_XLSX_FIXTURE_PATH); if (!buffer_result.has_value()) { - state.SkipWithError(buffer_result.error().c_str()); + state.SkipWithError(buffer_result.error()); return; } for (auto _ : state) @@ -290,7 +293,7 @@ static void BM_Xlsxio_Xlsx_Full(benchmark::State &state) auto buffer_result = read_file_to_buffer(EXCELREADER_XLSX_FIXTURE_PATH); if (!buffer_result.has_value()) { - state.SkipWithError(buffer_result.error().c_str()); + state.SkipWithError(buffer_result.error()); return; } auto &buffer = buffer_result.value(); @@ -332,3 +335,48 @@ static void BM_Xlsxio_Xlsx_Full(benchmark::State &state) } } BENCHMARK(BM_Xlsxio_Xlsx_Full); + +// DuckDB (https://github.com/duckdb/duckdb) reads via its "excel" extension's read_xlsx() table +// function, invoked here as a single aggregate query rather than pulled apart row by row in C++: +// DuckDB is a SQL engine, and an aggregate over every column is the idiomatic way to make it +// decode every cell, not an artificial concession to it. The expression matches +// accumulate_full_row() exactly - same columns, same "text length, numeric value" split, same +// epoch-days conversion for the two date columns - so the three sides remain the same "touch every +// cell into one accumulator" methodology, just expressed once in SQL instead of once per row. +static void BM_DuckDB_Xlsx_Full(benchmark::State &state) +{ + duckdb::DuckDB db(nullptr); + duckdb::Connection con(db); + + // INSTALL pulls the extension from DuckDB's extension repository on first use and caches it + // locally afterward; done once per process, outside the timed region. + auto setup = con.Query("INSTALL excel; LOAD excel;"); + if (setup->HasError()) + { + state.SkipWithError(setup->GetError()); + return; + } + + const std::string query = + "SELECT sum(length(\"Region\")) + sum(length(\"Country\")) + sum(length(\"Item Type\")) + " + "sum(length(\"Sales Channel\")) + sum(length(\"Order Priority\")) + " + "sum(CAST(\"Order Date\" - DATE '1970-01-01' AS BIGINT)) + sum(\"Order ID\") + " + "sum(CAST(\"Ship Date\" - DATE '1970-01-01' AS BIGINT)) + sum(\"Units Sold\") + " + "sum(CAST(\"Unit Price\" AS BIGINT)) + sum(CAST(\"Unit Cost\" AS BIGINT)) + " + "sum(CAST(\"Total Revenue\" AS BIGINT)) + sum(CAST(\"Total Cost\" AS BIGINT)) + " + "sum(CAST(\"Total Profit\" AS BIGINT)) AS acc FROM read_xlsx('" + + std::string(EXCELREADER_XLSX_FIXTURE_PATH) + "', header = true)"; + + for (auto _ : state) + { + auto result = con.Query(query); + if (result->HasError()) + { + state.SkipWithError(result->GetError()); + return; + } + int64_t acc = result->GetValue(0, 0); + benchmark::DoNotOptimize(acc); + } +} +BENCHMARK(BM_DuckDB_Xlsx_Full); diff --git a/cpp/benchmarks/benchmark_write.cpp b/cpp/benchmarks/benchmark_write.cpp new file mode 100644 index 0000000..d704d40 --- /dev/null +++ b/cpp/benchmarks/benchmark_write.cpp @@ -0,0 +1,184 @@ +// Write benchmarks over tests/ExcelReader.Benchmarks/Data/65K_Records_Data.xlsb (65,535 data rows), +// the same fixture the Rust, Python and .NET suites use. +// +// Both cases write the SAME seven columns, so the only difference between them is where the data +// starts out - which is the one thing this file is measuring: +// +// * write_columns is handed buffers that are already columnar - nothing is transposed and +// nothing is copied. It is the ceiling. +// * write_sheet starts from a std::vector and pays the row-to-column transpose. It is what +// a row-shaped caller actually experiences, and the number to compare against any cell-at-a- +// time writer. +// +// The gap between them is therefore the cost of holding row-shaped data, not a difference in how +// much gets written. For the comparison against other libraries, see benchmark_write_compare.cpp. +// +// State the CPU, OS and compiler version alongside any number published from this file. + +#include + +#include + +#include +#include +#include +#include + +struct Row +{ + std::string region; + std::string country; + std::string item_type; + std::chrono::sys_days order_date; + int64_t order_id; + int64_t units_sold; + double total_revenue; +}; + +template <> +struct xl::ExcelMapper +{ + static constexpr auto get_bindings() + { + return std::make_tuple( + xl::make_field("Region", &Row::region), + xl::make_field("Country", &Row::country), + xl::make_field("Item Type", &Row::item_type), + xl::make_field("Order Date", &Row::order_date), + xl::make_field("Order ID", &Row::order_id), + xl::make_field("Units Sold", &Row::units_sold), + xl::make_field("Total Revenue", &Row::total_revenue)); + } +}; + +// Reads the fixture once into row structs. Aborts rather than silently benchmarking an empty +// input - a suite that measures nothing is worse than no suite. +static const std::vector &fixture_rows() +{ + static const std::vector rows = [] + { + auto workbook = xl::Workbook::open(EXCELREADER_LARGE_FIXTURE_PATH); + if (!workbook.has_value()) + { + std::fprintf(stderr, "missing or unreadable fixture %s\n", EXCELREADER_LARGE_FIXTURE_PATH); + std::abort(); + } + auto table = xl::parse_sheet(*workbook); + if (!table.has_value() || table->size() == 0) + { + std::fprintf(stderr, "fixture %s parsed to zero rows\n", EXCELREADER_LARGE_FIXTURE_PATH); + std::abort(); + } + return table->to_vector(); + }(); + return rows; +} + +static std::filesystem::path bench_path(std::string_view name) +{ + return std::filesystem::temp_directory_path() / + std::filesystem::path(std::string("excelreader-bench-") + std::string(name)); +} + +namespace +{ + // The offsets/blob pair an XL_T_STRING column needs. xl::write_sheet builds one of these + // internally; the columnar benchmark below builds its own so both cases start from the same + // shape. + struct StringBuffer + { + std::vector offsets{0}; + std::vector data{}; + + void reserve(size_t count) + { + offsets.reserve(count + 1); + } + + void push(std::string_view value) + { + const uint8_t *bytes = reinterpret_cast(value.data()); + data.insert(data.end(), bytes, bytes + value.size()); + offsets.push_back(static_cast(data.size())); + } + }; +} + +static void BM_WriteSheet(benchmark::State &state) +{ + const std::vector &rows = fixture_rows(); + const std::filesystem::path path = bench_path("sheet.xlsx"); + for (auto _ : state) + { + auto result = xl::write_sheet(path.string(), XL_FORMAT_XLSX, rows); + benchmark::DoNotOptimize(result); + if (!result.has_value()) + { + state.SkipWithError("write_sheet failed"); + break; + } + } + state.SetItemsProcessed(static_cast(state.iterations()) * static_cast(rows.size())); + std::filesystem::remove(path); +} +BENCHMARK(BM_WriteSheet); + +static void BM_WriteColumns(benchmark::State &state) +{ + const std::vector &rows = fixture_rows(); + + // Transposed once, outside the measured region: this case exists to measure the write, not the + // transpose BM_WriteSheet already covers. + // + // ALL SEVEN columns, the same set BM_WriteSheet writes. An earlier version of this benchmark + // wrote only four, which made it look ~2x faster when a third of that gap was simply three + // fewer columns of work. + StringBuffer region; + StringBuffer country; + StringBuffer item_type; + std::vector order_dates; + std::vector order_ids; + std::vector units; + std::vector revenue; + region.reserve(rows.size()); + country.reserve(rows.size()); + item_type.reserve(rows.size()); + order_dates.reserve(rows.size()); + order_ids.reserve(rows.size()); + units.reserve(rows.size()); + revenue.reserve(rows.size()); + for (const Row &row : rows) + { + region.push(row.region); + country.push(row.country); + item_type.push(row.item_type); + order_dates.push_back(static_cast(row.order_date.time_since_epoch().count())); + order_ids.push_back(row.order_id); + units.push_back(row.units_sold); + revenue.push_back(row.total_revenue); + } + + const std::array columns{ + xl::string_column("Region", region.offsets, region.data), + xl::string_column("Country", country.offsets, country.data), + xl::string_column("Item Type", item_type.offsets, item_type.data), + xl::date_column("Order Date", order_dates), + xl::i64_column("Order ID", order_ids), + xl::i64_column("Units Sold", units), + xl::f64_column("Total Revenue", revenue)}; + + const std::filesystem::path path = bench_path("columns.xlsx"); + for (auto _ : state) + { + auto result = xl::write_columns(path.string(), XL_FORMAT_XLSX, columns); + benchmark::DoNotOptimize(result); + if (!result.has_value()) + { + state.SkipWithError("write_columns failed"); + break; + } + } + state.SetItemsProcessed(static_cast(state.iterations()) * static_cast(rows.size())); + std::filesystem::remove(path); +} +BENCHMARK(BM_WriteColumns); \ No newline at end of file diff --git a/cpp/benchmarks/benchmark_write_compare.cpp b/cpp/benchmarks/benchmark_write_compare.cpp new file mode 100644 index 0000000..a374f64 --- /dev/null +++ b/cpp/benchmarks/benchmark_write_compare.cpp @@ -0,0 +1,523 @@ +// Compares ExcelReader against xlnt (https://github.com/tfussell/xlnt), xlsxio +// (https://github.com/brechtsanders/xlsxio), libxlsxwriter +// (https://github.com/jmcnamara/libxlsxwriter) and DuckDB's (https://github.com/duckdb/duckdb) +// "excel" extension WRITING the full row shape of +// tests/ExcelReader.Benchmarks/Data/65K_Records_Data.xlsx: all 14 columns, 65,535 data rows plus a +// header row. The rows are read once at startup with ExcelReader and then written back out by each +// library in turn, so all five start from exactly the same in-memory data. +// +// WORK IS NOT MATCHED across all six cases, and the mismatch runs in both directions. Read the +// table with the caveats, not without them: +// +// * BM_ExcelReader_WriteColumns is handed buffers that are already columnar. Nothing is +// transposed. No cell-at-a-time API can reach this shape at all, so it is a ceiling, not a +// competitor's number. Compare it only against BM_ExcelReader_WriteSheet. +// * BM_ExcelReader_WriteSheet starts from a std::vector - the same shape every +// competitor below is handed - and pays the row-to-column transpose itself. THIS is the +// matched-work number, and the only one of ours that belongs next to the competitors. +// * ExcelReader attaches a number format to the two XL_T_DATE columns (so Excel shows a date +// rather than a serial), which every case below does NOT do: every competitor writes those +// columns as bare numbers, the cheaper option. That difference favours the competitors. +// * xlnt builds a full in-memory document model (styles, formats, formulas) before serializing. +// It is doing more than this library exposes, and its number reflects that. +// * xlsxio streams cells straight to the ZIP, the closest thing here to matched work on the +// competitor side - the same relationship it has to ExcelReader on the reading benchmark. +// * libxlsxwriter is also a straight streaming writer with no document-model overhead, same +// class of competitor as xlsxio - it is the one most worth comparing BM_ExcelReader_WriteSheet +// against, being C rather than C++ and, like ExcelReader's own core, built for throughput +// rather than a full object model. +// * DuckDB's rows are loaded into an in-memory table via its Appender API BEFORE the timed +// region, so BM_DuckDB_Write measures the COPY TO xlsx step alone - same treatment +// BM_ExcelReader_WriteColumns gets for its transpose. DuckDB is a full analytical query engine +// doing far more than any Excel-writing library here, and this measures one narrow slice of it. +// +// State the CPU, OS and compiler version alongside any number published from this file. +// +// WHY THIS FILE IS COMPILED TWICE (see cpp/benchmarks/CMakeLists.txt): xlsxio and libxlsxwriter +// each bring their own incompatible copy of minizip, and both export the SAME C symbols +// (zipOpen, zipOpenNewFileInZip, zipWriteInFileInZip, ...) from a static library: +// +// * xlsxio is built against minizip-ng's compat layer, whose zipOpenNewFileInZip takes +// uint16_t extrafield sizes. +// * libxlsxwriter vendors classic minizip (third_party/minizip/zip.c), whose signature takes +// 32-bit uInt sizes and whose body starts with `if (size_extrafield_local > 0xffff) return +// ZIP_PARAMERROR;` - a check that cannot exist in the minizip-ng version. +// +// Linked into one executable, the linker keeps exactly one definition of each name, so calls can +// cross between the two: a zipFile opened as minizip-ng's `mz_zip_compat*` gets read as classic +// minizip's `zip64_internal*`. That is how a Release build produced "Error creating file +// xl/workbook.xml inside zip file" on xlsxio's background thread - garbage read out of the wrong +// struct tripping the 0xffff check - while a Debug build, with different link ordering, did not. +// Neither library's numbers are trustworthy in that state. +// +// So each of the two gets its own executable, and this file's competitor cases are guarded to +// match. Do not merge the targets back together. + +#include + +#include +#include + +// xlsxio and libxlsxwriter MUST NOT be linked into the same executable - see the note above. This +// file is compiled twice, once with each define, by cpp/benchmarks/CMakeLists.txt. +#ifdef EXCELREADER_BENCH_XLSXIO +#include +#include +#endif +#ifdef EXCELREADER_BENCH_LIBXLSXWRITER +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace +{ + // Same 14 columns as benchmark_compare.cpp's read-side FullRow, with owned std::string text so + // the rows survive the TableView they were parsed from. + struct FullRow + { + std::string Region; + std::string Country; + std::string ItemType; + std::string SalesChannel; + std::string OrderPriority; + std::chrono::sys_days OrderDate; + int64_t OrderId; + std::chrono::sys_days ShipDate; + int64_t UnitsSold; + double UnitPrice; + double UnitCost; + double TotalRevenue; + double TotalCost; + double TotalProfit; + }; +} + +template <> +struct xl::ExcelMapper +{ + static constexpr auto get_bindings() + { + return std::make_tuple( + xl::make_field("Region", &FullRow::Region), + xl::make_field("Country", &FullRow::Country), + xl::make_field("Item Type", &FullRow::ItemType), + xl::make_field("Sales Channel", &FullRow::SalesChannel), + xl::make_field("Order Priority", &FullRow::OrderPriority), + xl::make_field("Order Date", &FullRow::OrderDate), + xl::make_field("Order ID", &FullRow::OrderId), + xl::make_field("Ship Date", &FullRow::ShipDate), + xl::make_field("Units Sold", &FullRow::UnitsSold), + xl::make_field("Unit Price", &FullRow::UnitPrice), + xl::make_field("Unit Cost", &FullRow::UnitCost), + xl::make_field("Total Revenue", &FullRow::TotalRevenue), + xl::make_field("Total Cost", &FullRow::TotalCost), + xl::make_field("Total Profit", &FullRow::TotalProfit)); + } +}; + +namespace +{ + constexpr std::array kHeaders{ + "Region", "Country", "Item Type", "Sales Channel", "Order Priority", "Order Date", + "Order ID", "Ship Date", "Units Sold", "Unit Price", "Unit Cost", "Total Revenue", + "Total Cost", "Total Profit"}; + + // Reads the fixture once into row structs. Aborts rather than silently benchmarking an empty + // input - a suite that measures nothing is worse than no suite. + const std::vector &fixture_rows() + { + static const std::vector rows = [] + { + auto workbook = xl::Workbook::open(EXCELREADER_XLSX_FIXTURE_PATH, XL_FORMAT_XLSX); + if (!workbook.has_value()) + { + std::fprintf(stderr, "missing or unreadable fixture %s\n", EXCELREADER_XLSX_FIXTURE_PATH); + std::abort(); + } + auto table = xl::parse_sheet(*workbook); + if (!table.has_value() || table->size() == 0) + { + std::fprintf(stderr, "fixture %s parsed to zero rows\n", EXCELREADER_XLSX_FIXTURE_PATH); + std::abort(); + } + return table->to_vector(); + }(); + return rows; + } + + // Suffixed with a steady_clock reading so concurrent or back-to-back runs of this executable + // cannot collide on one temp file. steady_clock rather than a process id needs no platform + // header (no , no ) - this file has neither today. + std::filesystem::path bench_path(std::string_view name) + { + const auto ticks = std::chrono::steady_clock::now().time_since_epoch().count(); + return std::filesystem::temp_directory_path() / + std::filesystem::path("excelreader-write-compare-" + std::to_string(ticks) + "-" + + std::string(name)); + } + + int32_t days(std::chrono::sys_days value) + { + return static_cast(value.time_since_epoch().count()); + } + + // The offsets/blob pair an XL_T_STRING column needs. + struct StringBuffer + { + std::vector offsets{0}; + std::vector data{}; + + void reserve(size_t count) + { + offsets.reserve(count + 1); + } + + void push(std::string_view value) + { + const uint8_t *bytes = reinterpret_cast(value.data()); + data.insert(data.end(), bytes, bytes + value.size()); + offsets.push_back(static_cast(data.size())); + } + }; +} + +static void BM_ExcelReader_WriteSheet(benchmark::State &state) +{ + const std::vector &rows = fixture_rows(); + const std::filesystem::path path = bench_path("sheet.xlsx"); + for (auto _ : state) + { + auto result = xl::write_sheet(path.string(), XL_FORMAT_XLSX, rows); + if (!result.has_value()) + { + state.SkipWithError(result.error().message.c_str()); + return; + } + benchmark::DoNotOptimize(result); + } + state.SetItemsProcessed(static_cast(state.iterations()) * static_cast(rows.size())); + std::filesystem::remove(path); +} +BENCHMARK(BM_ExcelReader_WriteSheet); + +static void BM_ExcelReader_WriteColumns(benchmark::State &state) +{ + const std::vector &rows = fixture_rows(); + + // Transposed once, outside the measured region: this case exists to measure the write, not the + // transpose BM_ExcelReader_WriteSheet already covers. + StringBuffer region; + StringBuffer country; + StringBuffer item_type; + StringBuffer sales_channel; + StringBuffer order_priority; + std::vector order_date; + std::vector order_id; + std::vector ship_date; + std::vector units_sold; + std::vector unit_price; + std::vector unit_cost; + std::vector total_revenue; + std::vector total_cost; + std::vector total_profit; + + const size_t count = rows.size(); + for (StringBuffer *buffer : {®ion, &country, &item_type, &sales_channel, &order_priority}) + { + buffer->reserve(count); + } + order_date.reserve(count); + order_id.reserve(count); + ship_date.reserve(count); + units_sold.reserve(count); + unit_price.reserve(count); + unit_cost.reserve(count); + total_revenue.reserve(count); + total_cost.reserve(count); + total_profit.reserve(count); + + for (const FullRow &row : rows) + { + region.push(row.Region); + country.push(row.Country); + item_type.push(row.ItemType); + sales_channel.push(row.SalesChannel); + order_priority.push(row.OrderPriority); + order_date.push_back(days(row.OrderDate)); + order_id.push_back(row.OrderId); + ship_date.push_back(days(row.ShipDate)); + units_sold.push_back(row.UnitsSold); + unit_price.push_back(row.UnitPrice); + unit_cost.push_back(row.UnitCost); + total_revenue.push_back(row.TotalRevenue); + total_cost.push_back(row.TotalCost); + total_profit.push_back(row.TotalProfit); + } + + const std::array columns{ + xl::string_column(kHeaders[0], region.offsets, region.data), + xl::string_column(kHeaders[1], country.offsets, country.data), + xl::string_column(kHeaders[2], item_type.offsets, item_type.data), + xl::string_column(kHeaders[3], sales_channel.offsets, sales_channel.data), + xl::string_column(kHeaders[4], order_priority.offsets, order_priority.data), + xl::date_column(kHeaders[5], order_date), + xl::i64_column(kHeaders[6], order_id), + xl::date_column(kHeaders[7], ship_date), + xl::i64_column(kHeaders[8], units_sold), + xl::f64_column(kHeaders[9], unit_price), + xl::f64_column(kHeaders[10], unit_cost), + xl::f64_column(kHeaders[11], total_revenue), + xl::f64_column(kHeaders[12], total_cost), + xl::f64_column(kHeaders[13], total_profit)}; + + const std::filesystem::path path = bench_path("columns.xlsx"); + for (auto _ : state) + { + auto result = xl::write_columns(path.string(), XL_FORMAT_XLSX, columns); + if (!result.has_value()) + { + state.SkipWithError(result.error().message.c_str()); + return; + } + benchmark::DoNotOptimize(result); + } + state.SetItemsProcessed(static_cast(state.iterations()) * static_cast(rows.size())); + std::filesystem::remove(path); +} +BENCHMARK(BM_ExcelReader_WriteColumns); + +#ifdef EXCELREADER_BENCH_XLSXIO +static void BM_Xlnt_Write(benchmark::State &state) +{ + const std::vector &rows = fixture_rows(); + const std::filesystem::path path = bench_path("xlnt.xlsx"); + for (auto _ : state) + { + xlnt::workbook workbook; + xlnt::worksheet sheet = workbook.active_sheet(); + + // xlnt's cell references are 1-based in both axes, so the header occupies row 1 and data + // starts at row 2 - the same layout ExcelReader's writer produces. + for (uint32_t column = 0; column < kHeaders.size(); ++column) + { + sheet.cell(column + 1, 1).value(kHeaders[column]); + } + + uint32_t row_index = 2; + for (const FullRow &row : rows) + { + sheet.cell(1, row_index).value(row.Region); + sheet.cell(2, row_index).value(row.Country); + sheet.cell(3, row_index).value(row.ItemType); + sheet.cell(4, row_index).value(row.SalesChannel); + sheet.cell(5, row_index).value(row.OrderPriority); + // Written as a bare serial number, not a styled date: attaching a number format here + // would be extra work ExcelReader does and this case deliberately skips. + sheet.cell(6, row_index).value(static_cast(days(row.OrderDate))); + sheet.cell(7, row_index).value(static_cast(row.OrderId)); + sheet.cell(8, row_index).value(static_cast(days(row.ShipDate))); + sheet.cell(9, row_index).value(static_cast(row.UnitsSold)); + sheet.cell(10, row_index).value(row.UnitPrice); + sheet.cell(11, row_index).value(row.UnitCost); + sheet.cell(12, row_index).value(row.TotalRevenue); + sheet.cell(13, row_index).value(row.TotalCost); + sheet.cell(14, row_index).value(row.TotalProfit); + ++row_index; + } + + workbook.save(path.string()); + benchmark::DoNotOptimize(sheet); + } + state.SetItemsProcessed(static_cast(state.iterations()) * static_cast(rows.size())); + std::filesystem::remove(path); +} +BENCHMARK(BM_Xlnt_Write); + +static void BM_Xlsxio_Write(benchmark::State &state) +{ + const std::vector &rows = fixture_rows(); + const std::filesystem::path path = bench_path("xlsxio.xlsx"); + const std::string target = path.string(); + for (auto _ : state) + { + xlsxiowriter handle = xlsxiowrite_open(target.c_str(), "Sheet1"); + if (!handle) + { + state.SkipWithError("xlsxiowrite_open failed"); + return; + } + // xlsxio infers each column's type from the first N rows unless told otherwise. Every cell + // below is written through an explicitly typed accessor, so that detection pass is pure + // overhead here - turning it off keeps this case measuring the write, not the sniffing. + xlsxiowrite_set_detection_rows(handle, 0); + + for (const char *header : kHeaders) + { + xlsxiowrite_add_column(handle, header, 0); + } + + for (const FullRow &row : rows) + { + xlsxiowrite_add_cell_string(handle, row.Region.c_str()); + xlsxiowrite_add_cell_string(handle, row.Country.c_str()); + xlsxiowrite_add_cell_string(handle, row.ItemType.c_str()); + xlsxiowrite_add_cell_string(handle, row.SalesChannel.c_str()); + xlsxiowrite_add_cell_string(handle, row.OrderPriority.c_str()); + // Bare serial numbers, same as the xlnt case above: xlsxiowrite_add_cell_datetime() + // would format them, which neither competitor case is asked to do here. + xlsxiowrite_add_cell_int(handle, days(row.OrderDate)); + xlsxiowrite_add_cell_int(handle, row.OrderId); + xlsxiowrite_add_cell_int(handle, days(row.ShipDate)); + xlsxiowrite_add_cell_int(handle, row.UnitsSold); + xlsxiowrite_add_cell_float(handle, row.UnitPrice); + xlsxiowrite_add_cell_float(handle, row.UnitCost); + xlsxiowrite_add_cell_float(handle, row.TotalRevenue); + xlsxiowrite_add_cell_float(handle, row.TotalCost); + xlsxiowrite_add_cell_float(handle, row.TotalProfit); + xlsxiowrite_next_row(handle); + } + + if (xlsxiowrite_close(handle) != 0) + { + state.SkipWithError("xlsxiowrite_close failed"); + return; + } + } + state.SetItemsProcessed(static_cast(state.iterations()) * static_cast(rows.size())); + std::filesystem::remove(path); +} +BENCHMARK(BM_Xlsxio_Write); +#endif // EXCELREADER_BENCH_XLSXIO + +#ifdef EXCELREADER_BENCH_LIBXLSXWRITER +static void BM_Libxlsxwriter_Write(benchmark::State &state) +{ + const std::vector &rows = fixture_rows(); + const std::filesystem::path path = bench_path("libxlsxwriter.xlsx"); + const std::string target = path.string(); + for (auto _ : state) + { + lxw_workbook *workbook = workbook_new(target.c_str()); + if (!workbook) + { + state.SkipWithError("workbook_new failed"); + return; + } + lxw_worksheet *sheet = workbook_add_worksheet(workbook, nullptr); + + // Row/column indices are 0-based here, unlike xlnt's cell() above. + for (lxw_col_t column = 0; column < static_cast(kHeaders.size()); ++column) + { + worksheet_write_string(sheet, 0, column, kHeaders[column], nullptr); + } + + lxw_row_t row_index = 1; + for (const FullRow &row : rows) + { + worksheet_write_string(sheet, row_index, 0, row.Region.c_str(), nullptr); + worksheet_write_string(sheet, row_index, 1, row.Country.c_str(), nullptr); + worksheet_write_string(sheet, row_index, 2, row.ItemType.c_str(), nullptr); + worksheet_write_string(sheet, row_index, 3, row.SalesChannel.c_str(), nullptr); + worksheet_write_string(sheet, row_index, 4, row.OrderPriority.c_str(), nullptr); + // Bare serial numbers, same as the xlnt and xlsxio cases above: a formatted date write + // would be extra work neither of those pays either. + worksheet_write_number(sheet, row_index, 5, static_cast(days(row.OrderDate)), nullptr); + worksheet_write_number(sheet, row_index, 6, static_cast(row.OrderId), nullptr); + worksheet_write_number(sheet, row_index, 7, static_cast(days(row.ShipDate)), nullptr); + worksheet_write_number(sheet, row_index, 8, static_cast(row.UnitsSold), nullptr); + worksheet_write_number(sheet, row_index, 9, row.UnitPrice, nullptr); + worksheet_write_number(sheet, row_index, 10, row.UnitCost, nullptr); + worksheet_write_number(sheet, row_index, 11, row.TotalRevenue, nullptr); + worksheet_write_number(sheet, row_index, 12, row.TotalCost, nullptr); + worksheet_write_number(sheet, row_index, 13, row.TotalProfit, nullptr); + ++row_index; + } + + if (workbook_close(workbook) != LXW_NO_ERROR) + { + state.SkipWithError("workbook_close failed"); + return; + } + } + state.SetItemsProcessed(static_cast(state.iterations()) * static_cast(rows.size())); + std::filesystem::remove(path); +} +BENCHMARK(BM_Libxlsxwriter_Write); +#endif // EXCELREADER_BENCH_LIBXLSXWRITER + +// DuckDB (https://github.com/duckdb/duckdb) writes via its "excel" extension's +// `COPY ... TO ... WITH (FORMAT xlsx)`. The rows are loaded into an in-memory DuckDB table via its +// Appender API (DuckDB's own fast bulk-load path, not a parsed INSERT statement) BEFORE the timed +// region starts - matching how BM_ExcelReader_WriteColumns transposes outside the loop - so what's +// measured is the COPY itself, not building the table. +static void BM_DuckDB_Write(benchmark::State &state) +{ + const std::vector &rows = fixture_rows(); + const std::filesystem::path path = bench_path("duckdb.xlsx"); + const std::string target = path.string(); + + duckdb::DuckDB db(nullptr); + duckdb::Connection con(db); + + auto setup = con.Query("INSTALL excel; LOAD excel;"); + if (setup->HasError()) + { + state.SkipWithError(setup->GetError().c_str()); + return; + } + + auto create = con.Query( + "CREATE TABLE fixture (" + "\"Region\" VARCHAR, \"Country\" VARCHAR, \"Item Type\" VARCHAR, " + "\"Sales Channel\" VARCHAR, \"Order Priority\" VARCHAR, \"Order Date\" DATE, " + "\"Order ID\" BIGINT, \"Ship Date\" DATE, \"Units Sold\" BIGINT, " + "\"Unit Price\" DOUBLE, \"Unit Cost\" DOUBLE, \"Total Revenue\" DOUBLE, " + "\"Total Cost\" DOUBLE, \"Total Profit\" DOUBLE)"); + if (create->HasError()) + { + state.SkipWithError(create->GetError().c_str()); + return; + } + + { + duckdb::Appender appender(con, "fixture"); + for (const FullRow &row : rows) + { + // .c_str() rather than the std::string itself: AppendRow deduces one Append + // specialization per argument's exact type, and DuckDB only provides one for + // `const char *` (matching duckdb's own test suite), not for std::string. + appender.AppendRow( + row.Region.c_str(), row.Country.c_str(), row.ItemType.c_str(), + row.SalesChannel.c_str(), row.OrderPriority.c_str(), + duckdb::date_t(days(row.OrderDate)), row.OrderId, duckdb::date_t(days(row.ShipDate)), + row.UnitsSold, row.UnitPrice, row.UnitCost, row.TotalRevenue, row.TotalCost, + row.TotalProfit); + } + appender.Close(); + } + + const std::string copy_query = + "COPY fixture TO '" + target + "' WITH (FORMAT xlsx, HEADER true)"; + for (auto _ : state) + { + auto result = con.Query(copy_query); + if (result->HasError()) + { + state.SkipWithError(result->GetError().c_str()); + return; + } + benchmark::DoNotOptimize(result); + } + state.SetItemsProcessed(static_cast(state.iterations()) * static_cast(rows.size())); + std::filesystem::remove(path); +} +BENCHMARK(BM_DuckDB_Write); diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 17af2a9..a8eefdf 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -15,3 +15,31 @@ if(WIN32) "$" "$") endif() + +add_executable(excelreader_cpp_write write.cpp) +target_link_libraries(excelreader_cpp_write PRIVATE xl::excelreader) + +add_test(NAME excelreader_cpp_write COMMAND excelreader_cpp_write) +if(WIN32) + # Same reason as the smoke test: the native DLL has no import-time PATH entry, so copy it next + # to the test executable for the loader to find. + add_custom_command(TARGET excelreader_cpp_write POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "$" + "$") +endif() + +add_executable(excelreader_cpp_arrow arrow.cpp) +target_link_libraries(excelreader_cpp_arrow PRIVATE xl::excelreader) +target_compile_definitions(excelreader_cpp_arrow PRIVATE + EXCELREADER_FIXTURE_PATH="${EXCELREADER_FIXTURE_PATH}") + +add_test(NAME excelreader_cpp_arrow COMMAND excelreader_cpp_arrow) +if(WIN32) + # Same reason as the smoke test: the native DLL has no import-time PATH entry, so copy it next + # to the test executable for the loader to find. + add_custom_command(TARGET excelreader_cpp_arrow POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "$" + "$") +endif() \ No newline at end of file diff --git a/cpp/tests/arrow.cpp b/cpp/tests/arrow.cpp new file mode 100644 index 0000000..40f667e --- /dev/null +++ b/cpp/tests/arrow.cpp @@ -0,0 +1,70 @@ +#include + +#include +#include +#include +#include + +struct Record +{ + std::string_view Coluna1; + int64_t Coluna3; +}; + +template <> +struct xl::ExcelMapper +{ + static constexpr auto get_bindings() + { + return std::make_tuple( + xl::make_field("Coluna1", &Record::Coluna1), + xl::make_field("Coluna3", &Record::Coluna3)); + } +}; + +#define CHECK(cond, msg) \ + do \ + { \ + if (!(cond)) \ + { \ + std::fprintf(stderr, "FAIL: %s (%s:%d)\n", msg, __FILE__, __LINE__); \ + return 1; \ + } \ + } while (0) + +int main() +{ + xl::OpenOptions options{.prefetch_decompression = 1}; + auto workbook = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, XL_FORMAT_XLSB, &options); + CHECK(workbook.has_value(), "xl::Workbook::open must succeed on the RealExcel.xlsb fixture"); + + auto table = xl::parse_arrow(*workbook); + CHECK(table.has_value(), "xl::parse_arrow must succeed"); + + // The export hands back ONE top-level struct array whose children are the columns. + CHECK(std::strcmp(table->schema.format, "+s") == 0, "top level must be a struct array"); + CHECK(table->schema.n_children == 2, "must have two child columns"); + CHECK(table->array.n_children == 2, "must have two child arrays"); + CHECK(std::strcmp(table->schema.children[0]->name, "Coluna1") == 0, "first column must be named Coluna1"); + CHECK(std::strcmp(table->schema.children[0]->format, "u") == 0, "Coluna1 must be utf8"); + CHECK(std::strcmp(table->schema.children[1]->format, "l") == 0, "Coluna3 must be int64"); + CHECK(table->array.length == 100, "RealExcel.xlsb has 100 data rows"); + + // Destructor must release both; running under a leak checker in CI is what proves it, but a + // move-then-destroy here at least exercises the moved-from path being inert. + { + xl::ArrowTable moved = std::move(*table); + CHECK(moved.array.release != nullptr, "moved-to table must still own a release callback"); + CHECK(table->array.release == nullptr, "moved-from table must be released/inert"); + } + + // An out-of-range header_row must fail cleanly, leaving no half-built ArrowTable behind - this + // matters here more than on the happy path because ~ArrowTable calls through the release + // function pointers it holds, so a half-initialized table on the failure path would mean the + // destructor walks into garbage. + auto failed = xl::parse_arrow(*workbook, 1'000'000); + CHECK(!failed.has_value(), "xl::parse_arrow must fail for an out-of-range header_row"); + + std::printf("OK: C++ arrow test passed\n"); + return 0; +} diff --git a/cpp/tests/write.cpp b/cpp/tests/write.cpp new file mode 100644 index 0000000..5fb143c --- /dev/null +++ b/cpp/tests/write.cpp @@ -0,0 +1,712 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#define CHECK(cond, msg) \ + do \ + { \ + if (!(cond)) \ + { \ + std::fprintf(stderr, "FAIL: %s (%s:%d)\n", msg, __FILE__, __LINE__); \ + return 1; \ + } \ + } while (0) + +struct WrittenRow +{ + std::string_view texto; + int64_t inteiro; + double numero; + std::chrono::year_month_day data; + std::chrono::microseconds hora; + std::chrono::system_clock::time_point instante; +}; + +template <> +struct xl::ExcelMapper +{ + static constexpr auto get_bindings() + { + return std::make_tuple( + xl::make_field("texto", &WrittenRow::texto), + xl::make_field("inteiro", &WrittenRow::inteiro), + xl::make_field("numero", &WrittenRow::numero), + xl::make_field("data", &WrittenRow::data), + xl::make_field("hora", &WrittenRow::hora), + xl::make_field("instante", &WrittenRow::instante)); + } +}; + +// A path under the system temp directory. Deleted by each test that creates it. +static std::filesystem::path temp_path(std::string_view name) +{ + return std::filesystem::temp_directory_path() / + std::filesystem::path(std::string("excelreader-cpp-") + std::string(name)); +} + +static int test_write_columns_round_trip() +{ + const std::vector offsets{0, 3, 6}; + const std::vector blob{'u', 'm', 'a', 'd', 'o', 'i'}; + const std::vector inteiros{1, 2}; + const std::vector numeros{0.5, 1.5}; + const std::vector datas{20454, 20455}; // 2026-01-01, 2026-01-02 + const std::vector horas{3600000000, 7200000000}; + const std::vector instantes{1767225600000000, 1767312000000000}; + + const std::array columns{ + xl::string_column("texto", offsets, blob), + xl::i64_column("inteiro", inteiros), + xl::f64_column("numero", numeros), + xl::date_column("data", datas), + xl::time_column("hora", horas), + xl::timestamp_column("instante", instantes)}; + + const std::filesystem::path path = temp_path("columns.xlsx"); + auto written = xl::write_columns(path.string(), XL_FORMAT_XLSX, columns); + CHECK(written.has_value(), "write_columns must succeed"); + { + auto workbook = xl::Workbook::open(path.string()); + CHECK(workbook.has_value(), "the written file must open"); + auto table = xl::parse_sheet(*workbook); + CHECK(table.has_value(), "the written file must parse back"); + CHECK(table->size() == 2, "two rows were written"); + + WrittenRow first = *table->begin(); + CHECK(first.texto == "uma", "row 0's string must round-trip"); + CHECK(first.inteiro == 1, "row 0's int64 must round-trip"); + CHECK(first.numero == 0.5, "row 0's double must round-trip"); + CHECK(first.data == std::chrono::year{2026} / std::chrono::month{1} / std::chrono::day{1}, + "row 0's date must round-trip"); + CHECK(first.hora == std::chrono::microseconds{3600000000}, "row 0's time must round-trip"); + auto second_row = table->at(1); + CHECK(second_row.has_value(), "row 1 must be in bounds"); + CHECK(second_row->texto == "doi", "row 1's string must round-trip"); + CHECK(second_row->inteiro == 2, "row 1's int64 must round-trip"); + } + std::filesystem::remove(path); + return 0; +} + +static int test_write_columns_to_memory_round_trip() +{ + const std::vector offsets{0, 3, 6}; + const std::vector blob{'u', 'm', 'a', 'd', 'o', 'i'}; + const std::vector inteiros{1, 2}; + const std::vector numeros{0.5, 1.5}; + const std::vector datas{20454, 20455}; // 2026-01-01, 2026-01-02 + const std::vector horas{3600000000, 7200000000}; + const std::vector instantes{1767225600000000, 1767312000000000}; + + const std::array columns{ + xl::string_column("texto", offsets, blob), + xl::i64_column("inteiro", inteiros), + xl::f64_column("numero", numeros), + xl::date_column("data", datas), + xl::time_column("hora", horas), + xl::timestamp_column("instante", instantes)}; + + auto bytes = xl::write_columns_to_memory(XL_FORMAT_XLSX, columns); + CHECK(bytes.has_value(), "write_columns_to_memory must succeed"); + CHECK(!bytes->empty(), "write_columns_to_memory must return non-empty bytes"); + + auto workbook = xl::Workbook::open_memory(*bytes, XL_FORMAT_XLSX); + CHECK(workbook.has_value(), "the returned bytes must open"); + auto table = xl::parse_sheet(*workbook); + CHECK(table.has_value(), "the returned bytes must parse back"); + CHECK(table->size() == 2, "two rows were written"); + + WrittenRow first = *table->begin(); + CHECK(first.texto == "uma", "row 0's string must round-trip"); + CHECK(first.inteiro == 1, "row 0's int64 must round-trip"); + return 0; +} + +static int test_write_columns_rejects_bad_input() +{ + const std::vector two{1, 2}; + const std::vector three{1, 2, 3}; + const std::vector empty_bitmap{}; + const std::filesystem::path path = temp_path("rejected.xlsx"); + + std::array mismatched{xl::i64_column("a", two), xl::i64_column("b", three)}; + CHECK(!xl::write_columns(path.string(), XL_FORMAT_XLSX, mismatched).has_value(), + "columns of different lengths must be rejected"); + + std::array partial_header{xl::i64_column("a", two), xl::i64_column("", two)}; + CHECK(!xl::write_columns(path.string(), XL_FORMAT_XLSX, partial_header).has_value(), + "a partial header row must be rejected"); + + // Two rows need one byte of bitmap. Hand it a non-null pointer with zero length: the ABI takes + // the bitmap without a length, so this is exactly the overrun the wrapper exists to refuse. + std::array short_bitmap{xl::i64_column("a", two)}; + short_bitmap[0].validity = reinterpret_cast(two.data()); + short_bitmap[0].validity_len = 0; + CHECK(!xl::write_columns(path.string(), XL_FORMAT_XLSX, short_bitmap).has_value(), + "a validity bitmap shorter than the row count must be rejected"); + + std::array fine{xl::i64_column("a", two)}; + CHECK(!xl::write_columns(path.string(), XL_FORMAT_AUTO, fine).has_value(), + "XL_FORMAT_AUTO must be rejected: a new file has no signature bytes to sniff"); + + std::span none{}; + CHECK(!xl::write_columns(path.string(), XL_FORMAT_XLSX, none).has_value(), + "an empty column set must be rejected"); + + std::filesystem::remove(path); + return 0; +} + +static int test_write_options() +{ + xl::WriteOptions defaults{}; + xl_write_options raw = defaults.to_c(); + CHECK(raw.struct_size == static_cast(sizeof(xl_write_options)), "to_c must fill struct_size"); + CHECK(raw.sheet_name == nullptr, "an empty sheet_name must lower to NULL, meaning Sheet1"); + CHECK(raw.sheet_name_len == 0, "an empty sheet_name must lower to length 0"); + CHECK(raw.csv_delimiter == 0, "an unset csv_delimiter must lower to 0"); + CHECK(raw.date1904 == XL_OPT_DEFAULT, "an unset tri-state must lower to XL_OPT_DEFAULT"); + + xl::WriteOptions configured{}; + configured.sheet_name = "Dados"; + configured.csv_delimiter = ';'; + configured.use_shared_strings = XL_OPT_TRUE; + xl_write_options set = configured.to_c(); + CHECK(set.sheet_name_len == 5, "sheet_name_len must be the byte length"); + CHECK(set.sheet_name != nullptr, "a non-empty sheet_name must lower to a pointer"); + CHECK(set.csv_delimiter == ';', "csv_delimiter must pass through unchanged"); + CHECK(set.use_shared_strings == XL_OPT_TRUE, "use_shared_strings must pass through unchanged"); + return 0; +} + +static int test_format_from_path() +{ + CHECK(xl::format_from_path("out.xlsx") == XL_FORMAT_XLSX, ".xlsx must resolve to XL_FORMAT_XLSX"); + CHECK(xl::format_from_path("out.XLSB") == XL_FORMAT_XLSB, "the extension match must be case-insensitive"); + CHECK(xl::format_from_path("out.xls") == XL_FORMAT_XLS, ".xls must resolve to XL_FORMAT_XLS"); + CHECK(xl::format_from_path("out.csv") == XL_FORMAT_CSV, ".csv must resolve to XL_FORMAT_CSV"); + CHECK(xl::format_from_path("out.txt") == XL_FORMAT_AUTO, "an unknown extension must resolve to AUTO"); + CHECK(xl::format_from_path("out") == XL_FORMAT_AUTO, "no extension at all must resolve to AUTO"); + // A dot in a directory name is not an extension. + CHECK(xl::format_from_path("v1.2/report") == XL_FORMAT_AUTO, "a dot before the last separator is not an extension"); + return 0; +} + +struct FlagRow +{ + bool ativo; +}; + +template <> +struct xl::ExcelMapper +{ + static constexpr auto get_bindings() + { + return std::make_tuple(xl::make_field("ativo", &FlagRow::ativo)); + } +}; + +static int test_bool_round_trip() +{ + // One byte per row, 0 or 1 - XL_T_BOOL's wire layout, not a bit-packed bitmap. + const std::vector flags{1, 0, 1}; + const std::array columns{xl::bool_column("ativo", flags)}; + + const std::filesystem::path path = temp_path("bools.xlsx"); + CHECK(xl::write_columns(path.string(), XL_FORMAT_XLSX, columns).has_value(), + "writing a bool column must succeed"); + { + auto workbook = xl::Workbook::open(path.string()); + CHECK(workbook.has_value(), "the written file must open"); + auto table = xl::parse_sheet(*workbook); + CHECK(table.has_value(), "the written file must parse back"); + CHECK(table->size() == 3, "three rows were written"); + + CHECK(table->at(0)->ativo, "row 0 was written true"); + CHECK(!table->at(1)->ativo, "row 1 was written false"); + CHECK(table->at(2)->ativo, "row 2 was written true"); + } + std::filesystem::remove(path); + return 0; +} + +struct NullableRow +{ + std::optional quantidade; +}; + +template <> +struct xl::ExcelMapper +{ + static constexpr auto get_bindings() + { + return std::make_tuple(xl::make_field("quantidade", &NullableRow::quantidade)); + } +}; + +static int test_optional_round_trip() +{ + const std::vector valores{10, 0, 30}; + // LSB-first: bit 0 and bit 2 set, bit 1 clear - row 1 is null. + const std::vector validity{0b00000101}; + const std::array columns{xl::i64_column("quantidade", valores, validity)}; + + const std::filesystem::path path = temp_path("nullable.xlsx"); + CHECK(xl::write_columns(path.string(), XL_FORMAT_XLSX, columns).has_value(), + "writing a nullable column must succeed"); + { + auto workbook = xl::Workbook::open(path.string()); + CHECK(workbook.has_value(), "the written file must open"); + auto table = xl::parse_sheet(*workbook); + CHECK(table.has_value(), "the written file must parse back"); + CHECK(table->size() == 3, "three rows were written"); + + CHECK(table->at(0)->quantidade == 10, "row 0 must round-trip its value"); + CHECK(!table->at(1)->quantidade.has_value(), "row 1 was written null and must come back empty"); + CHECK(table->at(2)->quantidade == 30, "row 2 must round-trip its value"); + } + std::filesystem::remove(path); + return 0; +} + +struct FullRow +{ + std::string texto; + int64_t inteiro; + double numero; + bool ativo; + std::chrono::year_month_day data; + std::chrono::microseconds hora; + std::chrono::system_clock::time_point instante; + std::optional opcional; +}; + +template <> +struct xl::ExcelMapper +{ + static constexpr auto get_bindings() + { + return std::make_tuple( + xl::make_field("texto", &FullRow::texto), + xl::make_field("inteiro", &FullRow::inteiro), + xl::make_field("numero", &FullRow::numero), + xl::make_field("ativo", &FullRow::ativo), + xl::make_field("data", &FullRow::data), + xl::make_field("hora", &FullRow::hora), + xl::make_field("instante", &FullRow::instante), + xl::make_field("opcional", &FullRow::opcional)); + } +}; + +static int test_write_sheet_round_trip() +{ + const std::vector rows{ + FullRow{"uma", 1, 0.5, true, + std::chrono::year{2026} / std::chrono::month{1} / std::chrono::day{1}, + std::chrono::microseconds{3600000000}, + std::chrono::system_clock::time_point{std::chrono::microseconds{1767225600000000}}, + std::optional{7}}, + FullRow{"duas", 2, 1.5, false, + std::chrono::year{2026} / std::chrono::month{1} / std::chrono::day{2}, + std::chrono::microseconds{7200000000}, + std::chrono::system_clock::time_point{std::chrono::microseconds{1767312000000000}}, + std::nullopt}}; + + const std::filesystem::path path = temp_path("sheet.xlsx"); + // The format is inferred from the .xlsx extension by this overload. + CHECK(xl::write_sheet(path.string(), rows).has_value(), "write_sheet must succeed"); + { + auto workbook = xl::Workbook::open(path.string()); + CHECK(workbook.has_value(), "the written file must open"); + auto table = xl::parse_sheet(*workbook); + CHECK(table.has_value(), "the written file must parse back"); + CHECK(table->size() == 2, "two rows were written"); + + auto first = table->at(0); + CHECK(first.has_value(), "row 0 must be in bounds"); + CHECK(first->texto == "uma", "row 0's string must round-trip"); + CHECK(first->inteiro == 1, "row 0's int64 must round-trip"); + CHECK(first->numero == 0.5, "row 0's double must round-trip"); + CHECK(first->ativo, "row 0's bool must round-trip"); + CHECK(first->data == std::chrono::year{2026} / std::chrono::month{1} / std::chrono::day{1}, + "row 0's date must round-trip"); + CHECK(first->hora == std::chrono::microseconds{3600000000}, "row 0's time must round-trip"); + CHECK(first->opcional == 7, "row 0's optional must round-trip its value"); + + auto second = table->at(1); + CHECK(second.has_value(), "row 1 must be in bounds"); + CHECK(second->texto == "duas", "row 1's string must round-trip"); + CHECK(!second->ativo, "row 1's bool must round-trip as false"); + CHECK(!second->opcional.has_value(), "row 1's nullopt must come back empty"); + } + std::filesystem::remove(path); + return 0; +} + +static int test_write_sheet_to_memory_round_trip() +{ + const std::vector rows{ + FullRow{"uma", 1, 0.5, true, + std::chrono::year{2026} / std::chrono::month{1} / std::chrono::day{1}, + std::chrono::microseconds{3600000000}, + std::chrono::system_clock::time_point{std::chrono::microseconds{1767225600000000}}, + std::optional{7}}, + FullRow{"duas", 2, 1.5, false, + std::chrono::year{2026} / std::chrono::month{1} / std::chrono::day{2}, + std::chrono::microseconds{7200000000}, + std::chrono::system_clock::time_point{std::chrono::microseconds{1767312000000000}}, + std::nullopt}}; + + auto bytes = xl::write_sheet_to_memory(XL_FORMAT_XLSX, rows); + CHECK(bytes.has_value(), "write_sheet_to_memory must succeed"); + + auto workbook = xl::Workbook::open_memory(*bytes, XL_FORMAT_XLSX); + CHECK(workbook.has_value(), "the returned bytes must open"); + auto table = xl::parse_sheet(*workbook); + CHECK(table.has_value(), "the returned bytes must parse back"); + CHECK(table->size() == 2, "two rows were written"); + + auto first = table->at(0); + CHECK(first.has_value(), "row 0 must be in bounds"); + CHECK(first->texto == "uma", "row 0's string must round-trip"); + CHECK(first->opcional == 7, "row 0's optional must round-trip its value"); + auto second = table->at(1); + CHECK(second.has_value(), "row 1 must be in bounds"); + CHECK(!second->opcional.has_value(), "row 1's nullopt must come back empty"); + return 0; +} + +static int test_write_sheet_options_and_csv() +{ + const std::vector rows{}; + + const std::filesystem::path path = temp_path("named.xlsx"); + xl::WriteOptions options{}; + options.sheet_name = "Dados"; + CHECK(xl::write_sheet(path.string(), XL_FORMAT_XLSX, rows, &options).has_value(), + "writing an empty sheet with a custom name must succeed"); + { + auto workbook = xl::Workbook::open(path.string()); + CHECK(workbook.has_value(), "the written file must open"); + auto name = workbook->sheet_name(); + CHECK(name.has_value(), "the sheet name must be readable"); + CHECK(*name == "Dados", "WriteOptions::sheet_name must reach the file"); + } + std::filesystem::remove(path); + + const std::filesystem::path csv = temp_path("sheet.csv"); + CHECK(xl::write_sheet(csv.string(), rows).has_value(), "a .csv path must infer XL_FORMAT_CSV"); + std::filesystem::remove(csv); + return 0; +} + +// Writes one header row plus one data row through the raw streaming C ABI (not the C++ wrapper, +// which does not cover it), then reopens the file with xl::Workbook to prove xl_close_write_handle +// actually produced a valid, readable workbook - not just a status code. This is what the earlier +// version of this test skipped: it never opened the file it wrote, so a workbook left without its +// trailing structure (a corrupt XLSX zip) would still have passed. +static int test_writer_handle() +{ + static const auto write_str = [](xl_writer_handle *handle, std::string_view value) + { + return xl_write_string(handle, reinterpret_cast(value.data()), + static_cast(value.size())); + }; + + const std::filesystem::path path = temp_path("writer_handle.xlsx"); + xl_writer_handle *handle = nullptr; + const std::string c_path = path.string(); + int status = xl_open_write_handle(reinterpret_cast(c_path.data()), + static_cast(c_path.size()), XL_FORMAT_XLSX, nullptr, &handle); + CHECK(status == XL_OK, "xl_open_write_handle must succeed"); + CHECK(handle != nullptr, "the returned handle must be non-null"); + + status = xl_start_sheet(handle, reinterpret_cast("Planilha1"), 9); + CHECK(status == XL_OK, "xl_start_sheet must succeed"); + + status = xl_start_row(handle); + CHECK(status == XL_OK, "xl_start_row must succeed for the header row"); + for (std::string_view header : {"texto", "inteiro", "numero", "data", "hora", "instante"}) + { + CHECK(write_str(handle, header) == XL_OK, "writing a header cell must succeed"); + } + status = xl_end_row(handle); + CHECK(status == XL_OK, "xl_end_row must succeed for the header row"); + + status = xl_start_row(handle); + CHECK(status == XL_OK, "xl_start_row must succeed for the data row"); + CHECK(write_str(handle, "uma") == XL_OK, "xl_write_string must succeed"); + CHECK(xl_write_int64(handle, 1) == XL_OK, "xl_write_int64 must succeed"); + CHECK(xl_write_float64(handle, 0.5) == XL_OK, "xl_write_float64 must succeed"); + CHECK(xl_write_date(handle, 20454) == XL_OK, "xl_write_date must succeed"); // 2026-01-01 + CHECK(xl_write_time(handle, 3600000000) == XL_OK, "xl_write_time must succeed"); + CHECK(xl_write_timestamp(handle, 1767225600000000) == XL_OK, "xl_write_timestamp must succeed"); + status = xl_end_row(handle); + CHECK(status == XL_OK, "xl_end_row must succeed for the data row"); + + status = xl_end_sheet(handle); + CHECK(status == XL_OK, "xl_end_sheet must succeed"); + status = xl_close_write_handle(handle); + CHECK(status == XL_OK, "xl_close_write_handle must succeed"); + { + auto workbook = xl::Workbook::open(path.string()); + CHECK(workbook.has_value(), "the file xl_close_write_handle produced must open"); + auto name = workbook->sheet_name(); + CHECK(name.has_value() && *name == "Planilha1", "xl_start_sheet's name must reach the file"); + auto table = xl::parse_sheet(*workbook); + CHECK(table.has_value(), "the written file must parse back"); + CHECK(table->size() == 1, "exactly one data row was written"); + WrittenRow first = *table->begin(); + CHECK(first.texto == "uma", "the streamed string cell must round-trip"); + CHECK(first.inteiro == 1, "the streamed int64 cell must round-trip"); + CHECK(first.numero == 0.5, "the streamed float64 cell must round-trip"); + CHECK(first.data == std::chrono::year{2026} / std::chrono::month{1} / std::chrono::day{1}, + "the streamed date cell must round-trip"); + CHECK(first.hora == std::chrono::microseconds{3600000000}, "the streamed time cell must round-trip"); + } + std::filesystem::remove(path); + return 0; +} + +// Every writer entry point must resolve a bad/closed handle as XL_INVALID_HANDLE, not +// XL_INVALID_ARGUMENT - the same convention xl_close uses on the reader side. +static int test_writer_handle_rejects_bad_handle() +{ + xl_writer_handle *bogus = reinterpret_cast(static_cast(0x1)); + CHECK(xl_start_sheet(bogus, reinterpret_cast("x"), 1) == XL_INVALID_HANDLE, + "xl_start_sheet on a bad handle must return XL_INVALID_HANDLE"); + CHECK(xl_start_row(bogus) == XL_INVALID_HANDLE, "xl_start_row on a bad handle must return XL_INVALID_HANDLE"); + CHECK(xl_write_int64(bogus, 1) == XL_INVALID_HANDLE, "xl_write_int64 on a bad handle must return XL_INVALID_HANDLE"); + CHECK(xl_end_row(bogus) == XL_INVALID_HANDLE, "xl_end_row on a bad handle must return XL_INVALID_HANDLE"); + CHECK(xl_end_sheet(bogus) == XL_INVALID_HANDLE, "xl_end_sheet on a bad handle must return XL_INVALID_HANDLE"); + CHECK(xl_close_write_handle(bogus) == XL_INVALID_HANDLE, "xl_close_write_handle on a bad handle must return XL_INVALID_HANDLE"); + CHECK(xl_close_write_handle(nullptr) == XL_INVALID_HANDLE, "xl_close_write_handle on a null handle must return XL_INVALID_HANDLE"); + return 0; +} + +// A row/sheet/write out of order must fail as XL_ERROR (not crash, not silently succeed) and must +// leave the handle usable, per the call-order contract documented on xl_writer_handle. +static int test_writer_handle_rejects_out_of_order_calls() +{ + const std::filesystem::path path = temp_path("writer_handle_order.xlsx"); + const std::string c_path = path.string(); + xl_writer_handle *handle = nullptr; + int status = xl_open_write_handle(reinterpret_cast(c_path.data()), + static_cast(c_path.size()), XL_FORMAT_XLSX, nullptr, &handle); + CHECK(status == XL_OK, "xl_open_write_handle must succeed"); + + CHECK(xl_start_row(handle) == XL_ERROR, "xl_start_row before xl_start_sheet must fail"); + CHECK(xl_write_int64(handle, 1) == XL_ERROR, "a cell write before xl_start_row must fail"); + CHECK(xl_end_row(handle) == XL_ERROR, "xl_end_row without an open row must fail"); + CHECK(xl_end_sheet(handle) == XL_ERROR, "xl_end_sheet without an open sheet must fail"); + + status = xl_start_sheet(handle, reinterpret_cast("S"), 1); + CHECK(status == XL_OK, "xl_start_sheet must still succeed after the earlier rejected calls"); + status = xl_close_write_handle(handle); + CHECK(status == XL_OK, "xl_close_write_handle must still succeed after the earlier rejected calls"); + + std::filesystem::remove(path); + return 0; +} + +// Exercises xl::WriterHandle::open (file-backed) through every write branch, including the +// std::optional null-cell path, then reopens the file to confirm the output matches what the +// raw-C xl_writer_handle test above wrote by hand. +static int test_writer_handle_class_round_trip() +{ + const std::filesystem::path path = temp_path("writer_handle_class.xlsx"); + { + // Scoped so the destructor closes and releases the handle - including the exclusive file + // lock xl_open_write_handle takes - before Workbook::open reopens the same path below. + auto handle = xl::WriterHandle::open(path.string(), XL_FORMAT_XLSX); + CHECK(handle.has_value(), "WriterHandle::open must succeed"); + + CHECK(handle->start_sheet("Planilha1").has_value(), "start_sheet must succeed"); + + CHECK(handle->start_row().has_value(), "start_row must succeed for the header row"); + for (std::string_view header : {"texto", "inteiro", "numero", "ativo", "data", "hora", "instante", "opcional"}) + { + CHECK(handle->write(header).has_value(), "writing a header cell must succeed"); + } + CHECK(handle->end_row().has_value(), "end_row must succeed for the header row"); + + CHECK(handle->start_row().has_value(), "start_row must succeed for the data row"); + CHECK(handle->write(std::string_view("uma")).has_value(), "write(string_view) must succeed"); + CHECK(handle->write(int64_t{1}).has_value(), "write(int64_t) must succeed"); + CHECK(handle->write(0.5).has_value(), "write(double) must succeed"); + CHECK(handle->write(true).has_value(), "write(bool) must succeed"); + CHECK(handle->write(std::chrono::year{2026} / std::chrono::month{1} / std::chrono::day{1}).has_value(), + "write(year_month_day) must succeed"); + CHECK(handle->write(std::chrono::microseconds{3600000000}).has_value(), "write(microseconds) must succeed"); + CHECK(handle->write(std::chrono::system_clock::time_point{std::chrono::microseconds{1767225600000000}}) + .has_value(), + "write(time_point) must succeed"); + CHECK(handle->write(std::optional{7}).has_value(), "write(optional{value}) must succeed"); + CHECK(handle->end_row().has_value(), "end_row must succeed for the data row"); + + CHECK(handle->start_row().has_value(), "start_row must succeed for the null-cell row"); + CHECK(handle->write(std::optional{}).has_value(), + "write(optional{}) must succeed"); + CHECK(handle->write(std::optional{}).has_value(), "write(optional{}) must succeed"); + CHECK(handle->write_null(XL_T_F64).has_value(), "write_null(XL_T_F64) must succeed"); + for (int i = 0; i < 5; ++i) + { + CHECK(handle->write_null(XL_T_STRING).has_value(), "padding the null-cell row out must succeed"); + } + CHECK(handle->end_row().has_value(), "end_row must succeed for the null-cell row"); + + CHECK(handle->end_sheet().has_value(), "end_sheet must succeed"); + } + { + auto workbook = xl::Workbook::open(path.string()); + CHECK(workbook.has_value(), "the file WriterHandle produced must open"); + auto table = xl::parse_sheet(*workbook); + CHECK(table.has_value(), "the written file must parse back"); + CHECK(table->size() == 2, "two data rows were written"); + + auto first = table->at(0); + CHECK(first.has_value(), "row 0 must be in bounds"); + CHECK(first->texto == "uma", "the streamed string cell must round-trip"); + CHECK(first->inteiro == 1, "the streamed int64 cell must round-trip"); + CHECK(first->numero == 0.5, "the streamed double cell must round-trip"); + CHECK(first->ativo, "the streamed bool cell must round-trip"); + CHECK(first->opcional == 7, "the streamed optional cell must round-trip its value"); + + auto second = table->at(1); + CHECK(second.has_value(), "row 1 must be in bounds"); + CHECK(second->texto.empty(), "write(optional{}) must have written a blank cell"); + CHECK(!second->opcional.has_value(), "write(optional{}) must have written a blank cell"); + } + std::filesystem::remove(path); + return 0; +} + +// Same as test_writer_handle_class_round_trip, but backed by open_memory()/bytes() instead of a +// file. +static int test_writer_handle_class_to_memory_round_trip() +{ + auto handle = xl::WriterHandle::open_memory(XL_FORMAT_XLSX); + CHECK(handle.has_value(), "WriterHandle::open_memory must succeed"); + + CHECK(handle->start_sheet("Dados").has_value(), "start_sheet must succeed"); + CHECK(handle->start_row().has_value(), "start_row must succeed"); + CHECK(handle->write(std::string_view("uma")).has_value(), "write(string_view) must succeed"); + CHECK(handle->write(int64_t{3}).has_value(), "write(int64_t) must succeed"); + CHECK(handle->end_row().has_value(), "end_row must succeed"); + + auto bytes = handle->bytes(); + CHECK(bytes.has_value(), "bytes() must succeed"); + CHECK(!bytes->empty(), "bytes() must return non-empty bytes"); + + auto workbook = xl::Workbook::open_memory(*bytes, XL_FORMAT_XLSX); + CHECK(workbook.has_value(), "the bytes bytes() returned must open"); + auto sheet_name = workbook->sheet_name(); + CHECK(sheet_name.has_value() && *sheet_name == "Dados", "start_sheet's name must reach the bytes"); + + // bytes() must not have released the handle - unlike end_sheet/start_sheet (bytes() already + // ended the sheet internally to produce a valid result, so calling those again would rightly + // fail), a second bytes() call is still valid and must return the same content. + auto bytes_again = handle->bytes(); + CHECK(bytes_again.has_value(), "a second bytes() call must still succeed"); + CHECK(*bytes_again == *bytes, "a second bytes() call must return the same content"); + return 0; +} + +// bytes() on a file-backed handle (opened via open(), not open_memory()) must fail cleanly - the +// same XL_INVALID_ARGUMENT xl_write_handle_bytes itself returns for that case. +static int test_writer_handle_class_bytes_rejects_a_file_backed_handle() +{ + const std::filesystem::path path = temp_path("writer_handle_class_file.xlsx"); + { + // Scoped for the same reason as test_writer_handle_class_round_trip: the handle holds the + // path open exclusively until its destructor runs, and std::filesystem::remove below needs + // that lock released first. + auto handle = xl::WriterHandle::open(path.string(), XL_FORMAT_XLSX); + CHECK(handle.has_value(), "WriterHandle::open must succeed"); + CHECK(handle->start_sheet("S").has_value(), "start_sheet must succeed"); + + auto bytes = handle->bytes(); + CHECK(!bytes.has_value(), "bytes() on a file-backed handle must fail"); + CHECK(bytes.error().code == XL_INVALID_ARGUMENT, + "bytes() on a file-backed handle must be XL_INVALID_ARGUMENT"); + } + std::filesystem::remove(path); + return 0; +} + +int main() +{ + if (test_write_options() != 0) + { + return 1; + } + if (test_format_from_path() != 0) + { + return 1; + } + if (test_write_columns_round_trip() != 0) + { + return 1; + } + if (test_write_columns_to_memory_round_trip() != 0) + { + return 1; + } + if (test_write_columns_rejects_bad_input() != 0) + { + return 1; + } + if (test_bool_round_trip() != 0) + { + return 1; + } + if (test_optional_round_trip() != 0) + { + return 1; + } + if (test_write_sheet_round_trip() != 0) + { + return 1; + } + if (test_write_sheet_to_memory_round_trip() != 0) + { + return 1; + } + if (test_write_sheet_options_and_csv() != 0) + { + return 1; + } + if (test_writer_handle() != 0) + { + return 1; + } + if (test_writer_handle_rejects_bad_handle() != 0) + { + return 1; + } + if (test_writer_handle_rejects_out_of_order_calls() != 0) + { + return 1; + } + if (test_writer_handle_class_round_trip() != 0) + { + return 1; + } + if (test_writer_handle_class_to_memory_round_trip() != 0) + { + return 1; + } + if (test_writer_handle_class_bytes_rejects_a_file_backed_handle() != 0) + { + return 1; + } + std::printf("OK\n"); + return 0; +} \ No newline at end of file diff --git a/python/README.md b/python/README.md index 109b8a1..7b1a885 100644 --- a/python/README.md +++ b/python/README.md @@ -243,6 +243,58 @@ options = OpenOptions( single-file batch work, not for a server already reading many files in parallel. See the root README for the measured trade. +## Benchmarks + +`benchmarks/bench_read.py` and `benchmarks/bench_write.py` over +`tests/ExcelReader.Benchmarks/Data/65K_Records_Data.xlsb` (65,535 data rows, 14 columns), the same +fixture the .NET, C++ and Rust suites use. Measured on Windows 10 (22H2), 16 logical CPUs +@ 3.39 GHz, CPython 3.14.5, 10 runs each (medians shown; `min` is in the scripts' own output). + +### Reading + +| API | Median | What it produces | +|---|---:|---| +| `parse_typed()` | 54.1 ms | typed columnar buffers, converted natively | +| `to_arrow()` | 54.3 ms | the same parse, handed to pyarrow zero-copy | +| `to_polars()` | 56.1 ms | typed columnar DataFrame, schema inferred | +| `polars.read_excel()` | 137.7 ms | typed columnar DataFrame, types inferred | +| `read_all_columnar()` | 508.5 ms | raw columnar cells, no per-cell Python objects | +| `rows()` | 1,054.6 ms | one `Cell` object per cell, streamed per row | +| `read_all()` | 1,616.3 ms | one `Cell` object per cell, all at once | + +Only the `to_polars()` / `polars.read_excel()` pair is a like-for-like comparison, and even that one +is loose: both produce a typed columnar DataFrame with inferred types, but the inference rules are +not identical. The rows above it produce different things and are listed to show what each API +costs, not to rank them — `read_all()` is 30x slower than `parse_typed()` because it materializes +917,504 Python objects, which is the price of that shape, not a slow parser. + +### Writing + +| API | Median | Output | +|---|---:|---| +| `write_workbook()` → xls | 45.6 ms | 17.7 MB | +| `write_workbook()` → csv | 64.2 ms | 8.2 MB | +| `write_workbook()` → xlsb | 74.1 ms | 5.2 MB | +| `write_workbook()` → xlsx | 129.3 ms | 5.1 MB | +| `write_polars()` → xlsx | 507.5 ms | 5.1 MB | +| `write_pandas()` → xlsx | 514.6 ms | 5.1 MB | +| `polars.DataFrame.write_excel()` | 4,487.8 ms | 5.6 MB | +| `pandas.DataFrame.to_excel()` | 6,972.0 ms | 5.5 MB | + +The two DataFrame comparisons are matched work — same DataFrame in, xlsx out both times: +`write_polars()` is ~8.8x faster than polars' own `write_excel()`, and `write_pandas()` ~13.5x +faster than `to_excel()`. Both of ours pay a conversion the raw path does not: the DataFrame goes +through Arrow and then a Python list before reaching the native columns, which is most of the gap +between the 507 ms row and the 129 ms one. Handing `write_workbook()` buffers that are already +columnar — what `parse_typed()` returns — skips all of it. + +`write_workbook(xlsx)` at 129.3 ms lands within a couple of milliseconds of the C++ binding's +`xl::write_columns` on the same 14 columns, which is the expected result: both are thin wrappers +over the same `xl_write_typed` call, and neither adds work per cell. + +`xls` being the fastest and largest is not a paradox — BIFF8 writes fixed-width records with no +compression, so it trades 3.5x the bytes for less work per cell. + ## Notes - A `Workbook` is **not** thread-safe. Use one per thread. diff --git a/python/pyproject.toml b/python/pyproject.toml index f585010..39c3d57 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -14,10 +14,15 @@ classifiers = [ "Programming Language :: Python :: 3", "License :: OSI Approved :: MIT License", ] -dependencies = ["typing_extensions>=4.0"] +dependencies = [] [project.optional-dependencies] -dev = ["pytest>=7.0"] +# typing_extensions is a type-checking-time-only import (see reader.py's TYPE_CHECKING guard for +# typing.Self, 3.11+ in the stdlib) — never evaluated at runtime, so it belongs in dev, not +# dependencies. Keeping it out of the wheel's runtime deps is the point: this package's whole +# selling point is a ctypes-only, interpreter-independent wheel with no compiled Python extension +# and (now) no runtime dependency at all. +dev = ["pytest>=7.0", "typing_extensions>=4.0"] # Optional: Workbook.read_all_columnar() returns NumPy int32 arrays when this is installed, and # array('i') (stdlib, always available) otherwise — see reader.py's _to_columnar_array(). Kept # optional on purpose: the package's whole selling point is a ctypes-only, interpreter-independent diff --git a/python/src/excelreader/_native.py b/python/src/excelreader/_native.py index af29ebb..e3cc0b5 100644 --- a/python/src/excelreader/_native.py +++ b/python/src/excelreader/_native.py @@ -5,10 +5,10 @@ from __future__ import annotations -from collections.abc import Sequence import ctypes import os import platform +from collections.abc import Sequence from functools import lru_cache from pathlib import Path from typing import TYPE_CHECKING @@ -27,13 +27,25 @@ # Bumped on any change to a struct layout, a status code, or the meaning of an existing function; # adding a new function does not bump it. Mirrors XL_ABI_VERSION in include/excelreader.h. -XL_ABI_VERSION = 2 +XL_ABI_VERSION = 3 XL_FORMAT_AUTO = 0 XL_FORMAT_XLS = 1 XL_FORMAT_XLSX = 2 XL_FORMAT_XLSB = 3 XL_FORMAT_CSV = 4 +# The one mapping from a public format name to its XL_FORMAT_* value. reader.py and writer.py both +# derive their tables from these rather than restating them. +FORMATS = { + "auto": XL_FORMAT_AUTO, + "xls": XL_FORMAT_XLS, + "xlsx": XL_FORMAT_XLSX, + "xlsb": XL_FORMAT_XLSB, + "csv": XL_FORMAT_CSV, +} +# xl_write_typed rejects XL_FORMAT_AUTO - a file being created has no signature bytes to sniff - so +# the write side gets the same table minus that entry. +WRITE_FORMATS = {name: value for name, value in FORMATS.items() if name != "auto"} # Every boolean-shaped NativeOpenOptions field uses one of these three states, never a plain 0/1 - # several of them default to true, so a bare 0 would be ambiguous between "off" and "use the library @@ -117,23 +129,18 @@ def column_spec_by_index(index: int, type_: int, *, nullable: bool = False) -> N ) -class NativeInferredColumnSpec(ctypes.Structure): - """Mirrors xl_column_spec's layout exactly, for the OUTPUT direction (xl_infer_schema). +class NativeInferredColumnSpec(NativeColumnSpec): + """The OUTPUT direction of the same xl_column_spec (xl_infer_schema fills these in). + + Layout is inherited, not restated: it is literally the same C struct, and a second copy of the + field list is a second place for it to drift from the header. Only the ownership rules differ, + which is why this carries its own name. Always carries `name_count` 0 or 1 — inference never guesses more than one candidate name per column. Unlike `column_spec_by_name`'s buffers, `names[0]` here is a raw pointer with no guaranteed NUL terminator, so decode it with `ctypes.string_at(names[0], name_lens[0])`. """ - _fields_ = [ - ("names", ctypes.POINTER(ctypes.POINTER(ctypes.c_uint8))), - ("name_lens", ctypes.POINTER(ctypes.c_int32)), - ("name_count", ctypes.c_int32), - ("index", ctypes.c_int32), - ("type", ctypes.c_int32), - ("nullable", ctypes.c_int32), - ] - class NativeInferredSchema(ctypes.Structure): """Mirrors xl_inferred_schema. `columns` is a native-owned array of `column_count` values, freed @@ -414,10 +421,6 @@ def _bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.xl_next_row.restype = c_int lib.xl_read_all_blob.argtypes = [p_void, p_bytes, c_int, p_int] lib.xl_read_all_blob.restype = c_int - lib.xl_next_row_decoded.argtypes = [p_void, ctypes.POINTER(NativeRow)] - lib.xl_next_row_decoded.restype = c_int - lib.xl_free_row.argtypes = [ctypes.POINTER(NativeRow)] - lib.xl_free_row.restype = None lib.xl_last_error.argtypes = [p_bytes, c_int, p_int] lib.xl_last_error.restype = c_int lib.xl_last_error_ptr.argtypes = [p_int] diff --git a/python/src/excelreader/reader.py b/python/src/excelreader/reader.py index 567ff51..6cb000f 100644 --- a/python/src/excelreader/reader.py +++ b/python/src/excelreader/reader.py @@ -7,8 +7,13 @@ from array import array from collections.abc import Iterator, Sequence from pathlib import Path +from typing import TYPE_CHECKING -from typing_extensions import Self +if TYPE_CHECKING: + # typing.Self is 3.11+; typing_extensions backports it for 3.9/3.10. Only needed for the + # annotation below, which `from __future__ import annotations` keeps unevaluated at runtime — + # so the wheel stays free of a runtime dependency on typing_extensions. + from typing_extensions import Self from excelreader import _native from excelreader.types import ( @@ -29,13 +34,7 @@ _numpy = None # NumPy is an optional extra (pip install excelreader-native[numpy]) — see # _buffer_to_array()/_to_columnar_array() below, the only two places that consult it. -_FORMATS = { - "auto": _native.XL_FORMAT_AUTO, - "xls": _native.XL_FORMAT_XLS, - "xlsx": _native.XL_FORMAT_XLSX, - "xlsb": _native.XL_FORMAT_XLSB, - "csv": _native.XL_FORMAT_CSV, -} +_FORMATS = _native.FORMATS _CELL_HEADER = struct.Struct(" int: diff --git a/python/tests/test_native.py b/python/tests/test_native.py index 46c8bf1..af1ecc6 100644 --- a/python/tests/test_native.py +++ b/python/tests/test_native.py @@ -32,8 +32,6 @@ def test_exported_functions_are_present(): "xl_is_date1904", "xl_next_row", "xl_read_all_blob", - "xl_next_row_decoded", - "xl_free_row", "xl_read_all_decoded", "xl_free_rows", "xl_parse_typed", diff --git a/rust/excelreader-derive/src/lib.rs b/rust/excelreader-derive/src/lib.rs index ba4fc47..d52618a 100644 --- a/rust/excelreader-derive/src/lib.rs +++ b/rust/excelreader-derive/src/lib.rs @@ -23,58 +23,117 @@ fn expand(input: DeriveInput) -> syn::Result { let struct_name = &input.ident; let fields = named_fields(&input)?; - let bindings = fields + // Parsed ONCE, up front: both halves below read the same ParsedField, so they cannot disagree + // about a field's kind, its primary name, or whether it is optional. Every attribute and type + // error surfaces here, before either half is generated. + let parsed = fields .iter() - .map(field_binding) + .map(parse_field) .collect::>>()?; + let bindings = parsed.iter().map(field_binding).collect::>(); + + // The write half needs one set of buffer identifiers per field, so it is generated by index + // rather than field by field: the declarations, the per-row pushes, and the OwnedColumn + // construction all have to agree on the same names. + let mut declarations = Vec::new(); + let mut pushes = Vec::new(); + let mut columns = Vec::new(); + for (index, field) in parsed.iter().enumerate() { + let plan = WriteField::new(index, field); + declarations.push(plan.declarations()); + pushes.push(plan.push()); + columns.push(plan.column()); + } + Ok(quote! { impl ::excelreader::workbook::ExcelMapper for #struct_name { fn bindings() -> ::std::vec::Vec<::excelreader::workbook::ColumnBinding> { ::std::vec![ #(#bindings),* ] } } + + impl ::excelreader::writer::ExcelWriter for #struct_name { + fn to_columns( + rows: &[Self], + ) -> ::core::result::Result< + ::std::vec::Vec<::excelreader::writer::OwnedColumn>, + ::excelreader::Error, + > { + let n = rows.len(); + #(#declarations)* + // ONE pass over the rows. Every field's append is monomorphic and inlined - there + // is no per-cell match, no boxed closure, and no trait object anywhere in here. + for (row, r) in rows.iter().enumerate() { + let _ = row; // unused when no field is Option<_> + #(#pushes)* + } + ::core::result::Result::Ok(::std::vec![ #(#columns),* ]) + } + } }) } fn named_fields( input: &DeriveInput, ) -> syn::Result<&syn::punctuated::Punctuated> { - match &input.data { - Data::Struct(data) => match &data.fields { - Fields::Named(fields) => Ok(&fields.named), - _ => Err(syn::Error::new_spanned( - input, - "ExcelMapper can only be derived for structs with named fields", - )), - }, - _ => Err(syn::Error::new_spanned( - input, - "ExcelMapper can only be derived for structs with named fields", - )), + // One pattern, one message: a tuple struct, a unit struct and an enum are all the same rejection, + // and writing it twice is how the two copies eventually stop matching. + if let Data::Struct(syn::DataStruct { + fields: Fields::Named(fields), + .. + }) = &input.data + { + return Ok(&fields.named); } + Err(syn::Error::new_spanned( + input, + "ExcelMapper can only be derived for structs with named fields", + )) +} + +/// Everything both halves of the derive need from one field, resolved once. +/// +/// The read half (`field_binding`) and the write half (`WriteField`) used to each re-run +/// `excel_names`/`unwrap_option`/`FieldKind::from_type` over the same field. Parsing once is not +/// just less work: it is what guarantees the two halves cannot disagree about a field's kind or +/// about which of its names is the primary one. +struct ParsedField<'a> { + ident: &'a syn::Ident, + /// Primary `#[excel(name = "...")]` first, then any aliases in declared order. + names: Vec, + kind: FieldKind, + is_option: bool, } -fn field_binding(field: &Field) -> syn::Result { - let field_ident = field.ident.as_ref().expect("named_fields guarantees Some"); - let name = excel_names(field)?; +fn parse_field(field: &Field) -> syn::Result> { let (inner_ty, is_option) = unwrap_option(&field.ty); - let kind = FieldKind::from_type(inner_ty)?; - let xl_type = kind.xl_type_tokens(); - let value = kind.value_tokens(); - let assign_value = if is_option { + Ok(ParsedField { + ident: field.ident.as_ref().expect("named_fields guarantees Some"), + names: excel_names(field)?, + kind: FieldKind::from_type(inner_ty)?, + is_option, + }) +} + +fn field_binding(parsed: &ParsedField<'_>) -> proc_macro2::TokenStream { + let field_ident = parsed.ident; + let name = &parsed.names; + let xl_type = parsed.kind.xl_type_tokens(); + let value = parsed.kind.value_tokens(); + let assign_value = if parsed.is_option { quote! { ::std::option::Option::Some(#value) } } else { value }; - Ok(quote! { + quote! { ::excelreader::workbook::ColumnBinding { names: &[#(#name),*], xl_type: #xl_type, assign: |r, col, row| r.#field_ident = #assign_value, } - }) + } } fn excel_names(field: &Field) -> syn::Result> { @@ -127,6 +186,7 @@ fn unwrap_option(ty: &Type) -> (&Type, bool) { (ty, false) } +#[derive(Clone, Copy)] enum FieldKind { Str, /// `i64` itself, and every other integer width. The column is always `XL_T_I64` on the wire; a @@ -140,6 +200,174 @@ enum FieldKind { Timestamp, } +struct WriteField { + ident: syn::Ident, + /// The PRIMARY #[excel(name = "...")] only. An alias must never reach a write spec: the ABI + /// rejects a write spec carrying more than one name. + name: LitStr, + kind: FieldKind, + is_option: bool, + values: syn::Ident, + offsets: syn::Ident, + data: syn::Ident, + validity: syn::Ident, +} + +impl WriteField { + fn new(index: usize, parsed: &ParsedField<'_>) -> WriteField { + WriteField { + ident: parsed.ident.clone(), + name: parsed + .names + .first() + .expect("excel_names always returns the primary name first") + .clone(), + kind: parsed.kind, + is_option: parsed.is_option, + values: quote::format_ident!("values_{index}"), + offsets: quote::format_ident!("offsets_{index}"), + data: quote::format_ident!("data_{index}"), + validity: quote::format_ident!("validity_{index}"), + } + } + + fn declarations(&self) -> proc_macro2::TokenStream { + let (values, offsets, data, validity) = + (&self.values, &self.offsets, &self.data, &self.validity); + let buffers = if matches!(self.kind, FieldKind::Str) { + quote! { + let mut #offsets: ::std::vec::Vec = ::std::vec::Vec::with_capacity(n + 1); + #offsets.push(0); + let mut #data: ::std::vec::Vec = ::std::vec::Vec::new(); + } + } else { + let element = self.kind.element_type(); + quote! { + let mut #values: ::std::vec::Vec<#element> = ::std::vec::Vec::with_capacity(n); + } + }; + if !self.is_option { + return buffers; + } + // ponytail: the bitmap is materialized for every Option<_> field, even one where no row + // turns out to be None - n/8 bytes spent for nothing in that case. Tracking a `saw_none` + // flag and passing None instead would reclaim them; not worth the branch until a profile + // says otherwise. + quote! { + #buffers + let mut #validity: ::std::vec::Vec = ::std::vec![0u8; n.div_ceil(8)]; + } + } + + fn push(&self) -> proc_macro2::TokenStream { + let ident = &self.ident; + let append = self.append_tokens(); + let placeholder = self.placeholder_tokens(); + if !self.is_option { + return quote! { + { + let value = &r.#ident; + #append + } + }; + } + let validity = &self.validity; + quote! { + match &r.#ident { + ::core::option::Option::Some(value) => { + ::excelreader::writer::set_valid(&mut #validity, row); + #append + } + ::core::option::Option::None => { #placeholder } + } + } + } + + /// Appends `value: &Inner` to this field's buffers. The conversions are the exact inverses of + /// `FieldKind::value_tokens` on the read side - if one gains a type, so must the other. + fn append_tokens(&self) -> proc_macro2::TokenStream { + let (values, offsets, data) = (&self.values, &self.offsets, &self.data); + let field = self.name.value(); + match self.kind { + FieldKind::Str => quote! { + ::excelreader::writer::push_str(&mut #offsets, &mut #data, value.as_str())?; + }, + // Through TryFrom, not `as`, for the same reason the read side does: an `as` cast + // would silently wrap a u64 that does not fit an i64 column, and a writer that quietly + // changes the number it was given is worse than one that stops. + FieldKind::Int => quote! { + #values.push( + i64::try_from(*value).map_err(|_| ::excelreader::Error { + code: ::excelreader::XL_INVALID_ARGUMENT, + message: ::std::format!( + "field `{}` holds a value that does not fit an i64 column", + #field + ), + })?, + ); + }, + FieldKind::Float => quote! { #values.push(f64::from(*value)); }, + FieldKind::Bool => quote! { #values.push(u8::from(*value)); }, + FieldKind::Date => quote! { + #values.push( + ::core::convert::Into::<::excelreader::Date>::into(*value).days_since_epoch, + ); + }, + FieldKind::Time => quote! { + #values.push( + ::core::convert::Into::<::excelreader::Time>::into(*value).micros_since_midnight, + ); + }, + FieldKind::Timestamp => quote! { + #values.push( + ::core::convert::Into::<::excelreader::Timestamp>::into(*value).micros_since_epoch, + ); + }, + } + } + + /// A null row still occupies a slot in the values buffer; its cleared validity bit is what + /// marks it absent. Zero (or the empty string) is the placeholder the writer never reads. + fn placeholder_tokens(&self) -> proc_macro2::TokenStream { + let (values, offsets, data) = (&self.values, &self.offsets, &self.data); + if matches!(self.kind, FieldKind::Str) { + return quote! { ::excelreader::writer::push_str(&mut #offsets, &mut #data, "")?; }; + } + quote! { #values.push(::core::default::Default::default()); } + } + + fn column(&self) -> proc_macro2::TokenStream { + let (values, offsets, data, validity) = + (&self.values, &self.offsets, &self.data, &self.validity); + let name = &self.name; + let payload = match self.kind { + FieldKind::Str => quote! { + ::excelreader::writer::OwnedColumnData::Str { offsets: #offsets, data: #data } + }, + FieldKind::Int => quote! { ::excelreader::writer::OwnedColumnData::I64(#values) }, + FieldKind::Float => quote! { ::excelreader::writer::OwnedColumnData::F64(#values) }, + FieldKind::Bool => quote! { ::excelreader::writer::OwnedColumnData::Bool(#values) }, + FieldKind::Date => quote! { ::excelreader::writer::OwnedColumnData::Date(#values) }, + FieldKind::Time => quote! { ::excelreader::writer::OwnedColumnData::Time(#values) }, + FieldKind::Timestamp => { + quote! { ::excelreader::writer::OwnedColumnData::Timestamp(#values) } + } + }; + let validity_expr = if self.is_option { + quote! { ::core::option::Option::Some(#validity) } + } else { + quote! { ::core::option::Option::None } + }; + quote! { + ::excelreader::writer::OwnedColumn { + name: ::core::option::Option::Some(#name), + data: #payload, + validity: #validity_expr, + } + } + } +} + /// Every integer type that maps onto an `XL_T_I64` column. `i64` is included so the conversion in /// `value_tokens` stays uniform - `TryFrom for i64` exists via the blanket `From` impl and /// compiles away to nothing. @@ -157,13 +385,8 @@ impl FieldKind { .map(|segment| segment.ident.to_string()), _ => None, }; - let name = name.as_deref(); - if let Some(name) = name { - if INT_TYPES.contains(&name) { - return Ok(FieldKind::Int); - } - } - match name { + match name.as_deref() { + Some(int) if INT_TYPES.contains(&int) => Ok(FieldKind::Int), Some("String") => Ok(FieldKind::Str), Some("f32" | "f64") => Ok(FieldKind::Float), Some("bool") => Ok(FieldKind::Bool), @@ -222,6 +445,17 @@ impl FieldKind { } } } + /// The Rust element type of this kind's output buffer, at the column type's exact wire width. + fn element_type(&self) -> proc_macro2::TokenStream { + match self { + // Str has no single element type - it uses an offsets/data pair instead. + FieldKind::Str => quote! { u8 }, + FieldKind::Int | FieldKind::Time | FieldKind::Timestamp => quote! { i64 }, + FieldKind::Float => quote! { f64 }, + FieldKind::Bool => quote! { u8 }, + FieldKind::Date => quote! { i32 }, + } + } } #[cfg(test)] @@ -401,4 +635,42 @@ mod tests { assert!(nome_pos < nom_pos, "primary name must precede its aliases"); assert!(nom_pos < name_pos, "aliases must stay in declared order"); } + #[test] + fn generates_an_excel_writer_impl_alongside_the_mapper() { + let output = expand_str( + r#" + struct Row { + #[excel(name = "Nome", alias = "Name")] + nome: String, + #[excel(name = "Peso")] + peso: Option, + } + "#, + ) + .expect("expand must succeed"); + + assert!(output.contains("impl :: excelreader :: writer :: ExcelWriter for Row")); + assert!( + output.contains("push_str"), + "a String field must go through push_str" + ); + assert!( + output.contains("set_valid"), + "an Option field must set a validity bit" + ); + // The alias belongs to the read side only: a write spec carrying two names is rejected by + // the ABI. + let writer_half = output + .split("ExcelWriter for Row") + .nth(1) + .expect("the writer impl must be present"); + assert!( + !writer_half.contains("\"Name\""), + "an alias must never reach a write column" + ); + assert!( + writer_half.contains("\"Nome\""), + "the primary name must reach the write column" + ); + } } diff --git a/rust/excelreader/Cargo.toml b/rust/excelreader/Cargo.toml index 787276a..acede2d 100644 --- a/rust/excelreader/Cargo.toml +++ b/rust/excelreader/Cargo.toml @@ -3,7 +3,7 @@ name = "excelreader" version = "0.0.0" edition = "2021" license = "MIT" -description = "Read Excel/CSV workbooks via ExcelReader's native ABI (open + schema-driven typed parse)." +description = "Read and write Excel/CSV workbooks via ExcelReader's native ABI (schema-driven typed parse and write)." repository = "https://github.com/GabrielMarquezMatte/ExcelReader" readme = "README.md" build = "build.rs" @@ -14,17 +14,23 @@ default = [] # NaiveDate/NaiveTime/NaiveDateTime, so `#[derive(ExcelMapper)]` can populate chrono fields # directly. Off by default: the newtypes alone need no dependency at all. chrono = ["dep:chrono"] +# Converts the native Arrow C Data Interface export into an `arrow::array::RecordBatch`. Off by +# default: arrow-rs is a large dependency and the crate's typed-parse path needs none of it. +arrow = ["dep:arrow"] [dependencies] excelreader-derive = { path = "../excelreader-derive", version = "0.0.0" } # `default-features = false` drops chrono's `oldtime`/`clock` stack: this crate only converts # between integer counts and calendar types, and never reads the system clock or a timezone database. chrono = { version = "0.4", optional = true, default-features = false, features = ["std"] } +arrow = { version = "56", optional = true, default-features = false, features = ["ffi"] } [dev-dependencies] trybuild = "1" criterion = "0.5" calamine = { version = "0.36.1", features = ["chrono"] } +# Comparison baseline for write_bench only — this crate's own write path never calls it. +rust_xlsxwriter = "0.98.2" [[bench]] name = "parse_bench" @@ -34,5 +40,9 @@ harness = false name = "compare_bench" harness = false +[[bench]] +name = "write_bench" +harness = false + [package.metadata.docs.rs] all-features = true diff --git a/rust/excelreader/README.md b/rust/excelreader/README.md index 75d22c9..231ec19 100644 --- a/rust/excelreader/README.md +++ b/rust/excelreader/README.md @@ -1,9 +1,9 @@ # excelreader (Rust) -Read Excel/CSV workbooks via ExcelReader's native ABI: opening a workbook (from a path or memory, -with the full open-options surface), sheet navigation, schema inference, and schema-driven typed -parse. No writing, no Arrow, no row-by-row decode yet - see the root README's Python section for -what those look like. +Read and write Excel/CSV workbooks via ExcelReader's native ABI: opening a workbook (from a path or +memory, with the full open-options surface), sheet navigation, schema inference, schema-driven typed +parse, and schema-driven writing. No Arrow, no row-by-row decode yet - see the root README's Python +section for what those look like. ## Usage @@ -71,6 +71,68 @@ for column in workbook.infer_schema(1, 100)? { `Workbook::open_with` takes an explicit format and `OpenOptions`; `Workbook::open_memory` reads from a byte slice. Note that format sniffing does not detect CSV - pass `XL_FORMAT_CSV` explicitly. +### Arrow export (`arrow` feature) + +```toml +excelreader = { version = "0.0.0", features = ["arrow"] } +``` + +```rust +use excelreader::arrow::parse_arrow; +use excelreader::workbook::Workbook; + +let mut workbook = Workbook::open("book.xlsx")?; +let batch = parse_arrow::(&mut workbook, 1)?; +println!("{} rows x {} columns", batch.num_rows(), batch.num_columns()); +``` + +Off by default — arrow-rs is a large dependency and the typed-parse path needs none of it. + +## Writing + +`#[derive(ExcelMapper)]` generates both halves, so the same struct reads and writes - the field +types in the table above apply unchanged: + +```rust +use excelreader::writer::write_sheet; +use excelreader::XL_FORMAT_XLSX; + +write_sheet("out.xlsx", XL_FORMAT_XLSX, &rows, None)?; +``` + +`Option` fields become an LSB-first validity bitmap: `None` writes a blank cell rather than a +zero. Only the primary `#[excel(name = "...")]` reaches the header - the `alias` list exists to +resolve a header on the way *in*, and the ABI rejects a write column carrying more than one name. + +For buffers that are already columnar, `write_columns` borrows them and copies nothing. The +lifetimes on `Column<'a>` are what turn the ABI's borrow contract into something the compiler +checks: + +```rust +use excelreader::writer::{write_columns, Column, ColumnData}; +use excelreader::XL_FORMAT_XLSX; + +let ids = [1i64, 2, 3]; +let columns = [Column { + name: Some("id"), + data: ColumnData::I64(&ids), + validity: None, +}]; +write_columns("out.xlsx", XL_FORMAT_XLSX, &columns, None)?; +``` + +`validity` is checked against the row count before the call: the ABI takes the bitmap without a +length and reads `(rows + 7) / 8` bytes on trust, so a short slice would be a buffer overrun rather +than a wrong answer. + +`WriteOptions` sets the sheet name, the CSV dialect, and the XLS/XLSB and XLSX/XLSB toggles. +`format_from_path` infers the format from an extension; it returns `XL_FORMAT_AUTO` for anything it +does not recognize, which the write then rejects - a file being created has no signature bytes to +sniff, so there is nothing to fall back on. + +`write_sheet` walks the slice once and appends each field to its own buffer, monomorphized per +field. That transpose is the only copy it makes; `write_columns` pays nothing. + ## Bounds and panics `TableView::get` returns `Option` and is `None` outside `0..len()`. The `column_*` accessors used @@ -92,7 +154,8 @@ through a layout that may have changed. ## Benchmarks Criterion suite in `benches/`. Measured on Windows 10 (22H2), 16 logical CPUs @ 3.39 GHz, -rustc 1.97.1 (Release), Criterion 0.5, 100 samples per benchmark (medians shown). +rustc 1.97.1 (Release), Criterion 0.5, 100 samples per benchmark (medians shown). `write_bench` +takes 20 samples instead — each of its iterations writes a whole 65,535-row file. `benches/parse_bench.rs` - `open`/`parse_sheet`/`infer_schema`, same methodology as the C++ suite: @@ -120,6 +183,28 @@ ExcelReader is ~2.2x faster than calamine for XLSX and ~1.2x faster for XLSB on calamine is a fast, well-optimized reader in its own right, so the gap is real but not the order of magnitude seen against slower libraries. +`benches/write_bench.rs` measures the two write layers and +[rust_xlsxwriter](https://github.com/jmcnamara/rust_xlsxwriter) writing the same 7 columns × 65,535 +rows to `.xlsx`, all three starting from the same in-memory `Vec`: + +| Benchmark | Median | +|---|---:| +| `columns` (`write_columns`, pre-transposed) | 62.1 ms | +| `sheet` (`write_sheet`, from `Vec`) | 67.2 ms | +| `rust_xlsxwriter` (cell-at-a-time) | 336.9 ms | + +`sheet` is the matched-work number — it starts from the same shape `rust_xlsxwriter` is handed and +pays the row-to-column transpose itself — and is ~5.0x faster. `columns` is a ceiling no +cell-at-a-time API can reach, since it is handed buffers that are already columnar; read it only +against `sheet`, as the cost of having row-shaped data in the first place. That cost turns out to be +about 8%: the transpose is nearly free next to producing the file. + +Two caveats, both running against the headline number rather than for it. ExcelReader does slightly +*more* work here: it attaches a number format to the date column so Excel shows a date, and writes a +header row, while the `rust_xlsxwriter` case writes that column as a bare serial number and no +header. And `rust_xlsxwriter` carries formatting and formula support this library does not expose at +all, so its number reflects a different feature set, not only a slower path. + Run locally: ```bash @@ -129,4 +214,4 @@ EXCELREADER_NATIVE_LIB_DIR=/path/to/native/lib/dir cargo bench -p excelreader `EXCELREADER_NATIVE_LIB_DIR` should point at a directory containing a locally-built `ExcelReader.Native.{dll,so,dylib}` - see [Build notes](#build-notes) above. Pass -`--bench parse_bench` or `--bench compare_bench` to run one suite only. +`--bench parse_bench`, `--bench compare_bench` or `--bench write_bench` to run one suite only. diff --git a/rust/excelreader/benches/write_bench.rs b/rust/excelreader/benches/write_bench.rs new file mode 100644 index 0000000..b00dad2 --- /dev/null +++ b/rust/excelreader/benches/write_bench.rs @@ -0,0 +1,123 @@ +//! Write benchmarks over tests/ExcelReader.Benchmarks/Data/65K_Records_Data.xlsx (65,535 data +//! rows, 14 columns), the same fixture the C++, Python and .NET suites use. +//! +//! WORK IS NOT MATCHED across all three groups, and the difference is structural rather than an +//! oversight: +//! +//! * `columns` hands the ABI buffers that are already columnar. Nothing is transposed, nothing +//! is copied. This is the ceiling, and no cell-at-a-time API can be compared to it fairly. +//! * `sheet` starts from a Vec and pays the row-to-column transpose. This is the +//! matched-work sibling: it does the same job rust_xlsxwriter does, from the same starting +//! shape. +//! * `rust_xlsxwriter` writes cell by cell through an API that also owns styling and formula +//! support this library does not expose. +//! +//! Read `sheet` against `rust_xlsxwriter`. Read `columns` only against `sheet`, as the cost of +//! having row-shaped data in the first place. +//! +//! Machine: state the CPU, OS and toolchain version alongside any number published from this file. + +use criterion::{black_box, criterion_group, criterion_main, Criterion}; +use excelreader::workbook::{parse_sheet, ExcelMapper, Workbook}; +use excelreader::writer::{write_columns, write_sheet, Column, OwnedColumn}; +use excelreader::{Date, XL_FORMAT_XLSX}; +use std::path::{Path, PathBuf}; + +const FIXTURE: &str = "../../tests/ExcelReader.Benchmarks/Data/65K_Records_Data.xlsx"; + +#[derive(Default, Clone, ExcelMapper)] +struct Row { + #[excel(name = "Region")] + region: String, + #[excel(name = "Country")] + country: String, + #[excel(name = "Item Type")] + item_type: String, + #[excel(name = "Order Date")] + order_date: Date, + #[excel(name = "Order ID")] + order_id: i64, + #[excel(name = "Units Sold")] + units_sold: i64, + #[excel(name = "Total Revenue")] + total_revenue: f64, +} + +/// Reads the fixture once into row structs. Panics rather than silently benchmarking nothing when +/// the fixture is missing or empty - a suite that measures an empty input is worse than no suite. +fn load_rows() -> Vec { + let path = Path::new(FIXTURE); + assert!( + path.exists(), + "missing benchmark fixture {FIXTURE} - run from rust/excelreader, and check the file is \ + present in the repo" + ); + let mut workbook = Workbook::open(FIXTURE).expect("the fixture must open"); + let table = parse_sheet::(&mut workbook, 1).expect("the fixture must parse"); + let rows: Vec = table.iter().collect(); + assert!(!rows.is_empty(), "the fixture parsed to zero rows"); + rows +} + +fn output_path(name: &str) -> PathBuf { + let mut path = std::env::temp_dir(); + path.push(format!("excelreader-bench-{}-{name}", std::process::id())); + path +} + +fn benchmark_write(criterion: &mut Criterion) { + let rows = load_rows(); + // Transposed once, outside the measured region: the `columns` group exists to measure the + // write, not the transpose the `sheet` group already covers. + let owned: Vec = + ::to_columns(&rows).expect("transpose must succeed"); + let borrowed: Vec> = owned.iter().map(OwnedColumn::as_column).collect(); + + let mut group = criterion.benchmark_group("write_xlsx_65k"); + group.sample_size(20); + + group.bench_function("columns", |b| { + let path = output_path("columns.xlsx"); + let target = path.to_str().expect("temp path must be UTF-8"); + b.iter(|| { + write_columns(target, XL_FORMAT_XLSX, black_box(&borrowed), None) + .expect("write_columns must succeed"); + }); + std::fs::remove_file(&path).ok(); + }); + + group.bench_function("sheet", |b| { + let path = output_path("sheet.xlsx"); + let target = path.to_str().expect("temp path must be UTF-8"); + b.iter(|| { + write_sheet(target, XL_FORMAT_XLSX, black_box(&rows), None) + .expect("write_sheet must succeed"); + }); + std::fs::remove_file(&path).ok(); + }); + + group.bench_function("rust_xlsxwriter", |b| { + let path = output_path("xlsxwriter.xlsx"); + b.iter(|| { + let mut workbook = rust_xlsxwriter::Workbook::new(); + let sheet = workbook.add_worksheet(); + for (index, row) in black_box(&rows).iter().enumerate() { + let r = (index + 1) as u32; + sheet.write_string(r, 0, &row.region).unwrap(); + sheet.write_string(r, 1, &row.country).unwrap(); + sheet.write_string(r, 2, &row.item_type).unwrap(); + sheet.write_number(r, 3, row.order_date.days_since_epoch as f64).unwrap(); + sheet.write_number(r, 4, row.order_id as f64).unwrap(); + sheet.write_number(r, 5, row.units_sold as f64).unwrap(); + sheet.write_number(r, 6, row.total_revenue).unwrap(); + } + workbook.save(&path).unwrap(); + }); + std::fs::remove_file(&path).ok(); + }); + + group.finish(); +} + +criterion_group!(benches, benchmark_write); +criterion_main!(benches); \ No newline at end of file diff --git a/rust/excelreader/build.rs b/rust/excelreader/build.rs index 53fcce2..5ee6264 100644 --- a/rust/excelreader/build.rs +++ b/rust/excelreader/build.rs @@ -18,6 +18,13 @@ const REPO: &str = "GabrielMarquezMatte/ExcelReader"; fn main() { println!("cargo:rerun-if-env-changed=EXCELREADER_NATIVE_LIB_DIR"); + // PHASE1_DEF_EXPORTS below embeds this file's content via include_str! at build-script COMPILE + // time. Emitting any rerun-if-* directive opts this build script out of Cargo's default "rerun + // if any file in the package changed" fallback, so without this the compiled build-script-build + // binary keeps an outdated symbol list baked in after excelreader.def changes - it is not + // recompiled, and only re-*run*, which just regenerates the import lib from the same stale + // constant. + println!("cargo:rerun-if-changed=excelreader.def"); let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap(); diff --git a/rust/excelreader/excelreader.def b/rust/excelreader/excelreader.def index 4140084..074643e 100644 --- a/rust/excelreader/excelreader.def +++ b/rust/excelreader/excelreader.def @@ -12,6 +12,26 @@ EXPORTS xl_is_date1904 xl_parse_typed xl_free_table + xl_parse_arrow + xl_write_typed + xl_write_typed_to_memory + xl_free_buffer xl_infer_schema xl_free_schema + xl_open_write_handle + xl_open_write_handle_to_memory + xl_start_sheet + xl_start_row + xl_write_string + xl_write_int64 + xl_write_float64 + xl_write_bool + xl_write_date + xl_write_time + xl_write_timestamp + xl_write_null + xl_end_row + xl_end_sheet + xl_close_write_handle + xl_write_handle_bytes xl_last_error_ptr diff --git a/rust/excelreader/src/arrow.rs b/rust/excelreader/src/arrow.rs new file mode 100644 index 0000000..736f525 --- /dev/null +++ b/rust/excelreader/src/arrow.rs @@ -0,0 +1,52 @@ +//! Arrow C Data Interface import, behind the `arrow` cargo feature. +//! +//! The native side already produces one top-level Arrow struct array whose children are the +//! columns, so the only work here is handing arrow-rs the FFI pair and letting it take ownership. + +use std::ffi::c_void; + +use arrow::array::{RecordBatch, StructArray}; +use arrow::ffi::{from_ffi, FFI_ArrowArray, FFI_ArrowSchema}; + +// `check` is `pub(crate)` in workbook.rs (not error.rs) - visible from this sibling module because +// pub(crate) means "crate-wide", not "same file". +use crate::workbook::{build_specs, check, ExcelMapper, Workbook}; +use crate::{Error, XL_ERROR}; + +/// Schema-driven parse of the current sheet into an Arrow [`RecordBatch`], using the same +/// `#[derive(ExcelMapper)]` mapping as [`crate::workbook::parse_sheet`]. +/// +/// `header_row` has the same meaning as in `parse_sheet` (0 = no header). Takes `&mut Workbook` +/// because the parse consumes the workbook's shared row cursor. +pub fn parse_arrow( + workbook: &mut Workbook, + header_row: i32, +) -> Result { + let arena = build_specs::(); + + // Both start with a null `release`, which the Arrow spec defines as "owns nothing" - so if the + // call below fails and leaves them untouched, dropping them is a no-op and nothing leaks. + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + + check(unsafe { + crate::xl_parse_arrow( + workbook.handle(), + arena.specs.as_ptr(), + arena.specs.len() as i32, + header_row, + &mut array as *mut FFI_ArrowArray as *mut c_void, + &mut schema as *mut FFI_ArrowSchema as *mut c_void, + ) + })?; + + // from_ffi consumes `array` by value: arrow-rs now owns it and will invoke its release callback + // when the resulting ArrayData is dropped. `schema` stays owned here and releases on drop at + // the end of this function, which is correct - the two are released independently. + let data = unsafe { from_ffi(array, &schema) }.map_err(|e| Error { + code: XL_ERROR, + message: format!("importing the native Arrow array failed: {e}"), + })?; + + Ok(RecordBatch::from(StructArray::from(data))) +} diff --git a/rust/excelreader/src/lib.rs b/rust/excelreader/src/lib.rs index 27824c8..7be2e19 100644 --- a/rust/excelreader/src/lib.rs +++ b/rust/excelreader/src/lib.rs @@ -6,9 +6,14 @@ mod error; mod options; mod temporal; pub mod workbook; +pub mod writer; +pub mod writer_handle; + +#[cfg(feature = "arrow")] +pub mod arrow; pub use error::Error; -pub use options::OpenOptions; +pub use options::{OpenOptions, WriteOptions}; pub use temporal::{Date, Time, Timestamp}; use std::os::raw::{c_int, c_void}; @@ -22,7 +27,7 @@ pub const XL_ERROR: i32 = -5; /// ABI revision this crate is compiled against. `Workbook::open` refuses to proceed when the loaded /// library's `xl_abi_version()` disagrees - see `workbook::check_abi_version`. -pub const XL_ABI_VERSION: i32 = 2; +pub const XL_ABI_VERSION: i32 = 3; pub const XL_T_STRING: i32 = 0; pub const XL_T_I64: i32 = 1; @@ -74,6 +79,22 @@ pub struct XlOpenOptions { pub intern_strings: i32, } +/// Mirrors `xl_write_options`. Field ORDER is the C struct's, not a tidied-up version of it: with +/// `repr(C)` the 4 bytes of padding after `sheet_name_len` land exactly where a C compiler puts +/// them, giving the 32-byte x64 layout `tests/ExcelReader.NativeSmoke/smoke.c` static-asserts. +/// Build one through [`WriteOptions`] rather than by hand. +#[repr(C)] +#[derive(Clone, Copy, Debug)] +pub struct XlWriteOptions { + pub struct_size: i32, + pub sheet_name_len: i32, + pub sheet_name: *const u8, + pub csv_delimiter: i32, + pub csv_quote: i32, + pub date1904: i32, + pub use_shared_strings: i32, +} + #[repr(C)] pub struct XlColumnSpec { pub names: *const *const u8, @@ -108,6 +129,22 @@ pub struct XlInferredSchema { pub column_count: i32, } +/// Mirrors `xl_buffer`: an owned block of native memory returned by `xl_write_typed_to_memory` or +/// `xl_write_handle_bytes`. Released with `xl_free_buffer` - see `writer::buffer_to_vec`, the one +/// place this crate touches the raw struct directly. +#[repr(C)] +pub struct XlBuffer { + pub data: *mut u8, + pub len: i64, +} + +/// Opaque streaming writer handle - never dereferenced by Rust, only passed back to `xl_*` +/// functions. See [`writer_handle::WriterHandle`] for the safe wrapper. +#[repr(C)] +pub struct XlWriterHandle { + _private: [u8; 0], +} + extern "C" { pub fn xl_abi_version() -> c_int; @@ -158,6 +195,23 @@ extern "C" { out_table: *mut XlTable, ) -> c_int; + /// Same schema-driven parse as `xl_parse_typed`, exported as one top-level Arrow struct + /// array/schema. `out_array`/`out_schema` are `struct ArrowArray*`/`struct ArrowSchema*` from + /// the Arrow C Data Interface, typed here as `c_void` because arrow-rs's own `#[repr(C)]` + /// `FFI_ArrowArray`/`FFI_ArrowSchema` are ABI-identical to them - redeclaring the spec structs + /// would be a second source of truth for a fixed, versioned ABI. + /// + /// On `XL_OK` the caller owns both and releases each through its OWN `release` callback, never + /// through `xl_free_table`. On any other status both outputs are left untouched. + pub fn xl_parse_arrow( + handle: *mut XlWorkbook, + specs: *const XlColumnSpec, + spec_count: i32, + header_row: i32, + out_array: *mut c_void, + out_schema: *mut c_void, + ) -> c_int; + pub fn xl_free_table(table: *mut XlTable); pub fn xl_infer_schema( @@ -168,6 +222,67 @@ extern "C" { ) -> c_int; pub fn xl_free_schema(schema: *mut XlInferredSchema); + pub fn xl_write_typed( + path: *const u8, + path_len: i32, + format: i32, + specs: *const XlColumnSpec, + table: *const XlTable, + options: *const XlWriteOptions, + ) -> c_int; + + /// Same as `xl_write_typed`, except the result is returned as `out_buffer` instead of being + /// written to a path. Only read `*out_buffer` when the call returns `XL_OK`; on failure it is + /// zeroed, and `xl_free_buffer` on a zeroed buffer is a no-op. + pub fn xl_write_typed_to_memory( + format: i32, + specs: *const XlColumnSpec, + table: *const XlTable, + options: *const XlWriteOptions, + out_buffer: *mut XlBuffer, + ) -> c_int; + + /// Releases a buffer returned by `xl_write_typed_to_memory` or `xl_write_handle_bytes` and + /// resets it to zero. Safe on a zeroed value. + pub fn xl_free_buffer(buffer: *mut XlBuffer); + + // ---- Streaming writer handle: see writer_handle::WriterHandle for the call-order contract. ---- + + pub fn xl_open_write_handle( + path: *const u8, + path_len: i32, + format: i32, + options: *const XlWriteOptions, + out_handle: *mut *mut XlWriterHandle, + ) -> c_int; + + pub fn xl_open_write_handle_to_memory( + format: i32, + options: *const XlWriteOptions, + out_handle: *mut *mut XlWriterHandle, + ) -> c_int; + + pub fn xl_start_sheet(handle: *mut XlWriterHandle, name: *const u8, name_len: i32) -> c_int; + pub fn xl_start_row(handle: *mut XlWriterHandle) -> c_int; + + pub fn xl_write_string(handle: *mut XlWriterHandle, value: *const u8, value_len: i32) -> c_int; + pub fn xl_write_int64(handle: *mut XlWriterHandle, value: i64) -> c_int; + pub fn xl_write_float64(handle: *mut XlWriterHandle, value: f64) -> c_int; + pub fn xl_write_bool(handle: *mut XlWriterHandle, value: i32) -> c_int; + pub fn xl_write_date(handle: *mut XlWriterHandle, days_since_epoch: i32) -> c_int; + pub fn xl_write_time(handle: *mut XlWriterHandle, micros_since_midnight: i64) -> c_int; + pub fn xl_write_timestamp(handle: *mut XlWriterHandle, micros_since_epoch: i64) -> c_int; + pub fn xl_write_null(handle: *mut XlWriterHandle, r#type: i32) -> c_int; + + pub fn xl_end_row(handle: *mut XlWriterHandle) -> c_int; + pub fn xl_end_sheet(handle: *mut XlWriterHandle) -> c_int; + pub fn xl_close_write_handle(handle: *mut XlWriterHandle) -> c_int; + + /// Reads back everything written so far to a handle opened by `xl_open_write_handle_to_memory`. + /// `XL_INVALID_ARGUMENT` for one from `xl_open_write_handle`. Only read `*out_buffer` when the + /// call returns `XL_OK`; on failure it is zeroed, and `xl_free_buffer` on a zeroed buffer is a + /// no-op. + pub fn xl_write_handle_bytes(handle: *mut XlWriterHandle, out_buffer: *mut XlBuffer) -> c_int; pub fn xl_last_error_ptr(out_len: *mut i32) -> *const u8; } diff --git a/rust/excelreader/src/options.rs b/rust/excelreader/src/options.rs index 4377929..902d9c3 100644 --- a/rust/excelreader/src/options.rs +++ b/rust/excelreader/src/options.rs @@ -9,7 +9,7 @@ //! reports a rejection through `xl_last_error`, so checking them here too would give those bounds a //! second place to drift from. -use crate::{XlOpenOptions, XL_OPT_DEFAULT, XL_OPT_FALSE, XL_OPT_TRUE}; +use crate::{XL_OPT_DEFAULT, XL_OPT_FALSE, XL_OPT_TRUE, XlOpenOptions, XlWriteOptions}; /// Options for [`Workbook::open_with`](crate::workbook::Workbook::open_with) and /// [`Workbook::open_memory`](crate::workbook::Workbook::open_memory). Every field is `None` by @@ -126,6 +126,88 @@ fn opt_number(value: Option) -> T { value.unwrap_or_default() } +/// Options for [`write_columns`](crate::writer::write_columns) and +/// [`write_sheet`](crate::writer::write_sheet). Every field is `None` by default, meaning "use the +/// library default"; set only the ones you want to override. +/// +/// ```no_run +/// use excelreader::WriteOptions; +/// +/// let options = WriteOptions::new().sheet_name("Dados").use_shared_strings(true); +/// ``` +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct WriteOptions { + /// Sheet name. `None` = `"Sheet1"`. Ignored for CSV. Excel's rules (1-31 characters, none of + /// `: \ / ? * [ ]`) are enforced natively and reported through `xl_last_error`. + pub sheet_name: Option, + + // CSV only; ignored for every other format. Byte value 1-255. + pub csv_delimiter: Option, + pub csv_quote: Option, + + /// XLS/XLSB only; ignored for XLSX and CSV. Library default is `false`. + pub date1904: Option, + /// XLSX/XLSB only; ignored for XLS and CSV. Shrinks files with many repeated strings, at the + /// cost of a string table. Library default is `false`. + pub use_shared_strings: Option, +} + +impl WriteOptions { + /// Every field unset - identical to passing no options at all. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Written by hand rather than generated by `setters!`: this is the one field that is not a + /// plain `Copy` scalar. + #[must_use] + pub fn sheet_name(mut self, value: impl Into) -> Self { + self.sheet_name = Some(value.into()); + self + } + + setters! { + /// CSV only. Byte value 1-255; the library default is `,`. + csv_delimiter: i32, + /// CSV only. Byte value 1-255; the library default is `"`. + csv_quote: i32, + /// XLS/XLSB only. Library default is `false`. + date1904: bool, + /// XLSX/XLSB only. Library default is `false`. + use_shared_strings: bool, + } + + /// Lowers this into the raw ABI struct, with `struct_size` filled in. + /// + /// `pub(crate)`, unlike [`OpenOptions::to_raw`], which is public: the struct this returns holds + /// a raw pointer INTO `self.sheet_name`, so handing it to a caller who can outlive `self` is a + /// use-after-free waiting to happen. `write_columns` takes `Option<&WriteOptions>` and does the + /// lowering itself, where the borrow provably covers the FFI call. + pub(crate) fn to_raw(&self) -> XlWriteOptions { + XlWriteOptions { + struct_size: std::mem::size_of::() as i32, + sheet_name_len: self.sheet_name.as_ref().map_or(0, |n| n.len() as i32), + sheet_name: self + .sheet_name + .as_ref() + .map_or(std::ptr::null(), |n| n.as_ptr()), + csv_delimiter: opt_number(self.csv_delimiter), + csv_quote: opt_number(self.csv_quote), + date1904: opt_state(self.date1904), + use_shared_strings: opt_state(self.use_shared_strings), + } + } +} + +/// The pointer an FFI options argument takes: a borrowed raw struct, or NULL for "every library +/// default". `raw` must outlive the call the returned pointer is handed to - taking it by reference +/// rather than by value is what makes the compiler check that. +#[inline] +pub(crate) fn ptr_or_null(raw: &Option) -> *const T { + raw.as_ref().map_or(std::ptr::null(), |value| value as *const T) +} + #[cfg(test)] mod tests { use super::*; @@ -163,4 +245,40 @@ mod tests { assert_eq!(raw.max_zip_entries, 1024); assert_eq!(raw.max_shared_string_bytes, 1 << 20); } + + #[test] + fn write_options_lower_to_the_abi_defaults() { + let raw = WriteOptions::new().to_raw(); + assert_eq!( + raw.struct_size, + std::mem::size_of::() as i32 + ); + assert!(raw.sheet_name.is_null(), "an unset sheet name means Sheet1"); + assert_eq!(raw.sheet_name_len, 0); + assert_eq!(raw.csv_delimiter, 0); + assert_eq!(raw.date1904, XL_OPT_DEFAULT); + } + + #[test] + fn write_options_lower_every_set_field() { + let options = WriteOptions::new() + .sheet_name("Dados") + .csv_delimiter(b';' as i32) + .date1904(false) + .use_shared_strings(true); + let raw = options.to_raw(); + assert_eq!(raw.sheet_name_len, 5); + assert!(!raw.sheet_name.is_null()); + assert_eq!(raw.csv_delimiter, b';' as i32); + assert_eq!(raw.date1904, XL_OPT_FALSE); + assert_eq!(raw.use_shared_strings, XL_OPT_TRUE); + } + + /// The C struct's x64 layout is pinned by XL_STATIC_ASSERT in + /// tests/ExcelReader.NativeSmoke/smoke.c (lines 66-73). A repr(C) struct that disagrees would + /// hand the native side garbage where it expects a length and a pointer. + #[test] + fn write_options_match_the_c_struct_size() { + assert_eq!(std::mem::size_of::(), 32); + } } diff --git a/rust/excelreader/src/workbook.rs b/rust/excelreader/src/workbook.rs index 1712b13..6146d48 100644 --- a/rust/excelreader/src/workbook.rs +++ b/rust/excelreader/src/workbook.rs @@ -1,11 +1,11 @@ use crate::{ Date, Error, OpenOptions, Time, Timestamp, XlColumn, XlColumnSpec, XlInferredSchema, - XlOpenOptions, XlTable, XlWorkbook, XL_BUFFER_TOO_SMALL, XL_ERROR, XL_FORMAT_AUTO, XL_OK, + XlTable, XlWorkbook, XL_BUFFER_TOO_SMALL, XL_ERROR, XL_FORMAT_AUTO, XL_OK, XL_T_BOOL, XL_T_DATE, XL_T_F64, XL_T_I64, XL_T_STRING, XL_T_TIME, XL_T_TIMESTAMP, }; use std::marker::PhantomData; -fn last_error(code: i32) -> Error { +pub(crate) fn last_error(code: i32) -> Error { unsafe { let mut len: i32 = 0; let ptr = crate::xl_last_error_ptr(&mut len); @@ -19,7 +19,7 @@ fn last_error(code: i32) -> Error { } } -fn check(code: i32) -> Result<(), Error> { +pub(crate) fn check(code: i32) -> Result<(), Error> { if code == XL_OK { Ok(()) } else { @@ -27,6 +27,24 @@ fn check(code: i32) -> Result<(), Error> { } } +/// Copies a native `XlBuffer` into an owned `Vec` and releases the native allocation via +/// `xl_free_buffer` - shared by `writer::write_columns_to_memory`/`write_sheet_to_memory` and +/// `writer_handle::WriterHandle::bytes`, the two places `xl_write_typed_to_memory`/ +/// `xl_write_handle_bytes` hand back an owned buffer. `buffer.data` may be null (an empty result), +/// which `from_raw_parts` cannot take - `slice::from_raw_parts` requires a non-null, well-aligned +/// pointer even for a zero-length slice. +pub(crate) fn buffer_to_vec(mut buffer: crate::XlBuffer) -> Vec { + let bytes = if buffer.data.is_null() || buffer.len <= 0 { + Vec::new() + } else { + unsafe { std::slice::from_raw_parts(buffer.data, buffer.len as usize).to_vec() } + }; + unsafe { + crate::xl_free_buffer(&mut buffer); + } + bytes +} + /// Verifies the loaded shared library speaks the ABI revision this crate was compiled against. /// /// The native binary is resolved at build time from a GitHub release asset (or from @@ -37,7 +55,7 @@ fn check(code: i32) -> Result<(), Error> { /// /// The result is cached: it cannot change for the lifetime of the process, and every /// `Workbook::open` would otherwise pay an FFI call for it. -fn check_abi_version() -> Result<(), Error> { +pub(crate) fn check_abi_version() -> Result<(), Error> { use std::sync::OnceLock; static CHECKED: OnceLock> = OnceLock::new(); @@ -83,9 +101,7 @@ impl Workbook { ) -> Result { check_abi_version()?; let raw = options.map(OpenOptions::to_raw); - let raw_ptr = raw - .as_ref() - .map_or(std::ptr::null(), |o| o as *const XlOpenOptions); + let raw_ptr = crate::options::ptr_or_null(&raw); let mut handle: *mut XlWorkbook = std::ptr::null_mut(); // `raw` outlives the call below, and the native side copies the path before returning. let status = unsafe { @@ -110,9 +126,7 @@ impl Workbook { ) -> Result { check_abi_version()?; let raw = options.map(OpenOptions::to_raw); - let raw_ptr = raw - .as_ref() - .map_or(std::ptr::null(), |o| o as *const XlOpenOptions); + let raw_ptr = crate::options::ptr_or_null(&raw); let mut handle: *mut XlWorkbook = std::ptr::null_mut(); let status = unsafe { crate::xl_open_memory_ex( @@ -197,6 +211,13 @@ impl Workbook { Ok(columns) } + /// The raw handle, for sibling modules (e.g. `arrow::parse_arrow`) that need to call an + /// `xl_*` function this struct has no wrapper for yet. Not part of the crate's public surface - + /// `pub(crate)`, not `pub`. + pub(crate) fn handle(&self) -> *mut XlWorkbook { + self.handle + } + /// Shared two-pass buffer dance for the `xl_*` functions that write a UTF-8 name into a caller /// buffer and report the required capacity through `XL_BUFFER_TOO_SMALL`. fn fill_string( @@ -205,7 +226,7 @@ impl Workbook { ) -> Result { // One sized attempt first: Excel caps sheet names at 31 characters, so 128 bytes clears // even the 4-byte-per-character worst case and the retry never runs in practice. - let mut buffer = vec![0u8; 128]; + let mut buffer = [0u8; 128]; let mut len: i32 = 0; let mut status = call( self.handle, @@ -213,18 +234,26 @@ impl Workbook { buffer.len() as i32, &mut len, ); - if status == XL_BUFFER_TOO_SMALL { - buffer = vec![0u8; len.max(0) as usize]; - status = call( - self.handle, - buffer.as_mut_ptr(), - buffer.len() as i32, - &mut len, - ); + if status != XL_BUFFER_TOO_SMALL { + check(status)?; + let buffer_slice = &buffer[..len.max(0) as usize]; + return str::from_utf8(buffer_slice) + .map(|s| s.to_string()) + .map_err(|e| Error { + code: XL_ERROR, + message: format!("native library returned a non-UTF-8 name: {e}"), + }); } + let mut vec_buffer = vec![0u8; len.max(0) as usize]; + status = call( + self.handle, + vec_buffer.as_mut_ptr(), + vec_buffer.len() as i32, + &mut len, + ); check(status)?; - buffer.truncate(len.max(0) as usize); - String::from_utf8(buffer).map_err(|e| Error { + vec_buffer.truncate(len.max(0) as usize); + String::from_utf8(vec_buffer).map_err(|e| Error { code: XL_ERROR, message: format!("native library returned a non-UTF-8 name: {e}"), }) @@ -500,13 +529,23 @@ impl Iterator for TableViewIter<'_, T> { impl ExactSizeIterator for TableViewIter<'_, T> {} -/// Schema-driven columnar parse of the current sheet, matching C++'s `xl::parse_sheet`. +/// Keeps the per-column name pointer/length vectors alive for as long as the `XlColumnSpec` array +/// that points into them. The two `_name_*` fields are never read - dropping them early would +/// leave `specs` holding dangling pointers, which is the whole reason they are stored here. /// -/// Takes `&mut Workbook` because the parse consumes the workbook's shared row cursor. -pub fn parse_sheet( - workbook: &mut Workbook, - header_row: i32, -) -> Result, Error> { +/// Also carries the `T::bindings()` this arena was built from, so callers that need both the flat +/// spec array (for the FFI call) and the typed bindings (for result-column lookups) can get both +/// from a single `T::bindings()` call instead of computing it twice. +pub(crate) struct SpecArena { + pub(crate) specs: Vec, + pub(crate) bindings: Vec>, + _name_ptrs: Vec>, + _name_lens: Vec>, +} + +/// Lowers `T`'s ExcelMapper bindings into the flat `xl_column_spec` array both `xl_parse_typed` and +/// `xl_parse_arrow` take - their column-spec input is identical. +pub(crate) fn build_specs() -> SpecArena { let bindings = T::bindings(); let name_ptrs: Vec> = bindings .iter() @@ -528,6 +567,23 @@ pub fn parse_sheet( nullable: 1, }) .collect(); + SpecArena { + specs, + bindings, + _name_ptrs: name_ptrs, + _name_lens: name_lens, + } +} + +/// Schema-driven columnar parse of the current sheet, matching C++'s `xl::parse_sheet`. +/// +/// Takes `&mut Workbook` because the parse consumes the workbook's shared row cursor. +pub fn parse_sheet( + workbook: &mut Workbook, + header_row: i32, +) -> Result, Error> { + let arena = build_specs::(); + let bindings = arena.bindings; let mut table = XlTable { column_count: 0, row_count: 0, @@ -536,8 +592,8 @@ pub fn parse_sheet( unsafe { let status = crate::xl_parse_typed( workbook.handle, - specs.as_ptr(), - specs.len() as i32, + arena.specs.as_ptr(), + arena.specs.len() as i32, header_row, &mut table, ); diff --git a/rust/excelreader/src/writer.rs b/rust/excelreader/src/writer.rs new file mode 100644 index 0000000..3e32d39 --- /dev/null +++ b/rust/excelreader/src/writer.rs @@ -0,0 +1,558 @@ +//! Writing a columnar table to a workbook file, through the ABI's single `xl_write_typed` export. +//! +//! Two layers sit here. [`write_columns`] takes buffers the caller already owns and hands them +//! straight to the ABI - the borrow is the whole point, and the lifetimes on [`Column`] are what +//! turn the ABI's prose contract ("every buffer is borrowed for the duration of the call") into +//! something the compiler enforces. [`write_sheet`] sits on top for callers holding a slice of +//! structs, and pays exactly one transpose to get them into columns. + +use crate::workbook::{check, check_abi_version}; +use crate::{ + Error, WriteOptions, XlColumn, XlColumnSpec, XlTable, XL_FORMAT_AUTO, + XL_FORMAT_CSV, XL_FORMAT_XLS, XL_FORMAT_XLSB, XL_FORMAT_XLSX, XL_INVALID_ARGUMENT, XL_T_BOOL, + XL_T_DATE, XL_T_F64, XL_T_I64, XL_T_STRING, XL_T_TIME, XL_T_TIMESTAMP, +}; +use std::os::raw::c_void; + +/// One column's buffers, borrowed from the caller. Each variant's slice is that column type's +/// exact wire layout (see `excelreader.h`), so nothing is converted on the way out. +pub enum ColumnData<'a> { + /// `offsets` has `rows + 1` entries into `data`, which is every row's UTF-8 bytes + /// concatenated. Unlike a table returned by the reader, `data` need not be interior to + /// `offsets` here. + Str { + offsets: &'a [i32], + data: &'a [u8], + }, + I64(&'a [i64]), + F64(&'a [f64]), + /// One byte per row, 0 or 1 - NOT a bit-packed bitmap. + Bool(&'a [u8]), + /// Days since 1970-01-01. + Date(&'a [i32]), + /// Microseconds since midnight. + Time(&'a [i64]), + /// Microseconds since 1970-01-01T00:00:00Z. + Timestamp(&'a [i64]), +} + +impl ColumnData<'_> { + #[must_use] + pub fn len(&self) -> i64 { + match self { + // An offsets array of n + 1 entries describes n rows; an empty one describes none. + ColumnData::Str { offsets, .. } => (offsets.len().max(1) - 1) as i64, + ColumnData::I64(values) | ColumnData::Time(values) | ColumnData::Timestamp(values) => { + values.len() as i64 + } + ColumnData::F64(values) => values.len() as i64, + ColumnData::Bool(values) => values.len() as i64, + ColumnData::Date(values) => values.len() as i64, + } + } + + #[must_use] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// The `XL_T_*` tag this variant writes as. + #[must_use] + pub fn xl_type(&self) -> i32 { + match self { + ColumnData::Str { .. } => XL_T_STRING, + ColumnData::I64(_) => XL_T_I64, + ColumnData::F64(_) => XL_T_F64, + ColumnData::Bool(_) => XL_T_BOOL, + ColumnData::Date(_) => XL_T_DATE, + ColumnData::Time(_) => XL_T_TIME, + ColumnData::Timestamp(_) => XL_T_TIMESTAMP, + } + } + + fn pointers(&self) -> (*const c_void, *const u8, i64) { + match self { + ColumnData::Str { offsets, data } => ( + offsets.as_ptr().cast::(), + data.as_ptr(), + data.len() as i64, + ), + ColumnData::I64(v) | ColumnData::Time(v) | ColumnData::Timestamp(v) => { + (v.as_ptr().cast::(), std::ptr::null(), 0) + } + ColumnData::F64(v) => (v.as_ptr().cast::(), std::ptr::null(), 0), + ColumnData::Bool(v) => (v.as_ptr().cast::(), std::ptr::null(), 0), + ColumnData::Date(v) => (v.as_ptr().cast::(), std::ptr::null(), 0), + } + } +} + +/// One input column: a name, its buffers, and an optional validity bitmap. +pub struct Column<'a> { + /// The header text. `None` in EVERY column means no header row is written; mixing `Some` and + /// `None` across the set is an error, not a partial header. + pub name: Option<&'a str>, + pub data: ColumnData<'a>, + /// LSB-first bitmap, bit `r` set = row `r` is valid. `None` = the column has no nulls. + pub validity: Option<&'a [u8]>, +} + +impl Column<'_> { + fn to_raw(&self) -> XlColumn { + let (values, data, data_len) = self.data.pointers(); + XlColumn { + r#type: self.data.xl_type(), + length: self.data.len(), + values, + validity: self.validity.map_or(std::ptr::null(), <[u8]>::as_ptr), + data, + data_len, + } + } +} + +/// Infers an `XL_FORMAT_*` from a path's extension, case-insensitively. Returns +/// [`XL_FORMAT_AUTO`] when the extension is absent or unrecognized - and since `xl_write_typed` +/// rejects `AUTO` with a message of its own, an unrecognized path fails the write rather than +/// silently picking a format. +#[must_use] +pub fn format_from_path(path: &str) -> i32 { + let name = path.rsplit(['/', '\\']).next().unwrap_or(path); + let Some((_, extension)) = name.rsplit_once('.') else { + return XL_FORMAT_AUTO; + }; + match extension.to_ascii_lowercase().as_str() { + "xlsx" => XL_FORMAT_XLSX, + "xlsb" => XL_FORMAT_XLSB, + "xls" => XL_FORMAT_XLS, + "csv" => XL_FORMAT_CSV, + _ => XL_FORMAT_AUTO, + } +} + +fn invalid(message: String) -> Error { + Error { + code: XL_INVALID_ARGUMENT, + message, + } +} + +/// Returns the row count every column agreed on, or the first problem found. +/// +/// Runs to completion before anything reaches the native side. The bitmap length check is the one +/// that must live here and cannot be delegated: `xl_write_typed` takes `validity` without a length +/// and reads `(rows + 7) / 8` bytes on trust, so a short slice is a buffer overrun the ABI has no +/// way to catch. +fn validate(columns: &[Column<'_>]) -> Result { + let Some(first) = columns.first() else { + return Err(invalid( + "write_columns needs at least one column.".to_string(), + )); + }; + let rows = first.data.len(); + let has_header = first.name.is_some(); + + for (index, column) in columns.iter().enumerate() { + if column.data.len() != rows { + return Err(invalid(format!( + "every column must have the same length; column 0 has {rows} rows but column \ + {index} has {}", + column.data.len() + ))); + } + if column.name.is_some() != has_header { + return Err(invalid(format!( + "every column must have a name, or none may - xl_write_typed cannot write a \ + partial header row (column {index})" + ))); + } + if let Some(bitmap) = column.validity { + let needed = (rows as usize).div_ceil(8); + if bitmap.len() < needed { + return Err(invalid(format!( + "the validity bitmap is {} bytes, but {rows} rows need {needed} (column \ + {index})", + bitmap.len() + ))); + } + } + if let ColumnData::Str { offsets, data } = &column.data { + if data.len() > i32::MAX as usize { + return Err(invalid(format!( + "the string blob is larger than 2 GiB, which int32 offsets cannot address \ + (column {index})" + ))); + } + if offsets.len() as i64 != rows + 1 { + return Err(invalid(format!( + "a string column needs {} offsets for {rows} rows; column {index} has {}", + rows + 1, + offsets.len() + ))); + } + } + } + Ok(rows) +} + +/// Writes `columns` to `path` as a single sheet, then closes the file. One-shot: no writer handle +/// exists before or after, and every buffer reachable from `columns` and `options` is borrowed for +/// the duration of the call and never freed by the native library. +/// +/// `format` must be one of `XL_FORMAT_XLS`/`XLSX`/`XLSB`/`CSV`. [`XL_FORMAT_AUTO`] is an error: a +/// file being created has no signature bytes to sniff. Use [`format_from_path`] to infer one. +/// +/// On failure the destination may exist and be incomplete; cleaning it up is the caller's. +pub fn write_columns( + path: &str, + format: i32, + columns: &[Column<'_>], + options: Option<&WriteOptions>, +) -> Result<(), Error> { + check_abi_version()?; + let row_count = validate(columns)?; + + // Three parallel arrays that must outlive the call: each spec's `names` points at one slot of + // `names`, and its `name_lens` at one slot of `name_lens`. A temporary would dangle. + let names: Vec<*const u8> = columns + .iter() + .map(|c| c.name.map_or(std::ptr::null(), str::as_ptr)) + .collect(); + let name_lens: Vec = columns + .iter() + .map(|c| c.name.map_or(0, |n| n.len() as i32)) + .collect(); + let specs: Vec = columns + .iter() + .enumerate() + .map(|(index, column)| XlColumnSpec { + names: &names[index], + name_lens: &name_lens[index], + // Exactly one name per write spec, or none: the ABI rejects a spec carrying an alias + // list, which only exists to resolve a header on the way IN. + name_count: i32::from(column.name.is_some()), + index: 0, + r#type: column.data.xl_type(), + nullable: 0, + }) + .collect(); + let raw_columns: Vec = columns.iter().map(Column::to_raw).collect(); + + let table = XlTable { + column_count: columns.len() as i32, + row_count, + // `columns` is `*mut` in the C struct only because the reader fills one in; the writer + // takes the table as `const` and never writes through it. + columns: raw_columns.as_ptr().cast_mut(), + }; + // Lowered here rather than by the caller: the raw struct holds a pointer into + // `options.sheet_name`, and this borrow provably covers the FFI call below. + let raw_options = options.map(WriteOptions::to_raw); + let options_ptr = crate::options::ptr_or_null(&raw_options); + + let status = unsafe { + crate::xl_write_typed( + path.as_ptr(), + path.len() as i32, + format, + specs.as_ptr(), + &table, + options_ptr, + ) + }; + check(status) +} + +/// In-memory equivalent of [`write_columns`]: same validation and column lowering, but the +/// workbook is built in memory and returned as bytes instead of being written to a path - so, +/// unlike [`write_columns`], there is no path to infer a format from and `format` is always +/// required. +/// +/// # Errors +/// Anything [`write_columns`] reports. +pub fn write_columns_to_memory( + format: i32, + columns: &[Column<'_>], + options: Option<&WriteOptions>, +) -> Result, Error> { + check_abi_version()?; + let row_count = validate(columns)?; + + let names: Vec<*const u8> = columns + .iter() + .map(|c| c.name.map_or(std::ptr::null(), str::as_ptr)) + .collect(); + let name_lens: Vec = columns + .iter() + .map(|c| c.name.map_or(0, |n| n.len() as i32)) + .collect(); + let specs: Vec = columns + .iter() + .enumerate() + .map(|(index, column)| XlColumnSpec { + names: &names[index], + name_lens: &name_lens[index], + name_count: i32::from(column.name.is_some()), + index: 0, + r#type: column.data.xl_type(), + nullable: 0, + }) + .collect(); + let raw_columns: Vec = columns.iter().map(Column::to_raw).collect(); + + let table = XlTable { + column_count: columns.len() as i32, + row_count, + columns: raw_columns.as_ptr().cast_mut(), + }; + let raw_options = options.map(WriteOptions::to_raw); + let options_ptr = crate::options::ptr_or_null(&raw_options); + + let mut buffer = crate::XlBuffer { + data: std::ptr::null_mut(), + len: 0, + }; + let status = unsafe { + crate::xl_write_typed_to_memory(format, specs.as_ptr(), &table, options_ptr, &mut buffer) + }; + check(status)?; + Ok(crate::workbook::buffer_to_vec(buffer)) +} + +/// The owning twin of [`ColumnData`], produced by [`ExcelWriter::to_columns`]. A transposed range +/// of structs has to own its columns somewhere; this is that somewhere. +pub enum OwnedColumnData { + Str { offsets: Vec, data: Vec }, + I64(Vec), + F64(Vec), + Bool(Vec), + Date(Vec), + Time(Vec), + Timestamp(Vec), +} + +/// One owned column. `name` is `&'static str` rather than `String` because it always comes from a +/// literal in a `#[excel(name = "...")]` attribute - keeping it borrowed means transposing a +/// million rows allocates nothing for names. +pub struct OwnedColumn { + pub name: Option<&'static str>, + pub data: OwnedColumnData, + /// LSB-first bitmap. `None` = the column has no nulls. + pub validity: Option>, +} + +impl OwnedColumn { + /// Borrows this column in the shape [`write_columns`] takes. + #[must_use] + pub fn as_column(&self) -> Column<'_> { + let data = match &self.data { + OwnedColumnData::Str { offsets, data } => ColumnData::Str { offsets, data }, + OwnedColumnData::I64(v) => ColumnData::I64(v), + OwnedColumnData::F64(v) => ColumnData::F64(v), + OwnedColumnData::Bool(v) => ColumnData::Bool(v), + OwnedColumnData::Date(v) => ColumnData::Date(v), + OwnedColumnData::Time(v) => ColumnData::Time(v), + OwnedColumnData::Timestamp(v) => ColumnData::Timestamp(v), + }; + Column { + name: self.name, + data, + validity: self.validity.as_deref(), + } + } +} + +/// Whether appending `added` bytes to a blob already `current` bytes long would push the next +/// offset past what an `int32` can hold. Split out of `push_str` so the arithmetic is testable +/// without materializing a 2 GiB buffer. +fn offset_ceiling_exceeded(current: usize, added: usize) -> bool { + current.saturating_add(added) > i32::MAX as usize +} + +/// Appends one string to a column's offsets/data pair. +/// +/// This is where the `int32` offset overflow is caught, and it has to be caught HERE rather than +/// after the fact: once `data` has grown past `i32::MAX` the offset that would record it has +/// already wrapped, and a wrapped offset is indistinguishable from a real one. +/// +/// # Errors +/// When appending `value` would push the blob past `i32::MAX` bytes. +pub fn push_str(offsets: &mut Vec, data: &mut Vec, value: &str) -> Result<(), Error> { + if offset_ceiling_exceeded(data.len(), value.len()) { + return Err(invalid( + "a string column exceeds 2 GiB, which int32 offsets cannot address.".to_string(), + )); + } + data.extend_from_slice(value.as_bytes()); + offsets.push(data.len() as i32); + Ok(()) +} + +/// Marks row `row` valid in an LSB-first bitmap. +/// +/// # Panics +/// If `validity` is shorter than `row / 8 + 1` bytes - a caller-side sizing bug, not recoverable +/// input. +pub fn set_valid(validity: &mut [u8], row: usize) { + validity[row / 8] |= 1 << (row % 8); +} + +/// Implemented by any struct [`write_sheet`] can write. Derive it with +/// `#[derive(ExcelMapper)]`, which emits this alongside the reading half, or write it by hand. +pub trait ExcelWriter: Sized { + /// Transposes `rows` into one column per field, in field order. + /// + /// # Errors + /// Only for values the ABI cannot represent: a string column past 2 GiB, or an integer that + /// does not fit `i64`. Ordinary data never fails here. + fn to_columns(rows: &[Self]) -> Result, Error>; +} + +/// Writes `rows` to `path` as a single sheet, using the same field mapping +/// [`parse_sheet`](crate::workbook::parse_sheet) reads with. +/// +/// `rows` is walked ONCE and each field appended to its own column buffer. That transpose is the +/// only copy this makes - it is what the ABI's columnar shape costs a row-shaped caller. If you +/// already hold columnar buffers, call [`write_columns`] and pay nothing. +/// +/// # Errors +/// Anything [`ExcelWriter::to_columns`] or [`write_columns`] reports. +pub fn write_sheet( + path: &str, + format: i32, + rows: &[T], + options: Option<&WriteOptions>, +) -> Result<(), Error> { + let owned = T::to_columns(rows)?; + let borrowed: Vec> = owned.iter().map(OwnedColumn::as_column).collect(); + write_columns(path, format, &borrowed, options) +} + +/// In-memory equivalent of [`write_sheet`]: same transpose, but returns bytes instead of writing +/// to a path - see [`write_columns_to_memory`] for why `format` is always required here. +/// +/// # Errors +/// Anything [`ExcelWriter::to_columns`] or [`write_columns_to_memory`] reports. +pub fn write_sheet_to_memory( + format: i32, + rows: &[T], + options: Option<&WriteOptions>, +) -> Result, Error> { + let owned = T::to_columns(rows)?; + let borrowed: Vec> = owned.iter().map(OwnedColumn::as_column).collect(); + write_columns_to_memory(format, &borrowed, options) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn format_is_inferred_from_the_extension() { + assert_eq!(format_from_path("out.xlsx"), XL_FORMAT_XLSX); + assert_eq!(format_from_path("out.XLSB"), XL_FORMAT_XLSB); + assert_eq!(format_from_path("out.xls"), XL_FORMAT_XLS); + assert_eq!(format_from_path("out.csv"), XL_FORMAT_CSV); + assert_eq!(format_from_path("out.txt"), XL_FORMAT_AUTO); + assert_eq!(format_from_path("out"), XL_FORMAT_AUTO); + // A dot in a directory name is not an extension. + assert_eq!(format_from_path("v1.2/report"), XL_FORMAT_AUTO); + } + + #[test] + fn validate_rejects_a_short_validity_bitmap() { + let values = [1i64, 2, 3, 4, 5, 6, 7, 8, 9]; + let bitmap = [0u8]; // 9 rows need 2 bytes + let columns = [Column { + name: Some("a"), + data: ColumnData::I64(&values), + validity: Some(&bitmap), + }]; + let error = validate(&columns).expect_err("a 1-byte bitmap cannot cover 9 rows"); + assert_eq!(error.code, XL_INVALID_ARGUMENT); + } + + #[test] + fn validate_accepts_an_exactly_sized_validity_bitmap() { + let values = [1i64; 8]; + let bitmap = [0xFFu8]; + let columns = [Column { + name: Some("a"), + data: ColumnData::I64(&values), + validity: Some(&bitmap), + }]; + assert_eq!(validate(&columns).expect("8 rows fit in 1 byte"), 8); + } + + #[test] + fn validate_rejects_a_string_column_with_the_wrong_offset_count() { + let offsets = [0i32, 3]; + let columns = [Column { + name: Some("a"), + data: ColumnData::Str { + offsets: &offsets, + data: b"abc", + }, + validity: None, + }]; + // One row, so the offsets array is right - but claiming two rows' worth is not. + assert_eq!(validate(&columns).expect("1 row, 2 offsets"), 1); + + let bad = [0i32, 3, 6, 9]; + let mixed = [ + Column { + name: Some("a"), + data: ColumnData::I64(&[1, 2]), + validity: None, + }, + Column { + name: Some("b"), + data: ColumnData::Str { + offsets: &bad, + data: b"abcdefghi", + }, + validity: None, + }, + ]; + assert!( + validate(&mixed).is_err(), + "3 rows next to a 2-row column must be rejected" + ); + } + #[test] + fn push_str_appends_bytes_and_one_offset_per_value() { + let mut offsets = vec![0i32]; + let mut data = Vec::new(); + push_str(&mut offsets, &mut data, "uma").expect("a short string must fit"); + push_str(&mut offsets, &mut data, "").expect("an empty string must fit"); + push_str(&mut offsets, &mut data, "duas").expect("a short string must fit"); + assert_eq!(offsets, vec![0, 3, 3, 7]); + assert_eq!(data, b"umaduas"); + } + + /// The ceiling itself, tested without allocating 2 GiB. `push_str` is a two-liner around this + /// predicate; the predicate is where the arithmetic that could be wrong lives. + #[test] + fn the_offset_ceiling_is_exceeded_exactly_at_int32_max() { + let ceiling = i32::MAX as usize; + assert!(!offset_ceiling_exceeded(0, 0)); + assert!( + !offset_ceiling_exceeded(ceiling - 1, 1), + "landing exactly on i32::MAX fits" + ); + assert!( + offset_ceiling_exceeded(ceiling, 1), + "one byte past i32::MAX does not" + ); + assert!(offset_ceiling_exceeded(ceiling - 1, 2)); + } + + #[test] + fn set_valid_sets_the_lsb_first_bit_for_a_row() { + let mut bitmap = vec![0u8; 2]; + set_valid(&mut bitmap, 0); + set_valid(&mut bitmap, 2); + set_valid(&mut bitmap, 9); + assert_eq!(bitmap, vec![0b0000_0101, 0b0000_0010]); + } +} diff --git a/rust/excelreader/src/writer_handle.rs b/rust/excelreader/src/writer_handle.rs new file mode 100644 index 0000000..8789772 --- /dev/null +++ b/rust/excelreader/src/writer_handle.rs @@ -0,0 +1,202 @@ +//! Row-by-row streaming write, through the ABI's `xl_writer_handle`. +//! +//! [`WriterHandle`] is the streaming counterpart to [`crate::writer::write_columns`]/ +//! [`crate::writer::write_sheet`]: instead of materializing a whole table up front, it writes +//! directly as each call arrives - one sheet and one row open at a time. Call order mirrors the C +//! ABI (see `xl_writer_handle` in `excelreader.h`): `open`/`open_with`/`open_memory`, then per +//! sheet `start_sheet`..`end_sheet`, each containing `start_row`..`end_row` with one `write_*` +//! call per cell in between, left to right. A call out of order returns an `Err` rather than +//! corrupting output, and the handle stays usable afterward - fix the call order and continue, or +//! let `Drop` discard it. + +use crate::workbook::{buffer_to_vec, check, check_abi_version}; +use crate::writer::format_from_path; +use crate::{ + Date, Error, Time, Timestamp, WriteOptions, XlBuffer, XlWriterHandle, XL_T_BOOL, XL_T_DATE, + XL_T_F64, XL_T_I64, XL_T_TIME, XL_T_TIMESTAMP, +}; + +/// A streaming writer handle - the row-by-row counterpart to +/// [`Workbook`](crate::workbook::Workbook). +/// +/// Not thread-safe - use one per thread, same contract as the C ABI. (The raw handle makes this +/// type neither `Send` nor `Sync`, so the compiler enforces that for you.) +/// +/// Dropping a `WriterHandle` closes and releases it (`xl_close_write_handle`), silently discarding +/// any error - same convention as [`Workbook`](crate::workbook::Workbook)'s `Drop`. Call +/// [`bytes`](Self::bytes) (memory-backed) or reopen the path (file-backed) to observe the actual +/// result; do not rely on the drop for that. +pub struct WriterHandle { + handle: *mut XlWriterHandle, +} + +impl WriterHandle { + /// Creates `path` (truncating it if it already exists), inferring the format from its + /// extension via [`format_from_path`](crate::writer::format_from_path). + pub fn open(path: &str, options: Option<&WriteOptions>) -> Result { + Self::open_with(path, format_from_path(path), options) + } + + /// Creates `path` with an explicit format. `format` must be one of `XL_FORMAT_XLS`/`XLSX`/ + /// `XLSB`/`CSV` - [`XL_FORMAT_AUTO`](crate::XL_FORMAT_AUTO) is an error, the same as + /// [`write_columns`](crate::writer::write_columns). + pub fn open_with( + path: &str, + format: i32, + options: Option<&WriteOptions>, + ) -> Result { + check_abi_version()?; + let raw = options.map(WriteOptions::to_raw); + let raw_ptr = crate::options::ptr_or_null(&raw); + let mut handle: *mut XlWriterHandle = std::ptr::null_mut(); + // `raw` outlives the call below, and the native side copies the path before returning. + let status = unsafe { + crate::xl_open_write_handle( + path.as_ptr(), + path.len() as i32, + format, + raw_ptr, + &mut handle, + ) + }; + check(status)?; + Ok(WriterHandle { handle }) + } + + /// In-memory equivalent of [`open_with`](Self::open_with): read the result back with + /// [`bytes`](Self::bytes). `format` is always required here - there is no path to infer one + /// from. + pub fn open_memory(format: i32, options: Option<&WriteOptions>) -> Result { + check_abi_version()?; + let raw = options.map(WriteOptions::to_raw); + let raw_ptr = crate::options::ptr_or_null(&raw); + let mut handle: *mut XlWriterHandle = std::ptr::null_mut(); + let status = unsafe { crate::xl_open_write_handle_to_memory(format, raw_ptr, &mut handle) }; + check(status)?; + Ok(WriterHandle { handle }) + } + + /// Starts a new sheet named `name`. Must not be called again before the current sheet, if + /// any, has been ended with [`end_sheet`](Self::end_sheet). + pub fn start_sheet(&mut self, name: &str) -> Result<(), Error> { + check(unsafe { crate::xl_start_sheet(self.handle, name.as_ptr(), name.len() as i32) }) + } + + /// Starts a new row on the current sheet. Must not be called again before the current row, if + /// any, has been ended with [`end_row`](Self::end_row). + pub fn start_row(&mut self) -> Result<(), Error> { + check(unsafe { crate::xl_start_row(self.handle) }) + } + + /// Writes the next cell of the current row as text, or a blank cell for `None`. + pub fn write_str(&mut self, value: Option<&str>) -> Result<(), Error> { + let (ptr, len) = value.map_or((std::ptr::null(), 0), |text| { + (text.as_ptr(), text.len() as i32) + }); + check(unsafe { crate::xl_write_string(self.handle, ptr, len) }) + } + + /// Writes the next cell of the current row as an integer, or a blank cell for `None`. + pub fn write_i64(&mut self, value: Option) -> Result<(), Error> { + match value { + Some(v) => check(unsafe { crate::xl_write_int64(self.handle, v) }), + None => self.write_null(XL_T_I64), + } + } + + /// Writes the next cell of the current row as a floating-point number, or a blank cell for + /// `None`. + pub fn write_f64(&mut self, value: Option) -> Result<(), Error> { + match value { + Some(v) => check(unsafe { crate::xl_write_float64(self.handle, v) }), + None => self.write_null(XL_T_F64), + } + } + + /// Writes the next cell of the current row as a boolean, or a blank cell for `None`. + pub fn write_bool(&mut self, value: Option) -> Result<(), Error> { + match value { + Some(v) => check(unsafe { crate::xl_write_bool(self.handle, i32::from(v)) }), + None => self.write_null(XL_T_BOOL), + } + } + + /// Writes the next cell of the current row as a date, or a blank cell for `None`. + pub fn write_date(&mut self, value: Option) -> Result<(), Error> { + match value { + Some(v) => check(unsafe { crate::xl_write_date(self.handle, v.days_since_epoch) }), + None => self.write_null(XL_T_DATE), + } + } + + /// Writes the next cell of the current row as a time of day, or a blank cell for `None`. + pub fn write_time(&mut self, value: Option