Cargo.toml

[package]
name = "minarrow"
version = "0.17.0"
edition = "2024"
authors = ['Peter G. Bower']
build = "build.rs"
license = "Apache-2.0"
readme = "README.md"
repository = "https://github.com/pbower/minarrow"
documentation = "https://docs.rs/minarrow/"
keywords = [
    "arrow",
    "apache-arrow",
    "polars",
    "data",
    "simd",
]
categories = ["data-structures", "science", "parsing", "external-ffi-bindings"]
description = "Apache Arrow-compatible, Rust-first columnar data library for high-performance computing, native streaming, and embedded workloads. Minimal dependencies, ultra-low-latency access, automatic 64-byte SIMD alignment, and fast compile times. Great for real-time analytics, HPC pipelines, and systems integration."
# All of the below external dependencies do not need to be enabled directly
# See [features] for the relevant feature that enables them.
[dependencies]

# Arrow and Polars are for optional to/from_apache_arrow() and to/from_polars() via the optional
# `cast_arrow` and `cast_polars` features.
arrow = { version = "59.2.0", default-features = false, features = [
    "ffi",
], optional = true }
arrow-schema = { version = "59.2.0", default-features = false, features = ["ffi"], optional = true }
polars-core = { version = "0.55.2", default-features = false, features = [
    "dtype-categorical",
    "dtype-date",
    "dtype-datetime",
    "dtype-duration",
    "dtype-time",
    "dtype-struct",
    "dtype-decimal",
], optional = true }
polars-arrow = { version = "0.55.2", optional = true }
time = { version = "0.3", optional = true, features = ["parsing", "formatting", "macros", "local-offset"] }
phf = { version = "0.13.1", features = ["macros"], optional = true }
num-traits = "0.2.19"
rayon = { version = "1.11.0", optional = true }
ryu = { version = "1.0.23", optional = true }
memchr = { version = "2.8.0", optional = true }
ahash = { version = "0.8.12", optional = true }
hashbrown = { version = "0.15", optional = true }
parking_lot = { version = "0.12", optional = true }
regex = { version = "1.12.3", optional = true }
vec64 = { version = "0.5.0" }
log = { version = "0.4.29", optional = true }

[dev-dependencies]
criterion = { version = "0.8.2", features = ["html_reports"] }

[build-dependencies]
cc = { version = "1", optional = true }

[target.'cfg(target_os = "linux")'.dependencies]
libc = { version = "0.2", optional = true }

[features]

# Enables logging output via the log crate. When disabled, warnings fall back to eprintln!
log = ["dep:log"]

# Adds parallel iterators via `Rayon`
parallel_proc = ["rayon"]

# Adds roundtrip FFI tests. Leave off if you don't need it
# in your build pipeline, as it's mostly C-code.
c_ffi_tests = ['cc']

# Swaps the default categorical type from CategoricalArray<u32> to CategoricalArray<u8>.
# When enabled, TextArray contains Categorical8 instead of Categorical32, keeping the
# match arm count at 1.
default_categorical_8 = []

# Adds Categorical16, Categorical64, and whichever of Categorical8/Categorical32
# is not the current default.
#
# Highly recommend keeping this off unless required, as it adds combinatorial
# weight to the binary and enum match arms.
extended_categorical = ["default_categorical_8"]

# Adds UInt8, UInt16, Int8, Int16 types.
#
# Highly recommend keeping these off unless required
# E.g., constrained or embedded environments, as they add combinatorial
# weight to the binary and enum match arms.
#
# For most analytical use cases, they get upcasted anyway.
extended_numeric_types = []

# Adds a cube object for stacking tables on an extra axis
# Useful for time series, and group analytics.
cube = ["views", "select", "hash"]

# Adds a unified scalar type, that's useful for `Array` aggregations, and other use cases where you end up with one value.
# However, it is one of several downcasting methods available in Rust, and when predominantly
# working with numbers, one might prefer using `my_function::<i32>()` semantics which addresses
# the type immediately, e.g., in conjunction with `T: Numeric`, `T: Integer` or `T:Float` generic functions,
# rather than getting a `Scalar` object make that then needs `.i32()` style access, or a manual match.
# It is a pain that Rust can't just get the value when it's wrapped in such cases, but this is an inherent type safety limitation.
scalar_type = []

# Adds a unified value enum, that can be used for engine-level orchestration or any situation
# where a catch-all, unified encompassing type is required to satisfy the compiler.
# It includes roundtrip `From` and `TryFrom` for each inner type so that signatures do
# not need to couple to it directly. Recommend leaving off if you don't need it.
value_type = []

# `ChunkedArray` and `ChunkedTable` objects that support iterating
# over multiple inner objects of the same type, for memory-mapped streaming etc.
chunked = []

# Int64-based string
large_string = []

# Provides windowed collection views for Numeric, String, and Temporal types.
# Often, everything can be done with only the `ArrayView` abstraction, or,
# the `ArrayViewT` (&Array, Offset, Length) tuple from aliases.
# These are for the cases where they fall short, e.g., you
# have numeric or text specific functions, and want to streamline type management.
# In those cases, these abstractions provide the equivalent of `Into<NumericArrayView>`
# for several types, and accept both the original and windowed view variants. Therefore,
# one can unify numeric entry points through here enabling a flexible API, at the cost
# of more surface complexity.
views = []

# Adds a 2D matrix that uses a flat buffer in the format compatible with BLAS and LAPACK Fortan and C kernels.
# Includes `TryFrom` conversion methods so it's easy to move from `Table` column selections into the matrix,
# without worrying too much about buffers and strides. Hence, if you are working *only* with matrices, you may want
# this from the get-go. If you are working predominantly with Tabular data but running PCA's and SVD's (for e.g.),
# you can keep your data in Table format and any functions that accept `Matrix` should also work for your `Table`,
# with a small once-off performance penalty of cloning the columns into a contiguous buffer, that becomes noticeable
# with large data sizes.
matrix = []

# N-dimensional f32/f64 container with compact column-major layout.
# Slicing and axis selection are available through the `select` feature.
# Useful for bridging into Python ML/AI runtimes via DLPack.
#
# Note that this does not include statistical routines.
ndarray = []

# DLPack tensor interchange with Python and other runtimes. Compatible
# ownership, alignment, layout, and protocol choices allow direct sharing.
# Requires the `ndarray` feature.
dlpack = ["ndarray"]

# Labelled N-dimensional array wrapping NdArray with named dimensions
# and optional coordinate labels per axis. Requires `ndarray` feature.
# Coordinate value lookup is available when `scalar_type`, `views`, and
# `select` are also enabled.
xarray = ["ndarray"]

# Adds `to_apache_arrow()` for casting into that library.
cast_arrow = ["arrow", "arrow-schema"]

# Adds `to_polars()` / `from_polars()` for casting between Minarrow and Polars.
# Pulls in `extended_categorical` and `extended_numeric_types` because Polars
# routinely produces UInt8/UInt16 dictionary keys (for small Enums) and uses
# the full range of compact numeric types (i8/i16/u8/u16). Importing those
# without the matching variants present would panic at the FFI boundary.
cast_polars = [
    "polars-core",
    "polars-arrow",
    "large_string",
    "extended_categorical",
    "extended_numeric_types",
]

# Adds `Datetime` array types.
datetime = []

# Adds SIMD for the Bitmask and Arithmetic kernels
# A much more extensive set of kernels is available under the downstream simd-kernels crate.
simd = []

# Adds full datetime functionality with the `time` crate including:
# - Human-readable datetime conversions
# - Timezone-aware operations
# - Date/time arithmetic (add/subtract durations, dates)
# - Comparison operations
# - Component extraction (year, month, day, hour, etc.)
# At, the expense of an external dependency.
#
# Without this feature, datetime values are raw integer offsets.
# The `ArrowType` stored in `Field` and/or `FieldArray` specifies the
# logical type (Date32, Date64, Timestamp, etc.) for Arrow FFI compatibility.
datetime_ops = ["dep:time", "dep:phf", "datetime"]

# Adds string arithmetic kernels
# Includes (small) external dependencies, and supports
# str concatenation with floats for the arithmetic kernels
# e.g., "Hello" + 1.0 = "Hello1", etc.
# Also overloads std::ops::Add, Mul, Sub, Div, Pow
# with best-case String equivalents (e.g., '+' concatenates),
# for type unification rather than panicking.
str_arithmetic = ["ryu", "memchr"]

# Replaces all hashmaps and hashsets used for count distinct operations
# and categorical dictionary interning with the faster ahash.
fast_hash = ["dep:ahash"]

# Enables cross-batch shared categorical dictionaries.
#
# With the feature on, every chunk of a `SuperTable` or `SuperArray` that holds a categorical
# column for the same field shares one dictionary, so the integer codes
# in different batches refer to the same strings. New values added to
# any chunk are visible to every other chunk in the same column.
#
# Without this feature, every `CategoricalArray<T>` owns its dictionary
# outright and codes are not mutually meaningful across batches, which
# leads to correctness issues.
#
# Additionally a Dictionary can be used standalone to sync disparate categorical arrays.
shared_dict = ["chunked", "vec64/append_only_vec"]

# Faster variant of `shared_dict` at common volumes.
#
# Bundles three changes at
# the cost of pulling in `ahash`, `hashbrown`, and `parking_lot` as
# direct deps:
# - `ahash::RandomState` for shard selection (replaces SipHash) and as
#   the inner HashMap hasher (via the chained `fast_hash` feature).
# - `parking_lot::Mutex` for shard locks (replaces `std::sync::Mutex`).
# - `hashbrown::HashMap` with `raw_entry_mut().from_hash(...)` for
#   hash-once lookup and insert (replaces a double-hash path).
#
# Comparitive performance vs. not having this feature on (via author's laptop):
#   Interns: 2m
#   Threads: 4
#   Iterations: 1 (i.e., indicative, subject to noise)
#
#   Max categories:         10      100     1000    10,000      100,000
#   Shared dict:
#       Total (ms):         68.4    54.8    62.8    61.2        85.7
#       Per Intern (ns):    34.2    27.4    31.4    30.6        42.9
#
#   Scalable dict:
#       Total (ms):         50.3    52.3    48.4    62.9        84.9
#       Per Intern (ns):    25.2    26.2    24.2    31.5        42.5
#
# Run `cargo test --release --features fast_dict --test \
# categorical_dict_stress -- --nocapture` for measured throughput.
#
# The standard path is already fast, however if you really want to squeeze out
# performance and don't care about dependency bloat you might prefer this feature.
fast_dict = [
    "shared_dict",
    "fast_hash",
    "dep:hashbrown",
    "dep:parking_lot",
]

# Adds typed arithmetic broadcasting for add, sub, mult, div, rem.
broadcast = []

# Adds Hash and Eq implementations for Scalar, and hash_element_at for Array.
# Floats use to_bits() so NaN == NaN under this convention.
hash = ["scalar_type"]

# Adds byte size trait for best-effort size calculation
size = []

# Arena bump allocator for bulk array and Table construction.
# Reduces per-batch allocation count from O(columns) to O(1) by writing
# all column buffers into a single 64-byte aligned allocation.
# Adds Arena, Table::from_arena, and an optimised consolidate path.
arena = []

# Adds an optional `metadata: BTreeMap<String, String>` field to `Table`.
# Captures schema-level metadata that Arrow producers like PyArrow embed
# in the top-level ArrowSchema.metadata, e.g. pandas categorical ordering.
table_metadata = []

# Adds pandas-style selection for Table and TableV with .c() and .r() methods
select = []

# Enables memfd-backed buffers for zero-copy cross-process sharing (Linux only).
# Use `MemfdBuffer::new()` to create a buffer, then pass the fd to child processes.
# Children call `MemfdBuffer::reopen()` to access the same physical memory.
memfd = ["dep:libc"]

# Consider enabling this for improved Vec64 performance on Linux.
#
# Warning: sending data zero-copy over the Arrow C data interface under
# `vmap64` is undefined behaviour. Do not enable `vmap64` alongside
# `cast_arrow`, `cast_polars`, `memfd`, or any other FFI export.
vmap64 = ["vec64/mmap"]

# Switches `Bitmask::len` from a plain field to an `AtomicUsize`, allowing
# the bit count to be published via Release on write and observed via
# Acquire on read. Composes with `Storage::LBuffer`-backed buffers.
#
# LBuffer notably breaks Arrow invariants that a buffer does not change size after
# being allocated, and thus one should be very careful when working in any
# shared memory and/or FFI contexts, that would require additional sychronisation
# primitives Minarrow does not provide. Within a single rust process, Minarrow
# arrays backed by buffer handle it within standard rust semantics.
#
# Caution: lbuffer experimental feature and is not currently recommended
# for production workloads.
lbuffer = []

default = [
    "views",
    "chunked",
    "large_string",
    "simd",
    "select",
    "log"
]

[package.metadata.cargo-all-features]

# All optional dependencies have semnatic names
skip_optional_dependencies = true

# The maximum number of features to try at once
max_combination_size = 2

[package.metadata.docs.rs]
all-features = true
rustdoc-args = ["--cfg", "docsrs"]

# Arithmetic example
[[example]]
name = "arithmetic"
path = "examples/arithmetic.rs"
required-features = ["broadcast", "value_type"]

# Broadcasting examples
[[example]]
name = "test_broadcasting"
path = "examples/broadcasting/test_broadcasting.rs"
required-features = ["broadcast", "value_type"]

[[example]]
name = "test_scalar_arithmetic"
path = "examples/broadcasting/test_scalar_arithmetic.rs"
required-features = ["broadcast", "value_type", "scalar_type"]

[[example]]
name = "test_string_broadcasting"
path = "examples/broadcasting/test_string_broadcasting.rs"
required-features = ["broadcast", "value_type"]

[[example]]
name = "test_value_ops"
path = "examples/broadcasting/test_value_ops.rs"
required-features = ["broadcast", "value_type"]

[[example]]
name = "test_value_macros"
path = "examples/broadcasting/test_value_macros.rs"
required-features = ["broadcast", "value_type"]

# N-dimensional examples
[[example]]
name = "ndarray"
path = "examples/ndarray.rs"
required-features = ["ndarray"]

[[example]]
name = "super_ndarray"
path = "examples/super_ndarray.rs"
required-features = ["ndarray"]

[[example]]
name = "xarray"
path = "examples/xarray.rs"
required-features = ["xarray", "scalar_type", "views", "select"]

[[bench]]
name = "hotloop_benchmark_simd"
harness = false
required-features = ["cast_arrow"]

[[bench]]
name = "hotloop_benchmark_std"
harness = false
required-features = ["cast_arrow"]

[[bench]]
name = "hotloop_benchmark_avg_simd"
harness = false
required-features = ["cast_arrow"]

[[bench]]
name = "hotloop_benchmark_avg_std"
harness = false
required-features = ["cast_arrow"]

[[bench]]
name = "benchmark_parallel_simd"
harness = false
required-features = ["parallel_proc"]

# FFI examples
[[example]]
name = "apache_arrow_ffi"
path = "examples/ffi/apache_arrow_ffi.rs"

[[example]]
name = "polars_ffi"
path = "examples/ffi/polars_ffi.rs"

# Print example
[[example]]
name = "print"
path = "examples/print.rs"

# Selection example
[[example]]
name = "selection"
path = "examples/selection.rs"
required-features = ["select", "views"]

# Selection test
[[test]]
name = "test_selection"
path = "tests/test_selection.rs"
required-features = ["select", "views"]

# LBuffer concurrent stress test
[[test]]
name = "lbuffer_stress"
path = "tests/lbuffer_stress.rs"
required-features = ["lbuffer"]

[[bench]]
name = "consolidate"
harness = false
required-features = ["chunked", "arena"]