From 576e738d177560ea7592dd853ca22765f42a8faf Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Apr 2026 13:54:29 +0000 Subject: [PATCH 01/14] feat: add types module for column inference and per-cell casting Public API on the nsv crate that consumers (nsv-duckdb rewire, ENSV, Python bindings) can share for type narrowing. Surface: - Type enum: Boolean, BigInt, Double, Date, Timestamp, Varchar - Type::try_cast(&str) -> Option, Type::accepts(&str) -> bool - try_parse_{boolean,bigint,double,date,timestamp} standalone parsers - Date / Timestamp value types - infer_column / infer_column_with for sample-based inference - DEFAULT_LADDER, DEFAULT_SAMPLE_SIZE Cast semantics: - Boolean: lowercase "true"/"false" only - BigInt: Rust's i64 FromStr - Double: Rust's f64 FromStr - Date / Timestamp: strict RFC 3339 (full-date / date-time) These are not byte-identical to nsv-duckdb's current DuckDB-delegated casters; rewiring the extension onto this module would tighten DATE / TIMESTAMP in particular and is intentionally out of scope here. --- src/lib.rs | 1 + src/types.rs | 540 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 541 insertions(+) create mode 100644 src/types.rs diff --git a/src/lib.rs b/src/lib.rs index ae3dce8..4d06b13 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,6 +16,7 @@ //! //! For smaller files, we use a sequential fast path to avoid thread overhead. +pub mod types; pub mod util; use memchr::memmem; diff --git a/src/types.rs b/src/types.rs new file mode 100644 index 0000000..71d17a7 --- /dev/null +++ b/src/types.rs @@ -0,0 +1,540 @@ +//! Type narrowing and per-cell parsing for NSV columns. +//! +//! NSV cells are bytes; this module gives consumers (DuckDB extension, ENSV, +//! Python bindings, …) a shared, DuckDB-agnostic way to: +//! +//! - parse a single cell into a typed value ([`Type::try_cast`]), +//! - validate a cell against a declared type ([`Type::accepts`]), +//! - infer a column type from a sample of cells ([`infer_column`]). +//! +//! Empty cells encode NULL in NSV, so the typed parsers reject `""`. The +//! inference pass skips empty cells: an all-NULL column resolves to +//! [`Type::Varchar`]. +//! +//! ## Cast semantics +//! +//! - **Boolean**: exactly `true` or `false`, lowercase ASCII. +//! - **BigInt**: parsed as `i64` via [`str::parse`]; rejects leading `+`, +//! whitespace, locale separators, scientific notation, and overflow. +//! - **Double**: parsed as `f64` via [`str::parse`]; accepts `inf`/`nan` +//! spellings that Rust's float parser accepts. +//! - **Date / Timestamp**: strict RFC 3339 (`full-date` and `date-time`). +//! No locale alternates, no missing offset, no space-separator leniency +//! beyond the one RFC 3339 §5.6 NOTE explicitly permits. +//! +//! These are *not* the same as DuckDB's casters. The `nsv-duckdb` extension +//! currently delegates to DuckDB's built-in casts; rewiring it onto this +//! module would tighten DATE/TIMESTAMP parsing in particular. That rewire +//! is out of scope here. +//! +//! ## Stability +//! +//! New module in a 0.0.x crate. Surface may change before 1.0. + +// ── Type enum ──────────────────────────────────────────────────────── + +/// Column type produced by inference, or accepted by per-cell casts. +/// +/// Variants are listed in the order used by [`DEFAULT_LADDER`]: most +/// specific first, [`Type::Varchar`] last as the always-accepting fallback. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Type { + Boolean, + BigInt, + Double, + Date, + Timestamp, + Varchar, +} + +impl Type { + /// Parse a non-empty cell into a typed value, or return `None` on failure. + /// + /// Empty input is always rejected (NSV uses empty cells for NULL). + pub fn try_cast<'a>(&self, s: &'a str) -> Option> { + if s.is_empty() { + return None; + } + match self { + Type::Boolean => try_parse_boolean(s).map(Value::Boolean), + Type::BigInt => try_parse_bigint(s).map(Value::BigInt), + Type::Double => try_parse_double(s).map(Value::Double), + Type::Date => try_parse_date(s).map(Value::Date), + Type::Timestamp => try_parse_timestamp(s).map(Value::Timestamp), + Type::Varchar => Some(Value::Varchar(s)), + } + } + + /// `true` if [`Type::try_cast`] would succeed. + pub fn accepts(&self, s: &str) -> bool { + self.try_cast(s).is_some() + } +} + +// ── Value enum ─────────────────────────────────────────────────────── + +/// Result of [`Type::try_cast`]. `Varchar` borrows from the input. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum Value<'a> { + Boolean(bool), + BigInt(i64), + Double(f64), + Date(Date), + Timestamp(Timestamp), + Varchar(&'a str), +} + +/// RFC 3339 `full-date`: `YYYY-MM-DD` in the proleptic Gregorian calendar. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct Date { + pub year: i32, + pub month: u8, + pub day: u8, +} + +/// RFC 3339 `date-time`: date + time-of-day + UTC offset. +/// +/// `nanosecond` carries the fractional second (0 if absent). +/// `offset_minutes` is the offset from UTC in minutes (`-00:00` is preserved +/// as `Some(0)`, distinct from `+00:00`/`Z` only via the `unknown_offset` flag). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct Timestamp { + pub year: i32, + pub month: u8, + pub day: u8, + pub hour: u8, + pub minute: u8, + pub second: u8, + pub nanosecond: u32, + pub offset_minutes: i16, + /// `true` iff the original offset was the RFC 3339 §4.3 `-00:00` + /// "unknown local offset" sentinel. + pub unknown_offset: bool, +} + +// ── Per-type parsers ───────────────────────────────────────────────── + +/// Parse exactly `true` or `false` (lowercase). +pub fn try_parse_boolean(s: &str) -> Option { + match s { + "true" => Some(true), + "false" => Some(false), + _ => None, + } +} + +/// Parse as `i64` via [`str::parse`]. +pub fn try_parse_bigint(s: &str) -> Option { + s.parse::().ok() +} + +/// Parse as `f64` via [`str::parse`]. +pub fn try_parse_double(s: &str) -> Option { + s.parse::().ok() +} + +/// Parse RFC 3339 `full-date`: `YYYY-MM-DD`. +pub fn try_parse_date(s: &str) -> Option { + let b = s.as_bytes(); + if b.len() != 10 || b[4] != b'-' || b[7] != b'-' { + return None; + } + let year = parse_u32(&b[0..4])? as i32; + let month = parse_u32(&b[5..7])? as u8; + let day = parse_u32(&b[8..10])? as u8; + if !is_valid_ymd(year, month, day) { + return None; + } + Some(Date { year, month, day }) +} + +/// Parse RFC 3339 `date-time`: `YYYY-MM-DD(T| )HH:MM:SS[.fraction](Z|±HH:MM)`. +/// +/// Per RFC 3339 §5.6, the date/time separator may be uppercase `T` or a +/// space; we accept both. The offset is required. +pub fn try_parse_timestamp(s: &str) -> Option { + let b = s.as_bytes(); + if b.len() < 20 { + return None; + } + if b[4] != b'-' || b[7] != b'-' { + return None; + } + if b[10] != b'T' && b[10] != b' ' { + return None; + } + if b[13] != b':' || b[16] != b':' { + return None; + } + + let year = parse_u32(&b[0..4])? as i32; + let month = parse_u32(&b[5..7])? as u8; + let day = parse_u32(&b[8..10])? as u8; + let hour = parse_u32(&b[11..13])? as u8; + let minute = parse_u32(&b[14..16])? as u8; + let second = parse_u32(&b[17..19])? as u8; + + if !is_valid_ymd(year, month, day) { + return None; + } + if hour > 23 || minute > 59 || second > 60 { + // RFC 3339 §5.6 allows second=60 for leap seconds. + return None; + } + + let mut i = 19; + let mut nanosecond: u32 = 0; + if i < b.len() && b[i] == b'.' { + i += 1; + let frac_start = i; + while i < b.len() && b[i].is_ascii_digit() { + i += 1; + } + let frac = &b[frac_start..i]; + if frac.is_empty() { + return None; + } + nanosecond = scale_fraction_to_nanos(frac)?; + } + + let (offset_minutes, unknown_offset) = parse_offset(&b[i..])?; + + Some(Timestamp { + year, + month, + day, + hour, + minute, + second, + nanosecond, + offset_minutes, + unknown_offset, + }) +} + +fn parse_offset(b: &[u8]) -> Option<(i16, bool)> { + if b == b"Z" { + return Some((0, false)); + } + if b.len() != 6 { + return None; + } + let sign = match b[0] { + b'+' => 1i16, + b'-' => -1i16, + _ => return None, + }; + if b[3] != b':' { + return None; + } + let hh = parse_u32(&b[1..3])?; + let mm = parse_u32(&b[4..6])?; + if hh > 23 || mm > 59 { + return None; + } + let total = sign * (hh as i16 * 60 + mm as i16); + let unknown = sign == -1 && hh == 0 && mm == 0; + Some((total, unknown)) +} + +fn scale_fraction_to_nanos(digits: &[u8]) -> Option { + // Use up to 9 fractional digits; truncate the rest. RFC 3339 doesn't + // bound the fraction length, but anything past nanoseconds is dropped. + let take = digits.len().min(9); + let mut value: u32 = 0; + for &d in &digits[..take] { + if !d.is_ascii_digit() { + return None; + } + value = value * 10 + (d - b'0') as u32; + } + for &d in &digits[take..] { + if !d.is_ascii_digit() { + return None; + } + } + for _ in take..9 { + value *= 10; + } + Some(value) +} + +fn parse_u32(b: &[u8]) -> Option { + let mut v: u32 = 0; + for &c in b { + if !c.is_ascii_digit() { + return None; + } + v = v.checked_mul(10)?.checked_add((c - b'0') as u32)?; + } + Some(v) +} + +fn is_valid_ymd(year: i32, month: u8, day: u8) -> bool { + if !(1..=12).contains(&month) || day < 1 { + return false; + } + let max = days_in_month(year, month); + day <= max +} + +fn days_in_month(year: i32, month: u8) -> u8 { + match month { + 1 | 3 | 5 | 7 | 8 | 10 | 12 => 31, + 4 | 6 | 9 | 11 => 30, + 2 => { + if is_leap_year(year) { + 29 + } else { + 28 + } + } + _ => 0, + } +} + +fn is_leap_year(year: i32) -> bool { + (year % 4 == 0 && year % 100 != 0) || year % 400 == 0 +} + +// ── Inference ──────────────────────────────────────────────────────── + +/// Default inference ladder: most specific first, [`Type::Varchar`] last. +/// +/// Mirrors the candidate set used by the `nsv-duckdb` extension today +/// (`BOOLEAN, BIGINT, DOUBLE, DATE, TIMESTAMP, VARCHAR`). +pub const DEFAULT_LADDER: &[Type] = &[ + Type::Boolean, + Type::BigInt, + Type::Double, + Type::Date, + Type::Timestamp, + Type::Varchar, +]; + +/// Recommended sample size for column inference. +pub const DEFAULT_SAMPLE_SIZE: usize = 1000; + +/// Infer a column type from a sample of cells using [`DEFAULT_LADDER`]. +/// +/// Empty cells are treated as NULL and skipped. An all-NULL sample +/// resolves to [`Type::Varchar`] (matching the `nsv-duckdb` convention). +/// Callers should pre-truncate to [`DEFAULT_SAMPLE_SIZE`] cells. +pub fn infer_column(cells: I) -> Type +where + I: IntoIterator, + S: AsRef, +{ + infer_column_with(cells, DEFAULT_LADDER) +} + +/// Infer a column type using a caller-supplied ladder. +/// +/// The first candidate that accepts every non-empty cell wins; if no +/// candidate accepts every cell, returns [`Type::Varchar`]. If `ladder` +/// is empty, returns [`Type::Varchar`]. +pub fn infer_column_with(cells: I, ladder: &[Type]) -> Type +where + I: IntoIterator, + S: AsRef, +{ + // Materialize once; we walk the sample per candidate. + let sample: Vec = cells.into_iter().collect(); + let non_empty: Vec<&str> = sample + .iter() + .map(|s| s.as_ref()) + .filter(|s| !s.is_empty()) + .collect(); + + if non_empty.is_empty() { + return Type::Varchar; + } + + for &candidate in ladder { + if candidate == Type::Varchar { + return Type::Varchar; + } + if non_empty.iter().all(|s| candidate.accepts(s)) { + return candidate; + } + } + Type::Varchar +} + +// ── Tests ──────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + + // Boolean + #[test] + fn boolean_accepts_canonical_only() { + assert_eq!(try_parse_boolean("true"), Some(true)); + assert_eq!(try_parse_boolean("false"), Some(false)); + assert_eq!(try_parse_boolean("True"), None); + assert_eq!(try_parse_boolean("TRUE"), None); + assert_eq!(try_parse_boolean("1"), None); + assert_eq!(try_parse_boolean(""), None); + assert_eq!(try_parse_boolean(" true"), None); + } + + // BigInt + #[test] + fn bigint_round_numbers() { + assert_eq!(try_parse_bigint("0"), Some(0)); + assert_eq!(try_parse_bigint("-1"), Some(-1)); + assert_eq!(try_parse_bigint("9223372036854775807"), Some(i64::MAX)); + assert_eq!(try_parse_bigint("-9223372036854775808"), Some(i64::MIN)); + } + + #[test] + fn bigint_rejects_overflow_and_junk() { + assert_eq!(try_parse_bigint("9223372036854775808"), None); + assert_eq!(try_parse_bigint("1.0"), None); + assert_eq!(try_parse_bigint("1e3"), None); + assert_eq!(try_parse_bigint(" 1"), None); + assert_eq!(try_parse_bigint("1_000"), None); + assert_eq!(try_parse_bigint(""), None); + } + + // Double + #[test] + fn double_round_numbers() { + assert_eq!(try_parse_double("0"), Some(0.0)); + assert_eq!(try_parse_double("2.5"), Some(2.5)); + assert_eq!(try_parse_double("-1e10"), Some(-1e10)); + } + + #[test] + fn double_rejects_junk() { + assert_eq!(try_parse_double(""), None); + assert_eq!(try_parse_double("abc"), None); + assert_eq!(try_parse_double("1,5"), None); + } + + // Date + #[test] + fn date_strict_rfc3339() { + let d = try_parse_date("2024-01-15").unwrap(); + assert_eq!((d.year, d.month, d.day), (2024, 1, 15)); + assert_eq!(try_parse_date("2024-02-29").map(|d| d.day), Some(29)); + assert_eq!(try_parse_date("2023-02-29"), None); + assert_eq!(try_parse_date("2000-02-29").map(|d| d.day), Some(29)); + assert_eq!(try_parse_date("1900-02-29"), None); + } + + #[test] + fn date_rejects_alternate_formats() { + assert_eq!(try_parse_date(""), None); + assert_eq!(try_parse_date("2024-1-15"), None); + assert_eq!(try_parse_date("2024/01/15"), None); + assert_eq!(try_parse_date("24-01-15"), None); + assert_eq!(try_parse_date("2024-13-01"), None); + assert_eq!(try_parse_date("2024-04-31"), None); + assert_eq!(try_parse_date("2024-01-15T"), None); + } + + // Timestamp + #[test] + fn timestamp_strict_rfc3339() { + let t = try_parse_timestamp("2024-01-15T12:34:56Z").unwrap(); + assert_eq!(t.hour, 12); + assert_eq!(t.offset_minutes, 0); + assert!(!t.unknown_offset); + + let t = try_parse_timestamp("2024-01-15T12:34:56.789+02:00").unwrap(); + assert_eq!(t.nanosecond, 789_000_000); + assert_eq!(t.offset_minutes, 120); + + let t = try_parse_timestamp("2024-01-15 12:34:56-05:30").unwrap(); + assert_eq!(t.offset_minutes, -(5 * 60 + 30)); + + let t = try_parse_timestamp("2024-01-15T12:34:56-00:00").unwrap(); + assert_eq!(t.offset_minutes, 0); + assert!(t.unknown_offset); + } + + #[test] + fn timestamp_fractional_truncates_past_nanos() { + let t = try_parse_timestamp("2024-01-15T00:00:00.1234567890123Z").unwrap(); + assert_eq!(t.nanosecond, 123_456_789); + } + + #[test] + fn timestamp_rejects_non_rfc3339() { + assert_eq!(try_parse_timestamp(""), None); + assert_eq!(try_parse_timestamp("2024-01-15T12:34:56"), None); // no offset + assert_eq!(try_parse_timestamp("2024-01-15T12:34Z"), None); + assert_eq!(try_parse_timestamp("2024-01-15T25:00:00Z"), None); + assert_eq!(try_parse_timestamp("2024-01-15T12:34:56.Z"), None); + assert_eq!(try_parse_timestamp("2024-01-15T12:34:56+25:00"), None); + } + + // try_cast / accepts + #[test] + fn try_cast_dispatches_per_type() { + assert!(matches!(Type::Boolean.try_cast("true"), Some(Value::Boolean(true)))); + assert!(matches!(Type::BigInt.try_cast("42"), Some(Value::BigInt(42)))); + assert!(matches!(Type::Double.try_cast("2.5"), Some(Value::Double(_)))); + assert!(matches!(Type::Varchar.try_cast("anything"), Some(Value::Varchar("anything")))); + assert!(Type::Boolean.try_cast("").is_none()); + assert!(Type::Varchar.try_cast("").is_none()); + } + + #[test] + fn accepts_matches_try_cast() { + for s in ["true", "42", "3.14", "2024-01-01", "2024-01-01T00:00:00Z", "x", ""] { + for t in DEFAULT_LADDER { + assert_eq!(t.accepts(s), t.try_cast(s).is_some(), "{:?} on {:?}", t, s); + } + } + } + + // Inference + #[test] + fn infer_picks_most_specific() { + assert_eq!(infer_column(["true", "false", "true"]), Type::Boolean); + assert_eq!(infer_column(["1", "2", "3"]), Type::BigInt); + assert_eq!(infer_column(["1", "2.0", "3"]), Type::Double); + assert_eq!(infer_column(["2024-01-01", "2024-12-31"]), Type::Date); + assert_eq!( + infer_column(["2024-01-01T00:00:00Z", "2024-12-31T23:59:59Z"]), + Type::Timestamp + ); + assert_eq!(infer_column(["a", "b"]), Type::Varchar); + } + + #[test] + fn infer_skips_empty_cells() { + assert_eq!(infer_column(["1", "", "2", ""]), Type::BigInt); + assert_eq!(infer_column(["", "", ""]), Type::Varchar); + let empty: [&str; 0] = []; + assert_eq!(infer_column(empty), Type::Varchar); + } + + #[test] + fn infer_falls_through_on_mixed() { + assert_eq!(infer_column(["1", "abc"]), Type::Varchar); + assert_eq!(infer_column(["true", "1"]), Type::Varchar); + } + + #[test] + fn infer_with_custom_ladder() { + // Skip BigInt: "42" should fall through to Double. + let ladder = &[Type::Boolean, Type::Double, Type::Varchar]; + assert_eq!(infer_column_with(["42"], ladder), Type::Double); + assert_eq!(infer_column_with(["42"], &[]), Type::Varchar); + } + + /// Mirrors the four-column fixture in `nsv-duckdb`'s test suite: + /// `('Alice', '30', '50000.50', 'true'), ('Bob', '25', '75000.25', 'false')` + /// → `(VARCHAR, BIGINT, DOUBLE, BOOLEAN)`. + #[test] + fn infer_matches_nsv_duckdb_fixture() { + assert_eq!(infer_column(["Alice", "Bob"]), Type::Varchar); + assert_eq!(infer_column(["30", "25"]), Type::BigInt); + assert_eq!(infer_column(["50000.50", "75000.25"]), Type::Double); + assert_eq!(infer_column(["true", "false"]), Type::Boolean); + } +} From 9291c273eb6dfbc64f235bba9f855df80d960df3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Apr 2026 14:06:30 +0000 Subject: [PATCH 02/14] drop Boolean from first iteration Per review: lowercase-only true/false vocabulary wasn't grounded in ENSV or DuckDB. Cut Boolean entirely from the initial surface; it can come back when its accepted spelling is settled. Removes Type::Boolean, Value::Boolean, try_parse_boolean, and the related tests. DEFAULT_LADDER is now [BigInt, Double, Date, Timestamp, Varchar]. --- src/types.rs | 49 ++++--------------------------------------------- 1 file changed, 4 insertions(+), 45 deletions(-) diff --git a/src/types.rs b/src/types.rs index 71d17a7..218c43f 100644 --- a/src/types.rs +++ b/src/types.rs @@ -13,7 +13,6 @@ //! //! ## Cast semantics //! -//! - **Boolean**: exactly `true` or `false`, lowercase ASCII. //! - **BigInt**: parsed as `i64` via [`str::parse`]; rejects leading `+`, //! whitespace, locale separators, scientific notation, and overflow. //! - **Double**: parsed as `f64` via [`str::parse`]; accepts `inf`/`nan` @@ -39,7 +38,6 @@ /// specific first, [`Type::Varchar`] last as the always-accepting fallback. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum Type { - Boolean, BigInt, Double, Date, @@ -56,7 +54,6 @@ impl Type { return None; } match self { - Type::Boolean => try_parse_boolean(s).map(Value::Boolean), Type::BigInt => try_parse_bigint(s).map(Value::BigInt), Type::Double => try_parse_double(s).map(Value::Double), Type::Date => try_parse_date(s).map(Value::Date), @@ -76,7 +73,6 @@ impl Type { /// Result of [`Type::try_cast`]. `Varchar` borrows from the input. #[derive(Debug, Clone, Copy, PartialEq)] pub enum Value<'a> { - Boolean(bool), BigInt(i64), Double(f64), Date(Date), @@ -114,15 +110,6 @@ pub struct Timestamp { // ── Per-type parsers ───────────────────────────────────────────────── -/// Parse exactly `true` or `false` (lowercase). -pub fn try_parse_boolean(s: &str) -> Option { - match s { - "true" => Some(true), - "false" => Some(false), - _ => None, - } -} - /// Parse as `i64` via [`str::parse`]. pub fn try_parse_bigint(s: &str) -> Option { s.parse::().ok() @@ -300,11 +287,7 @@ fn is_leap_year(year: i32) -> bool { // ── Inference ──────────────────────────────────────────────────────── /// Default inference ladder: most specific first, [`Type::Varchar`] last. -/// -/// Mirrors the candidate set used by the `nsv-duckdb` extension today -/// (`BOOLEAN, BIGINT, DOUBLE, DATE, TIMESTAMP, VARCHAR`). pub const DEFAULT_LADDER: &[Type] = &[ - Type::Boolean, Type::BigInt, Type::Double, Type::Date, @@ -367,18 +350,6 @@ where mod tests { use super::*; - // Boolean - #[test] - fn boolean_accepts_canonical_only() { - assert_eq!(try_parse_boolean("true"), Some(true)); - assert_eq!(try_parse_boolean("false"), Some(false)); - assert_eq!(try_parse_boolean("True"), None); - assert_eq!(try_parse_boolean("TRUE"), None); - assert_eq!(try_parse_boolean("1"), None); - assert_eq!(try_parse_boolean(""), None); - assert_eq!(try_parse_boolean(" true"), None); - } - // BigInt #[test] fn bigint_round_numbers() { @@ -474,11 +445,10 @@ mod tests { // try_cast / accepts #[test] fn try_cast_dispatches_per_type() { - assert!(matches!(Type::Boolean.try_cast("true"), Some(Value::Boolean(true)))); assert!(matches!(Type::BigInt.try_cast("42"), Some(Value::BigInt(42)))); assert!(matches!(Type::Double.try_cast("2.5"), Some(Value::Double(_)))); assert!(matches!(Type::Varchar.try_cast("anything"), Some(Value::Varchar("anything")))); - assert!(Type::Boolean.try_cast("").is_none()); + assert!(Type::BigInt.try_cast("").is_none()); assert!(Type::Varchar.try_cast("").is_none()); } @@ -494,7 +464,6 @@ mod tests { // Inference #[test] fn infer_picks_most_specific() { - assert_eq!(infer_column(["true", "false", "true"]), Type::Boolean); assert_eq!(infer_column(["1", "2", "3"]), Type::BigInt); assert_eq!(infer_column(["1", "2.0", "3"]), Type::Double); assert_eq!(infer_column(["2024-01-01", "2024-12-31"]), Type::Date); @@ -502,6 +471,7 @@ mod tests { infer_column(["2024-01-01T00:00:00Z", "2024-12-31T23:59:59Z"]), Type::Timestamp ); + assert_eq!(infer_column(["true", "false"]), Type::Varchar); assert_eq!(infer_column(["a", "b"]), Type::Varchar); } @@ -516,25 +486,14 @@ mod tests { #[test] fn infer_falls_through_on_mixed() { assert_eq!(infer_column(["1", "abc"]), Type::Varchar); - assert_eq!(infer_column(["true", "1"]), Type::Varchar); + assert_eq!(infer_column(["1.0", "abc"]), Type::Varchar); } #[test] fn infer_with_custom_ladder() { // Skip BigInt: "42" should fall through to Double. - let ladder = &[Type::Boolean, Type::Double, Type::Varchar]; + let ladder = &[Type::Double, Type::Varchar]; assert_eq!(infer_column_with(["42"], ladder), Type::Double); assert_eq!(infer_column_with(["42"], &[]), Type::Varchar); } - - /// Mirrors the four-column fixture in `nsv-duckdb`'s test suite: - /// `('Alice', '30', '50000.50', 'true'), ('Bob', '25', '75000.25', 'false')` - /// → `(VARCHAR, BIGINT, DOUBLE, BOOLEAN)`. - #[test] - fn infer_matches_nsv_duckdb_fixture() { - assert_eq!(infer_column(["Alice", "Bob"]), Type::Varchar); - assert_eq!(infer_column(["30", "25"]), Type::BigInt); - assert_eq!(infer_column(["50000.50", "75000.25"]), Type::Double); - assert_eq!(infer_column(["true", "false"]), Type::Boolean); - } } From a67aa69f8156f6fc0ad99bffd50e208c1a3d84f0 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Apr 2026 14:16:05 +0000 Subject: [PATCH 03/14] drop ladder framing and DuckDB cross-references MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per review: 'ladder' implies a widening relationship between types; it's just the order inference tries candidates. Renamed DEFAULT_LADDER -> DEFAULT_INFERENCE_ORDER, infer_column_with's 'ladder' parameter -> 'order'. Reframed module docstring to: - drop 'matches DuckDB's caster' / 'matches nsv-duckdb convention' cross-references (their constraints aren't ours) - state our accepted spellings directly, not as a diff against DuckDB - call out that consumers needing their own conversion semantics (DuckDB BOOL spellings, pandas na_values, …) should bypass this module rather than ask us to widen it --- src/types.rs | 72 +++++++++++++++++++++++++++------------------------- 1 file changed, 38 insertions(+), 34 deletions(-) diff --git a/src/types.rs b/src/types.rs index 218c43f..8284678 100644 --- a/src/types.rs +++ b/src/types.rs @@ -1,41 +1,44 @@ -//! Type narrowing and per-cell parsing for NSV columns. +//! Per-cell parsing and column type inference for NSV. //! -//! NSV cells are bytes; this module gives consumers (DuckDB extension, ENSV, -//! Python bindings, …) a shared, DuckDB-agnostic way to: +//! Surface: //! //! - parse a single cell into a typed value ([`Type::try_cast`]), //! - validate a cell against a declared type ([`Type::accepts`]), //! - infer a column type from a sample of cells ([`infer_column`]). //! -//! Empty cells encode NULL in NSV, so the typed parsers reject `""`. The -//! inference pass skips empty cells: an all-NULL column resolves to +//! Empty cells encode NULL in NSV; the typed parsers reject `""`, the +//! inference pass skips empty cells, and an all-NULL column resolves to //! [`Type::Varchar`]. //! -//! ## Cast semantics +//! Consumers that need their own conversion semantics (e.g. a DuckDB +//! extension applying DuckDB's BOOL spellings, a pandas reader applying +//! pandas' `na_values`) should bypass this module and parse cells +//! themselves. This module commits to one specific set of accepted +//! spellings; widening it to be everyone's-conversion-rules is not the +//! goal. //! -//! - **BigInt**: parsed as `i64` via [`str::parse`]; rejects leading `+`, -//! whitespace, locale separators, scientific notation, and overflow. -//! - **Double**: parsed as `f64` via [`str::parse`]; accepts `inf`/`nan` -//! spellings that Rust's float parser accepts. -//! - **Date / Timestamp**: strict RFC 3339 (`full-date` and `date-time`). -//! No locale alternates, no missing offset, no space-separator leniency -//! beyond the one RFC 3339 §5.6 NOTE explicitly permits. +//! ## Accepted spellings //! -//! These are *not* the same as DuckDB's casters. The `nsv-duckdb` extension -//! currently delegates to DuckDB's built-in casts; rewiring it onto this -//! module would tighten DATE/TIMESTAMP parsing in particular. That rewire -//! is out of scope here. +//! - **BigInt**: signed decimal integer fitting in `i64`. No leading `+`, +//! no whitespace, no separators, no scientific notation. +//! - **Double**: decimal or scientific notation parseable as `f64`, +//! including `inf`/`nan` spellings Rust's float parser accepts. +//! - **Date**: `YYYY-MM-DD` (RFC 3339 `full-date`), validated against +//! the proleptic Gregorian calendar. +//! - **Timestamp**: `YYYY-MM-DD(T| )HH:MM:SS[.fraction](Z|±HH:MM)` +//! (RFC 3339 `date-time`). Offset is required. //! //! ## Stability //! -//! New module in a 0.0.x crate. Surface may change before 1.0. +//! New module in a 0.0.x crate. The set of types and the accepted +//! spellings are not yet settled — both may change before 1.0. // ── Type enum ──────────────────────────────────────────────────────── /// Column type produced by inference, or accepted by per-cell casts. /// -/// Variants are listed in the order used by [`DEFAULT_LADDER`]: most -/// specific first, [`Type::Varchar`] last as the always-accepting fallback. +/// Variants are listed in the order [`DEFAULT_INFERENCE_ORDER`] tries +/// them. [`Type::Varchar`] always accepts and so sits last. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum Type { BigInt, @@ -286,8 +289,9 @@ fn is_leap_year(year: i32) -> bool { // ── Inference ──────────────────────────────────────────────────────── -/// Default inference ladder: most specific first, [`Type::Varchar`] last. -pub const DEFAULT_LADDER: &[Type] = &[ +/// Order in which [`infer_column`] tries candidate types. The first +/// candidate that accepts every non-empty cell wins. +pub const DEFAULT_INFERENCE_ORDER: &[Type] = &[ Type::BigInt, Type::Double, Type::Date, @@ -298,25 +302,25 @@ pub const DEFAULT_LADDER: &[Type] = &[ /// Recommended sample size for column inference. pub const DEFAULT_SAMPLE_SIZE: usize = 1000; -/// Infer a column type from a sample of cells using [`DEFAULT_LADDER`]. +/// Infer a column type from a sample of cells using [`DEFAULT_INFERENCE_ORDER`]. /// /// Empty cells are treated as NULL and skipped. An all-NULL sample -/// resolves to [`Type::Varchar`] (matching the `nsv-duckdb` convention). -/// Callers should pre-truncate to [`DEFAULT_SAMPLE_SIZE`] cells. +/// resolves to [`Type::Varchar`]. Callers should pre-truncate to +/// [`DEFAULT_SAMPLE_SIZE`] cells. pub fn infer_column(cells: I) -> Type where I: IntoIterator, S: AsRef, { - infer_column_with(cells, DEFAULT_LADDER) + infer_column_with(cells, DEFAULT_INFERENCE_ORDER) } -/// Infer a column type using a caller-supplied ladder. +/// Infer a column type using a caller-supplied candidate order. /// /// The first candidate that accepts every non-empty cell wins; if no -/// candidate accepts every cell, returns [`Type::Varchar`]. If `ladder` +/// candidate accepts every cell, returns [`Type::Varchar`]. If `order` /// is empty, returns [`Type::Varchar`]. -pub fn infer_column_with(cells: I, ladder: &[Type]) -> Type +pub fn infer_column_with(cells: I, order: &[Type]) -> Type where I: IntoIterator, S: AsRef, @@ -333,7 +337,7 @@ where return Type::Varchar; } - for &candidate in ladder { + for &candidate in order { if candidate == Type::Varchar { return Type::Varchar; } @@ -455,7 +459,7 @@ mod tests { #[test] fn accepts_matches_try_cast() { for s in ["true", "42", "3.14", "2024-01-01", "2024-01-01T00:00:00Z", "x", ""] { - for t in DEFAULT_LADDER { + for t in DEFAULT_INFERENCE_ORDER { assert_eq!(t.accepts(s), t.try_cast(s).is_some(), "{:?} on {:?}", t, s); } } @@ -490,10 +494,10 @@ mod tests { } #[test] - fn infer_with_custom_ladder() { + fn infer_with_custom_order() { // Skip BigInt: "42" should fall through to Double. - let ladder = &[Type::Double, Type::Varchar]; - assert_eq!(infer_column_with(["42"], ladder), Type::Double); + let order = &[Type::Double, Type::Varchar]; + assert_eq!(infer_column_with(["42"], order), Type::Double); assert_eq!(infer_column_with(["42"], &[]), Type::Varchar); } } From fa3cb29b4b15f81ea34fca57fdc2560b6fcea36d Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Apr 2026 14:38:22 +0000 Subject: [PATCH 04/14] remove types module Inference and per-cell parsing don't belong in the generic NSV crate. Each consumer (DuckDB extension, ENSV, pandas binding) brings its own conversion logic, parameterized by its own type model and constraints. The crate stays format-only. --- src/types.rs | 503 --------------------------------------------------- 1 file changed, 503 deletions(-) delete mode 100644 src/types.rs diff --git a/src/types.rs b/src/types.rs deleted file mode 100644 index 8284678..0000000 --- a/src/types.rs +++ /dev/null @@ -1,503 +0,0 @@ -//! Per-cell parsing and column type inference for NSV. -//! -//! Surface: -//! -//! - parse a single cell into a typed value ([`Type::try_cast`]), -//! - validate a cell against a declared type ([`Type::accepts`]), -//! - infer a column type from a sample of cells ([`infer_column`]). -//! -//! Empty cells encode NULL in NSV; the typed parsers reject `""`, the -//! inference pass skips empty cells, and an all-NULL column resolves to -//! [`Type::Varchar`]. -//! -//! Consumers that need their own conversion semantics (e.g. a DuckDB -//! extension applying DuckDB's BOOL spellings, a pandas reader applying -//! pandas' `na_values`) should bypass this module and parse cells -//! themselves. This module commits to one specific set of accepted -//! spellings; widening it to be everyone's-conversion-rules is not the -//! goal. -//! -//! ## Accepted spellings -//! -//! - **BigInt**: signed decimal integer fitting in `i64`. No leading `+`, -//! no whitespace, no separators, no scientific notation. -//! - **Double**: decimal or scientific notation parseable as `f64`, -//! including `inf`/`nan` spellings Rust's float parser accepts. -//! - **Date**: `YYYY-MM-DD` (RFC 3339 `full-date`), validated against -//! the proleptic Gregorian calendar. -//! - **Timestamp**: `YYYY-MM-DD(T| )HH:MM:SS[.fraction](Z|±HH:MM)` -//! (RFC 3339 `date-time`). Offset is required. -//! -//! ## Stability -//! -//! New module in a 0.0.x crate. The set of types and the accepted -//! spellings are not yet settled — both may change before 1.0. - -// ── Type enum ──────────────────────────────────────────────────────── - -/// Column type produced by inference, or accepted by per-cell casts. -/// -/// Variants are listed in the order [`DEFAULT_INFERENCE_ORDER`] tries -/// them. [`Type::Varchar`] always accepts and so sits last. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum Type { - BigInt, - Double, - Date, - Timestamp, - Varchar, -} - -impl Type { - /// Parse a non-empty cell into a typed value, or return `None` on failure. - /// - /// Empty input is always rejected (NSV uses empty cells for NULL). - pub fn try_cast<'a>(&self, s: &'a str) -> Option> { - if s.is_empty() { - return None; - } - match self { - Type::BigInt => try_parse_bigint(s).map(Value::BigInt), - Type::Double => try_parse_double(s).map(Value::Double), - Type::Date => try_parse_date(s).map(Value::Date), - Type::Timestamp => try_parse_timestamp(s).map(Value::Timestamp), - Type::Varchar => Some(Value::Varchar(s)), - } - } - - /// `true` if [`Type::try_cast`] would succeed. - pub fn accepts(&self, s: &str) -> bool { - self.try_cast(s).is_some() - } -} - -// ── Value enum ─────────────────────────────────────────────────────── - -/// Result of [`Type::try_cast`]. `Varchar` borrows from the input. -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum Value<'a> { - BigInt(i64), - Double(f64), - Date(Date), - Timestamp(Timestamp), - Varchar(&'a str), -} - -/// RFC 3339 `full-date`: `YYYY-MM-DD` in the proleptic Gregorian calendar. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct Date { - pub year: i32, - pub month: u8, - pub day: u8, -} - -/// RFC 3339 `date-time`: date + time-of-day + UTC offset. -/// -/// `nanosecond` carries the fractional second (0 if absent). -/// `offset_minutes` is the offset from UTC in minutes (`-00:00` is preserved -/// as `Some(0)`, distinct from `+00:00`/`Z` only via the `unknown_offset` flag). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct Timestamp { - pub year: i32, - pub month: u8, - pub day: u8, - pub hour: u8, - pub minute: u8, - pub second: u8, - pub nanosecond: u32, - pub offset_minutes: i16, - /// `true` iff the original offset was the RFC 3339 §4.3 `-00:00` - /// "unknown local offset" sentinel. - pub unknown_offset: bool, -} - -// ── Per-type parsers ───────────────────────────────────────────────── - -/// Parse as `i64` via [`str::parse`]. -pub fn try_parse_bigint(s: &str) -> Option { - s.parse::().ok() -} - -/// Parse as `f64` via [`str::parse`]. -pub fn try_parse_double(s: &str) -> Option { - s.parse::().ok() -} - -/// Parse RFC 3339 `full-date`: `YYYY-MM-DD`. -pub fn try_parse_date(s: &str) -> Option { - let b = s.as_bytes(); - if b.len() != 10 || b[4] != b'-' || b[7] != b'-' { - return None; - } - let year = parse_u32(&b[0..4])? as i32; - let month = parse_u32(&b[5..7])? as u8; - let day = parse_u32(&b[8..10])? as u8; - if !is_valid_ymd(year, month, day) { - return None; - } - Some(Date { year, month, day }) -} - -/// Parse RFC 3339 `date-time`: `YYYY-MM-DD(T| )HH:MM:SS[.fraction](Z|±HH:MM)`. -/// -/// Per RFC 3339 §5.6, the date/time separator may be uppercase `T` or a -/// space; we accept both. The offset is required. -pub fn try_parse_timestamp(s: &str) -> Option { - let b = s.as_bytes(); - if b.len() < 20 { - return None; - } - if b[4] != b'-' || b[7] != b'-' { - return None; - } - if b[10] != b'T' && b[10] != b' ' { - return None; - } - if b[13] != b':' || b[16] != b':' { - return None; - } - - let year = parse_u32(&b[0..4])? as i32; - let month = parse_u32(&b[5..7])? as u8; - let day = parse_u32(&b[8..10])? as u8; - let hour = parse_u32(&b[11..13])? as u8; - let minute = parse_u32(&b[14..16])? as u8; - let second = parse_u32(&b[17..19])? as u8; - - if !is_valid_ymd(year, month, day) { - return None; - } - if hour > 23 || minute > 59 || second > 60 { - // RFC 3339 §5.6 allows second=60 for leap seconds. - return None; - } - - let mut i = 19; - let mut nanosecond: u32 = 0; - if i < b.len() && b[i] == b'.' { - i += 1; - let frac_start = i; - while i < b.len() && b[i].is_ascii_digit() { - i += 1; - } - let frac = &b[frac_start..i]; - if frac.is_empty() { - return None; - } - nanosecond = scale_fraction_to_nanos(frac)?; - } - - let (offset_minutes, unknown_offset) = parse_offset(&b[i..])?; - - Some(Timestamp { - year, - month, - day, - hour, - minute, - second, - nanosecond, - offset_minutes, - unknown_offset, - }) -} - -fn parse_offset(b: &[u8]) -> Option<(i16, bool)> { - if b == b"Z" { - return Some((0, false)); - } - if b.len() != 6 { - return None; - } - let sign = match b[0] { - b'+' => 1i16, - b'-' => -1i16, - _ => return None, - }; - if b[3] != b':' { - return None; - } - let hh = parse_u32(&b[1..3])?; - let mm = parse_u32(&b[4..6])?; - if hh > 23 || mm > 59 { - return None; - } - let total = sign * (hh as i16 * 60 + mm as i16); - let unknown = sign == -1 && hh == 0 && mm == 0; - Some((total, unknown)) -} - -fn scale_fraction_to_nanos(digits: &[u8]) -> Option { - // Use up to 9 fractional digits; truncate the rest. RFC 3339 doesn't - // bound the fraction length, but anything past nanoseconds is dropped. - let take = digits.len().min(9); - let mut value: u32 = 0; - for &d in &digits[..take] { - if !d.is_ascii_digit() { - return None; - } - value = value * 10 + (d - b'0') as u32; - } - for &d in &digits[take..] { - if !d.is_ascii_digit() { - return None; - } - } - for _ in take..9 { - value *= 10; - } - Some(value) -} - -fn parse_u32(b: &[u8]) -> Option { - let mut v: u32 = 0; - for &c in b { - if !c.is_ascii_digit() { - return None; - } - v = v.checked_mul(10)?.checked_add((c - b'0') as u32)?; - } - Some(v) -} - -fn is_valid_ymd(year: i32, month: u8, day: u8) -> bool { - if !(1..=12).contains(&month) || day < 1 { - return false; - } - let max = days_in_month(year, month); - day <= max -} - -fn days_in_month(year: i32, month: u8) -> u8 { - match month { - 1 | 3 | 5 | 7 | 8 | 10 | 12 => 31, - 4 | 6 | 9 | 11 => 30, - 2 => { - if is_leap_year(year) { - 29 - } else { - 28 - } - } - _ => 0, - } -} - -fn is_leap_year(year: i32) -> bool { - (year % 4 == 0 && year % 100 != 0) || year % 400 == 0 -} - -// ── Inference ──────────────────────────────────────────────────────── - -/// Order in which [`infer_column`] tries candidate types. The first -/// candidate that accepts every non-empty cell wins. -pub const DEFAULT_INFERENCE_ORDER: &[Type] = &[ - Type::BigInt, - Type::Double, - Type::Date, - Type::Timestamp, - Type::Varchar, -]; - -/// Recommended sample size for column inference. -pub const DEFAULT_SAMPLE_SIZE: usize = 1000; - -/// Infer a column type from a sample of cells using [`DEFAULT_INFERENCE_ORDER`]. -/// -/// Empty cells are treated as NULL and skipped. An all-NULL sample -/// resolves to [`Type::Varchar`]. Callers should pre-truncate to -/// [`DEFAULT_SAMPLE_SIZE`] cells. -pub fn infer_column(cells: I) -> Type -where - I: IntoIterator, - S: AsRef, -{ - infer_column_with(cells, DEFAULT_INFERENCE_ORDER) -} - -/// Infer a column type using a caller-supplied candidate order. -/// -/// The first candidate that accepts every non-empty cell wins; if no -/// candidate accepts every cell, returns [`Type::Varchar`]. If `order` -/// is empty, returns [`Type::Varchar`]. -pub fn infer_column_with(cells: I, order: &[Type]) -> Type -where - I: IntoIterator, - S: AsRef, -{ - // Materialize once; we walk the sample per candidate. - let sample: Vec = cells.into_iter().collect(); - let non_empty: Vec<&str> = sample - .iter() - .map(|s| s.as_ref()) - .filter(|s| !s.is_empty()) - .collect(); - - if non_empty.is_empty() { - return Type::Varchar; - } - - for &candidate in order { - if candidate == Type::Varchar { - return Type::Varchar; - } - if non_empty.iter().all(|s| candidate.accepts(s)) { - return candidate; - } - } - Type::Varchar -} - -// ── Tests ──────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - - // BigInt - #[test] - fn bigint_round_numbers() { - assert_eq!(try_parse_bigint("0"), Some(0)); - assert_eq!(try_parse_bigint("-1"), Some(-1)); - assert_eq!(try_parse_bigint("9223372036854775807"), Some(i64::MAX)); - assert_eq!(try_parse_bigint("-9223372036854775808"), Some(i64::MIN)); - } - - #[test] - fn bigint_rejects_overflow_and_junk() { - assert_eq!(try_parse_bigint("9223372036854775808"), None); - assert_eq!(try_parse_bigint("1.0"), None); - assert_eq!(try_parse_bigint("1e3"), None); - assert_eq!(try_parse_bigint(" 1"), None); - assert_eq!(try_parse_bigint("1_000"), None); - assert_eq!(try_parse_bigint(""), None); - } - - // Double - #[test] - fn double_round_numbers() { - assert_eq!(try_parse_double("0"), Some(0.0)); - assert_eq!(try_parse_double("2.5"), Some(2.5)); - assert_eq!(try_parse_double("-1e10"), Some(-1e10)); - } - - #[test] - fn double_rejects_junk() { - assert_eq!(try_parse_double(""), None); - assert_eq!(try_parse_double("abc"), None); - assert_eq!(try_parse_double("1,5"), None); - } - - // Date - #[test] - fn date_strict_rfc3339() { - let d = try_parse_date("2024-01-15").unwrap(); - assert_eq!((d.year, d.month, d.day), (2024, 1, 15)); - assert_eq!(try_parse_date("2024-02-29").map(|d| d.day), Some(29)); - assert_eq!(try_parse_date("2023-02-29"), None); - assert_eq!(try_parse_date("2000-02-29").map(|d| d.day), Some(29)); - assert_eq!(try_parse_date("1900-02-29"), None); - } - - #[test] - fn date_rejects_alternate_formats() { - assert_eq!(try_parse_date(""), None); - assert_eq!(try_parse_date("2024-1-15"), None); - assert_eq!(try_parse_date("2024/01/15"), None); - assert_eq!(try_parse_date("24-01-15"), None); - assert_eq!(try_parse_date("2024-13-01"), None); - assert_eq!(try_parse_date("2024-04-31"), None); - assert_eq!(try_parse_date("2024-01-15T"), None); - } - - // Timestamp - #[test] - fn timestamp_strict_rfc3339() { - let t = try_parse_timestamp("2024-01-15T12:34:56Z").unwrap(); - assert_eq!(t.hour, 12); - assert_eq!(t.offset_minutes, 0); - assert!(!t.unknown_offset); - - let t = try_parse_timestamp("2024-01-15T12:34:56.789+02:00").unwrap(); - assert_eq!(t.nanosecond, 789_000_000); - assert_eq!(t.offset_minutes, 120); - - let t = try_parse_timestamp("2024-01-15 12:34:56-05:30").unwrap(); - assert_eq!(t.offset_minutes, -(5 * 60 + 30)); - - let t = try_parse_timestamp("2024-01-15T12:34:56-00:00").unwrap(); - assert_eq!(t.offset_minutes, 0); - assert!(t.unknown_offset); - } - - #[test] - fn timestamp_fractional_truncates_past_nanos() { - let t = try_parse_timestamp("2024-01-15T00:00:00.1234567890123Z").unwrap(); - assert_eq!(t.nanosecond, 123_456_789); - } - - #[test] - fn timestamp_rejects_non_rfc3339() { - assert_eq!(try_parse_timestamp(""), None); - assert_eq!(try_parse_timestamp("2024-01-15T12:34:56"), None); // no offset - assert_eq!(try_parse_timestamp("2024-01-15T12:34Z"), None); - assert_eq!(try_parse_timestamp("2024-01-15T25:00:00Z"), None); - assert_eq!(try_parse_timestamp("2024-01-15T12:34:56.Z"), None); - assert_eq!(try_parse_timestamp("2024-01-15T12:34:56+25:00"), None); - } - - // try_cast / accepts - #[test] - fn try_cast_dispatches_per_type() { - assert!(matches!(Type::BigInt.try_cast("42"), Some(Value::BigInt(42)))); - assert!(matches!(Type::Double.try_cast("2.5"), Some(Value::Double(_)))); - assert!(matches!(Type::Varchar.try_cast("anything"), Some(Value::Varchar("anything")))); - assert!(Type::BigInt.try_cast("").is_none()); - assert!(Type::Varchar.try_cast("").is_none()); - } - - #[test] - fn accepts_matches_try_cast() { - for s in ["true", "42", "3.14", "2024-01-01", "2024-01-01T00:00:00Z", "x", ""] { - for t in DEFAULT_INFERENCE_ORDER { - assert_eq!(t.accepts(s), t.try_cast(s).is_some(), "{:?} on {:?}", t, s); - } - } - } - - // Inference - #[test] - fn infer_picks_most_specific() { - assert_eq!(infer_column(["1", "2", "3"]), Type::BigInt); - assert_eq!(infer_column(["1", "2.0", "3"]), Type::Double); - assert_eq!(infer_column(["2024-01-01", "2024-12-31"]), Type::Date); - assert_eq!( - infer_column(["2024-01-01T00:00:00Z", "2024-12-31T23:59:59Z"]), - Type::Timestamp - ); - assert_eq!(infer_column(["true", "false"]), Type::Varchar); - assert_eq!(infer_column(["a", "b"]), Type::Varchar); - } - - #[test] - fn infer_skips_empty_cells() { - assert_eq!(infer_column(["1", "", "2", ""]), Type::BigInt); - assert_eq!(infer_column(["", "", ""]), Type::Varchar); - let empty: [&str; 0] = []; - assert_eq!(infer_column(empty), Type::Varchar); - } - - #[test] - fn infer_falls_through_on_mixed() { - assert_eq!(infer_column(["1", "abc"]), Type::Varchar); - assert_eq!(infer_column(["1.0", "abc"]), Type::Varchar); - } - - #[test] - fn infer_with_custom_order() { - // Skip BigInt: "42" should fall through to Double. - let order = &[Type::Double, Type::Varchar]; - assert_eq!(infer_column_with(["42"], order), Type::Double); - assert_eq!(infer_column_with(["42"], &[]), Type::Varchar); - } -} From 2c974c31aad60f034f8c778e6178ce9ceb2a97d1 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Apr 2026 14:38:23 +0000 Subject: [PATCH 05/14] feat: per-column skip-unescape on projected decode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add decode_bytes_projected_with_unescape, taking a per-projected-column flag to skip the unescape pass for columns whose accepted spellings cannot contain '\\n' or '\\\\' (typically numeric / temporal cells under a schema). Skipped columns return Cow::Borrowed slices of the input — zero copy, zero allocation. Caller is responsible for pairing the flag with the column's declared type. decode_bytes_projected stays as-is, delegating to the new function with all-true flags. Sequential and parallel paths both thread the flags through. 7 new tests; 49 -> 55 total. --- src/lib.rs | 155 +++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 145 insertions(+), 10 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 4d06b13..9f688b9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,7 +16,6 @@ //! //! For smaller files, we use a sequential fast path to avoid thread overhead. -pub mod types; pub mod util; use memchr::memmem; @@ -271,20 +270,52 @@ fn build_col_map(columns: &[usize]) -> (Vec, usize) { /// /// Cells are returned as `Cow<[u8]>` — borrowed when no unescaping was needed. pub fn decode_bytes_projected<'a>(input: &'a [u8], columns: &[usize]) -> Vec>> { + decode_bytes_projected_with_unescape(input, columns, &vec![true; columns.len()]) +} + +/// Like [`decode_bytes_projected`], but takes a per-projected-column flag +/// indicating whether to unescape that column. Columns where +/// `unescape[i] == false` are returned as borrowed slices of `input`, +/// bypassing the unescape pass entirely. +/// +/// Skipping unescape is sound for any column whose accepted spellings +/// cannot contain `\n` or `\\` — e.g. numeric or temporal columns under +/// a schema that restricts them to a non-escaping subset of ASCII. +/// The caller is responsible for matching the flag to the column's +/// declared type; cells in a `false` column are returned verbatim and +/// will retain any escape sequences they contain. +/// +/// # Panics +/// +/// Panics if `unescape.len() != columns.len()`. +pub fn decode_bytes_projected_with_unescape<'a>( + input: &'a [u8], + columns: &[usize], + unescape: &[bool], +) -> Vec>> { + assert_eq!( + unescape.len(), + columns.len(), + "unescape flags must match projected columns" + ); if input.is_empty() || columns.is_empty() { return Vec::new(); } #[cfg(feature = "parallel")] if input.len() >= PARALLEL_THRESHOLD { - return decode_projected_parallel(input, columns); + return decode_projected_parallel(input, columns, unescape); } - decode_projected_sequential(input, columns) + decode_projected_sequential(input, columns, unescape) } /// Sequential single-pass projected decode. -fn decode_projected_sequential<'a>(input: &'a [u8], columns: &[usize]) -> Vec>> { +fn decode_projected_sequential<'a>( + input: &'a [u8], + columns: &[usize], + unescape: &[bool], +) -> Vec>> { let (col_map, max_col) = build_col_map(columns); let stride = columns.len(); let mut data: Vec>> = Vec::new(); @@ -299,7 +330,12 @@ fn decode_projected_sequential<'a>(input: &'a [u8], columns: &[usize]) -> Vec(input: &'a [u8], columns: &[usize]) -> Vec(input: &'a [u8], columns: &[usize]) -> Vec(input: &'a [u8], columns: &[usize]) -> Vec>> { +fn decode_projected_parallel<'a>( + input: &'a [u8], + columns: &[usize], + unescape: &[bool], +) -> Vec>> { let num_threads = rayon::current_num_threads(); let chunk_size = input.len() / num_threads; if chunk_size == 0 { - return decode_projected_sequential(input, columns); + return decode_projected_sequential(input, columns, unescape); } let finder = memmem::Finder::new(b"\n\n"); @@ -362,14 +407,14 @@ fn decode_projected_parallel<'a>(input: &'a [u8], columns: &[usize]) -> Vec = splits.windows(2).map(|w| &input[w[0]..w[1]]).collect(); let chunk_results: Vec>>> = chunks .par_iter() - .map(|chunk| decode_projected_sequential(chunk, columns)) + .map(|chunk| decode_projected_sequential(chunk, columns, unescape)) .collect(); let total_rows: usize = chunk_results.iter().map(|r| r.len()).sum(); @@ -1121,6 +1166,96 @@ mod tests { assert_eq!(projected_all, full); } + // ── Skip-unescape tests ── + + #[test] + fn test_skip_unescape_returns_raw_bytes() { + // Cell contains an escape sequence \\n (encoded as backslash-n). + // With unescape=false we should get the raw bytes back, including + // the backslash. With unescape=true we should get an actual LF. + let nsv = b"col\n\nLine 1\\nLine 2\n\n"; + + let raw = decode_bytes_projected_with_unescape(nsv, &[0], &[false]); + assert_eq!(raw[1][0].as_ref(), b"Line 1\\nLine 2"); + assert!(matches!(raw[1][0], Cow::Borrowed(_))); + + let unescaped = decode_bytes_projected_with_unescape(nsv, &[0], &[true]); + assert_eq!(unescaped[1][0].as_ref(), b"Line 1\nLine 2"); + } + + #[test] + fn test_skip_unescape_no_escapes_zero_copy() { + let nsv = b"name\nage\n\nAlice\n30\n\nBob\n25\n\n"; + let projected = decode_bytes_projected_with_unescape(nsv, &[0, 1], &[true, false]); + // The age column (no escapes ever) is borrowed regardless. + assert_eq!(projected[1][1].as_ref(), b"30"); + assert!(matches!(projected[1][1], Cow::Borrowed(_))); + // The name column unescapes — also borrowed since "Alice" has no escapes. + assert!(matches!(projected[1][0], Cow::Borrowed(_))); + } + + #[test] + fn test_skip_unescape_mixed_columns() { + // c0 needs unescape (has \\n), c1 is plain numeric, c2 has \\\\. + let nsv = b"c0\nc1\nc2\n\nA\\nB\n42\n\\\\\n\n"; + let projected = decode_bytes_projected_with_unescape( + nsv, + &[0, 1, 2], + &[true, false, false], + ); + assert_eq!(projected[1][0].as_ref(), b"A\nB"); // unescaped + assert_eq!(projected[1][1].as_ref(), b"42"); // raw, no escapes anyway + assert_eq!(projected[1][2].as_ref(), b"\\\\"); // raw, escapes preserved + } + + #[test] + fn test_skip_unescape_default_matches_all_true() { + let nsv = b"a\nb\n\n1\n2\n\n3\n4\n\n"; + let default = owned(decode_bytes_projected(nsv, &[0, 1])); + let explicit = owned(decode_bytes_projected_with_unescape( + nsv, + &[0, 1], + &[true, true], + )); + assert_eq!(default, explicit); + } + + #[test] + #[should_panic(expected = "unescape flags must match projected columns")] + fn test_skip_unescape_panics_on_length_mismatch() { + let nsv = b"a\nb\n\n"; + let _ = decode_bytes_projected_with_unescape(nsv, &[0, 1], &[true]); + } + + #[test] + fn test_skip_unescape_parallel() { + // Force the parallel path with > PARALLEL_THRESHOLD bytes. + let mut data = Vec::new(); + for i in 0..10_000 { + data.push(vec![ + format!("escaped\\n{}", i), // typed col would never look like this + format!("{}", i), // numeric, no escapes + ]); + } + // Encode raw — bypass nsv::encode so the literal backslash survives. + let mut buf = Vec::new(); + for row in &data { + for cell in row { + buf.extend_from_slice(cell.as_bytes()); + buf.push(b'\n'); + } + buf.push(b'\n'); + } + assert!(buf.len() > PARALLEL_THRESHOLD); + + let projected = decode_bytes_projected_with_unescape(&buf, &[0, 1], &[false, false]); + assert_eq!(projected.len(), data.len()); + for (i, row) in data.iter().enumerate() { + assert_eq!(projected[i][0].as_ref(), row[0].as_bytes()); + assert_eq!(projected[i][1].as_ref(), row[1].as_bytes()); + } + } + // ── Streaming tests ── use std::io::Cursor; From 22c1478ce5e5d7af05d67fffac43572232e11423 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Apr 2026 15:12:16 +0000 Subject: [PATCH 06/14] rework skip-unescape to use original-column indices MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per review (#14): - 'You materialize a vector of booleans just to say don't bother' - skip flags 'get mixed up with columns's potential permutation' decode_bytes_projected_with_unescape(input, columns, &[bool]) -> decode_bytes_projected_skip_unescape(input, columns, &[usize]) Indices in skip_unescape live in the original column space, same as columns itself, so permuting columns does not require permuting the skip set. Indices not present in columns are ignored. decode_bytes_projected stays the no-skip case, now calling through with &[] — no parallel array materialized at the API boundary. Internal lookup is a precomputed bool map sized to max(columns, skip_unescape). Tests updated; new tests cover the permutation case and the unknown-index case. --- src/lib.rs | 134 +++++++++++++++++++++++++++-------------------------- 1 file changed, 68 insertions(+), 66 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 9f688b9..c123e06 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -270,53 +270,63 @@ fn build_col_map(columns: &[usize]) -> (Vec, usize) { /// /// Cells are returned as `Cow<[u8]>` — borrowed when no unescaping was needed. pub fn decode_bytes_projected<'a>(input: &'a [u8], columns: &[usize]) -> Vec>> { - decode_bytes_projected_with_unescape(input, columns, &vec![true; columns.len()]) + decode_bytes_projected_skip_unescape(input, columns, &[]) } -/// Like [`decode_bytes_projected`], but takes a per-projected-column flag -/// indicating whether to unescape that column. Columns where -/// `unescape[i] == false` are returned as borrowed slices of `input`, -/// bypassing the unescape pass entirely. +/// Like [`decode_bytes_projected`], but skips the unescape pass for cells +/// in the columns listed in `skip_unescape`. Indices in `skip_unescape` +/// refer to the **original** column space (the same space `columns` is +/// drawn from), not the projection order, so permuting `columns` does +/// not require permuting `skip_unescape`. Indices not present in +/// `columns` are simply ignored. /// -/// Skipping unescape is sound for any column whose accepted spellings -/// cannot contain `\n` or `\\` — e.g. numeric or temporal columns under -/// a schema that restricts them to a non-escaping subset of ASCII. -/// The caller is responsible for matching the flag to the column's -/// declared type; cells in a `false` column are returned verbatim and -/// will retain any escape sequences they contain. -/// -/// # Panics -/// -/// Panics if `unescape.len() != columns.len()`. -pub fn decode_bytes_projected_with_unescape<'a>( +/// Cells in skipped columns are returned as `Cow::Borrowed` slices of +/// `input` — zero copy, zero allocation. Skipping is sound for any +/// column whose accepted spellings cannot contain `\n` or `\\` +/// (typically numeric or temporal cells under a schema). Cells in a +/// skipped column retain any escape sequences they contain; this is +/// intentional. +pub fn decode_bytes_projected_skip_unescape<'a>( input: &'a [u8], columns: &[usize], - unescape: &[bool], + skip_unescape: &[usize], ) -> Vec>> { - assert_eq!( - unescape.len(), - columns.len(), - "unescape flags must match projected columns" - ); if input.is_empty() || columns.is_empty() { return Vec::new(); } #[cfg(feature = "parallel")] if input.len() >= PARALLEL_THRESHOLD { - return decode_projected_parallel(input, columns, unescape); + return decode_projected_parallel(input, columns, skip_unescape); } - decode_projected_sequential(input, columns, unescape) + decode_projected_sequential(input, columns, skip_unescape) +} + +/// Build a `skip_map[col_idx] = true` lookup over original-column space, +/// sized to cover both `columns` and `skip_unescape`. +fn build_skip_map(columns: &[usize], skip_unescape: &[usize]) -> Vec { + let max_col = columns + .iter() + .chain(skip_unescape.iter()) + .copied() + .max() + .unwrap_or(0); + let mut map = vec![false; max_col + 1]; + for &c in skip_unescape { + map[c] = true; + } + map } /// Sequential single-pass projected decode. fn decode_projected_sequential<'a>( input: &'a [u8], columns: &[usize], - unescape: &[bool], + skip_unescape: &[usize], ) -> Vec>> { let (col_map, max_col) = build_col_map(columns); + let skip_map = build_skip_map(columns, skip_unescape); let stride = columns.len(); let mut data: Vec>> = Vec::new(); let mut row: Vec> = vec![Cow::Borrowed(b""); stride]; @@ -331,10 +341,10 @@ fn decode_projected_sequential<'a>( if let Some(&proj_idx) = col_map.get(col_idx) { if proj_idx != usize::MAX { let raw = &input[start..pos]; - row[proj_idx] = if unescape[proj_idx] { - unescape_bytes(raw) - } else { + row[proj_idx] = if skip_map.get(col_idx).copied().unwrap_or(false) { Cow::Borrowed(raw) + } else { + unescape_bytes(raw) }; } } @@ -358,10 +368,10 @@ fn decode_projected_sequential<'a>( if let Some(&proj_idx) = col_map.get(col_idx) { if proj_idx != usize::MAX { let raw = &input[start..]; - row[proj_idx] = if unescape[proj_idx] { - unescape_bytes(raw) - } else { + row[proj_idx] = if skip_map.get(col_idx).copied().unwrap_or(false) { Cow::Borrowed(raw) + } else { + unescape_bytes(raw) }; } } @@ -381,13 +391,13 @@ fn decode_projected_sequential<'a>( fn decode_projected_parallel<'a>( input: &'a [u8], columns: &[usize], - unescape: &[bool], + skip_unescape: &[usize], ) -> Vec>> { let num_threads = rayon::current_num_threads(); let chunk_size = input.len() / num_threads; if chunk_size == 0 { - return decode_projected_sequential(input, columns, unescape); + return decode_projected_sequential(input, columns, skip_unescape); } let finder = memmem::Finder::new(b"\n\n"); @@ -407,14 +417,14 @@ fn decode_projected_parallel<'a>( splits.dedup(); if splits.len() <= 2 { - return decode_projected_sequential(input, columns, unescape); + return decode_projected_sequential(input, columns, skip_unescape); } let chunks: Vec<&[u8]> = splits.windows(2).map(|w| &input[w[0]..w[1]]).collect(); let chunk_results: Vec>>> = chunks .par_iter() - .map(|chunk| decode_projected_sequential(chunk, columns, unescape)) + .map(|chunk| decode_projected_sequential(chunk, columns, skip_unescape)) .collect(); let total_rows: usize = chunk_results.iter().map(|r| r.len()).sum(); @@ -1171,62 +1181,54 @@ mod tests { #[test] fn test_skip_unescape_returns_raw_bytes() { // Cell contains an escape sequence \\n (encoded as backslash-n). - // With unescape=false we should get the raw bytes back, including - // the backslash. With unescape=true we should get an actual LF. + // Without skipping we get an actual LF; with skip we get the raw bytes. let nsv = b"col\n\nLine 1\\nLine 2\n\n"; - let raw = decode_bytes_projected_with_unescape(nsv, &[0], &[false]); + let raw = decode_bytes_projected_skip_unescape(nsv, &[0], &[0]); assert_eq!(raw[1][0].as_ref(), b"Line 1\\nLine 2"); assert!(matches!(raw[1][0], Cow::Borrowed(_))); - let unescaped = decode_bytes_projected_with_unescape(nsv, &[0], &[true]); + let unescaped = decode_bytes_projected_skip_unescape(nsv, &[0], &[]); assert_eq!(unescaped[1][0].as_ref(), b"Line 1\nLine 2"); } #[test] - fn test_skip_unescape_no_escapes_zero_copy() { - let nsv = b"name\nage\n\nAlice\n30\n\nBob\n25\n\n"; - let projected = decode_bytes_projected_with_unescape(nsv, &[0, 1], &[true, false]); - // The age column (no escapes ever) is borrowed regardless. - assert_eq!(projected[1][1].as_ref(), b"30"); - assert!(matches!(projected[1][1], Cow::Borrowed(_))); - // The name column unescapes — also borrowed since "Alice" has no escapes. - assert!(matches!(projected[1][0], Cow::Borrowed(_))); + fn test_skip_unescape_uses_original_column_indices() { + // c0 has an escape, c1 doesn't. Project in REVERSE order ([1, 0]) + // and skip c0 (original index 0). The escape in c0 should survive + // regardless of where it lands in the projection order. + let nsv = b"c0\nc1\n\nA\\nB\n42\n\n"; + let projected = decode_bytes_projected_skip_unescape(nsv, &[1, 0], &[0]); + assert_eq!(projected[1][0].as_ref(), b"42"); // c1 in slot 0 + assert_eq!(projected[1][1].as_ref(), b"A\\nB"); // c0 in slot 1, raw + } + + #[test] + fn test_skip_unescape_unknown_index_ignored() { + // c5 isn't in `columns`; listing it in skip_unescape is a no-op. + let nsv = b"c0\n\nA\\nB\n\n"; + let projected = decode_bytes_projected_skip_unescape(nsv, &[0], &[5]); + assert_eq!(projected[1][0].as_ref(), b"A\nB"); // c0 still unescaped } #[test] fn test_skip_unescape_mixed_columns() { // c0 needs unescape (has \\n), c1 is plain numeric, c2 has \\\\. let nsv = b"c0\nc1\nc2\n\nA\\nB\n42\n\\\\\n\n"; - let projected = decode_bytes_projected_with_unescape( - nsv, - &[0, 1, 2], - &[true, false, false], - ); + let projected = decode_bytes_projected_skip_unescape(nsv, &[0, 1, 2], &[1, 2]); assert_eq!(projected[1][0].as_ref(), b"A\nB"); // unescaped assert_eq!(projected[1][1].as_ref(), b"42"); // raw, no escapes anyway assert_eq!(projected[1][2].as_ref(), b"\\\\"); // raw, escapes preserved } #[test] - fn test_skip_unescape_default_matches_all_true() { + fn test_skip_unescape_empty_matches_default() { let nsv = b"a\nb\n\n1\n2\n\n3\n4\n\n"; let default = owned(decode_bytes_projected(nsv, &[0, 1])); - let explicit = owned(decode_bytes_projected_with_unescape( - nsv, - &[0, 1], - &[true, true], - )); + let explicit = owned(decode_bytes_projected_skip_unescape(nsv, &[0, 1], &[])); assert_eq!(default, explicit); } - #[test] - #[should_panic(expected = "unescape flags must match projected columns")] - fn test_skip_unescape_panics_on_length_mismatch() { - let nsv = b"a\nb\n\n"; - let _ = decode_bytes_projected_with_unescape(nsv, &[0, 1], &[true]); - } - #[test] fn test_skip_unescape_parallel() { // Force the parallel path with > PARALLEL_THRESHOLD bytes. @@ -1248,7 +1250,7 @@ mod tests { } assert!(buf.len() > PARALLEL_THRESHOLD); - let projected = decode_bytes_projected_with_unescape(&buf, &[0, 1], &[false, false]); + let projected = decode_bytes_projected_skip_unescape(&buf, &[0, 1], &[0, 1]); assert_eq!(projected.len(), data.len()); for (i, row) in data.iter().enumerate() { assert_eq!(projected[i][0].as_ref(), row[0].as_bytes()); From d8279f26a7c3b367b11ef6d12a00d530383bb39b Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Apr 2026 15:28:36 +0000 Subject: [PATCH 07/14] rework projected decode to take &[(usize, ColumnKind)] Per review (#14): - parallel arrays for offsets + skip flags are fragile - tuple shape collapses both into one sequence and survives permutation by construction - the enum is the natural place to grow type-aware decode later API: pub enum ColumnKind { String, Other } pub fn decode_bytes_projected(input, &[(usize, ColumnKind)]) -> Vec>> For now the kind only gates unescape: String unescapes, Other returns raw Cow::Borrowed slices. Other is the catch-all for non-string columns under a schema (numeric, temporal, ...) whose accepted spellings cannot contain '\\n' or '\\\\'. Removes decode_bytes_projected_skip_unescape entirely. Internal build_projection produces both col_map and unescape_map in one pass. Tests rewritten with s(c) / o(c) helpers; benches updated. 53 tests pass, clippy clean. --- benches/parse.rs | 22 +++--- src/lib.rs | 181 +++++++++++++++++++++-------------------------- 2 files changed, 93 insertions(+), 110 deletions(-) diff --git a/benches/parse.rs b/benches/parse.rs index e7b7b14..63b1706 100644 --- a/benches/parse.rs +++ b/benches/parse.rs @@ -1,5 +1,7 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion, BenchmarkId}; -use nsv::{encode, decode, decode_bytes, decode_bytes_projected}; +use nsv::{encode, decode, decode_bytes, decode_bytes_projected, ColumnKind}; + +fn s(c: usize) -> (usize, ColumnKind) { (c, ColumnKind::String) } fn generate_test_data(rows: usize, cells_per_row: usize) -> Vec> { (0..rows) @@ -84,19 +86,19 @@ fn bench_projection_10k(c: &mut Criterion) { }); group.bench_function("projected_1_of_10", |b| { - b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &[0])) + b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &[s(0)])) }); group.bench_function("projected_2_of_10", |b| { - b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &[0, 5])) + b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &[s(0), s(5)])) }); group.bench_function("projected_5_of_10", |b| { - b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &[0, 2, 4, 6, 8])) + b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &[s(0), s(2), s(4), s(6), s(8)])) }); group.bench_function("projected_all_10", |b| { - b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9])) + b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &[s(0), s(1), s(2), s(3), s(4), s(5), s(6), s(7), s(8), s(9)])) }); group.finish(); @@ -114,11 +116,11 @@ fn bench_projection_100k(c: &mut Criterion) { }); group.bench_function("projected_1_of_10", |b| { - b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &[0])) + b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &[s(0)])) }); group.bench_function("projected_2_of_10", |b| { - b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &[0, 5])) + b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &[s(0), s(5)])) }); group.finish(); @@ -137,15 +139,15 @@ fn bench_projection_wide(c: &mut Criterion) { }); group.bench_function("projected_1_of_100", |b| { - b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &[50])) + b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &[s(50)])) }); group.bench_function("projected_5_of_100", |b| { - b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &[0, 25, 50, 75, 99])) + b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &[s(0), s(25), s(50), s(75), s(99)])) }); group.bench_function("projected_10_of_100", |b| { - b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &[0, 10, 20, 30, 40, 50, 60, 70, 80, 90])) + b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &[s(0), s(10), s(20), s(30), s(40), s(50), s(60), s(70), s(80), s(90)])) }); group.finish(); diff --git a/src/lib.rs b/src/lib.rs index c123e06..7efdef1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -251,45 +251,56 @@ pub fn escape_bytes(s: &[u8]) -> Cow<'_, [u8]> { // columns entirely (no allocation, no unescape), and directly produces // the final `Vec>>`. -/// Build a column-map: `col_map[original_col] = projected_index`. -/// Entries for non-projected columns are `usize::MAX`. -fn build_col_map(columns: &[usize]) -> (Vec, usize) { - let max_col = columns.iter().copied().max().unwrap_or(0); +/// Column kind for projected decoding. +/// +/// Used to gate per-column unescape: only [`ColumnKind::String`] cells +/// need to interpret `\n` and `\\`. [`ColumnKind::Other`] is the catch-all +/// for non-string columns under a schema (numeric, temporal, …) whose +/// accepted spellings cannot contain `\n` or `\\` and so are returned +/// raw — zero copy. +/// +/// More variants may be added as the projected-decode API grows; for +/// now their only effect is on unescape. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ColumnKind { + /// NSV string column — cells go through the unescape pass. + String, + /// Non-string column — cells are returned raw, escape sequences and all. + Other, +} + +/// Per-column projection: `(original_col, kind, unescape)` — built once +/// from a `&[(usize, ColumnKind)]` and indexed by original column. +fn build_projection( + columns: &[(usize, ColumnKind)], +) -> (Vec, Vec, usize) { + let max_col = columns.iter().map(|&(c, _)| c).max().unwrap_or(0); let mut col_map = vec![usize::MAX; max_col + 1]; - for (proj_idx, &orig_col) in columns.iter().enumerate() { + let mut unescape_map = vec![false; max_col + 1]; + for (proj_idx, &(orig_col, kind)) in columns.iter().enumerate() { col_map[orig_col] = proj_idx; + unescape_map[orig_col] = matches!(kind, ColumnKind::String); } - (col_map, max_col) + (col_map, unescape_map, max_col) } /// Decode only the specified columns from raw bytes. /// -/// Single-pass: scans for cell/row boundaries and directly unescapes -/// only the cells in projected columns. No intermediate structural index. -/// Each inner vec has exactly `columns.len()` entries (same order as `columns`). +/// Each entry of `columns` pairs an original-column index with its +/// [`ColumnKind`]. Only [`ColumnKind::String`] cells are unescaped; +/// [`ColumnKind::Other`] cells are returned as borrowed slices of `input` +/// — zero copy, zero allocation. /// -/// Cells are returned as `Cow<[u8]>` — borrowed when no unescaping was needed. -pub fn decode_bytes_projected<'a>(input: &'a [u8], columns: &[usize]) -> Vec>> { - decode_bytes_projected_skip_unescape(input, columns, &[]) -} - -/// Like [`decode_bytes_projected`], but skips the unescape pass for cells -/// in the columns listed in `skip_unescape`. Indices in `skip_unescape` -/// refer to the **original** column space (the same space `columns` is -/// drawn from), not the projection order, so permuting `columns` does -/// not require permuting `skip_unescape`. Indices not present in -/// `columns` are simply ignored. +/// Single-pass: scans for cell/row boundaries and writes directly into +/// the projected output. Each inner vec has exactly `columns.len()` +/// entries (same order as `columns`). /// -/// Cells in skipped columns are returned as `Cow::Borrowed` slices of -/// `input` — zero copy, zero allocation. Skipping is sound for any -/// column whose accepted spellings cannot contain `\n` or `\\` -/// (typically numeric or temporal cells under a schema). Cells in a -/// skipped column retain any escape sequences they contain; this is -/// intentional. -pub fn decode_bytes_projected_skip_unescape<'a>( +/// Cells are returned as `Cow<[u8]>` — borrowed when no unescaping was +/// needed (always, for `Other`; opportunistically, for `String` cells +/// without escape sequences). +pub fn decode_bytes_projected<'a>( input: &'a [u8], - columns: &[usize], - skip_unescape: &[usize], + columns: &[(usize, ColumnKind)], ) -> Vec>> { if input.is_empty() || columns.is_empty() { return Vec::new(); @@ -297,36 +308,18 @@ pub fn decode_bytes_projected_skip_unescape<'a>( #[cfg(feature = "parallel")] if input.len() >= PARALLEL_THRESHOLD { - return decode_projected_parallel(input, columns, skip_unescape); + return decode_projected_parallel(input, columns); } - decode_projected_sequential(input, columns, skip_unescape) -} - -/// Build a `skip_map[col_idx] = true` lookup over original-column space, -/// sized to cover both `columns` and `skip_unescape`. -fn build_skip_map(columns: &[usize], skip_unescape: &[usize]) -> Vec { - let max_col = columns - .iter() - .chain(skip_unescape.iter()) - .copied() - .max() - .unwrap_or(0); - let mut map = vec![false; max_col + 1]; - for &c in skip_unescape { - map[c] = true; - } - map + decode_projected_sequential(input, columns) } /// Sequential single-pass projected decode. fn decode_projected_sequential<'a>( input: &'a [u8], - columns: &[usize], - skip_unescape: &[usize], + columns: &[(usize, ColumnKind)], ) -> Vec>> { - let (col_map, max_col) = build_col_map(columns); - let skip_map = build_skip_map(columns, skip_unescape); + let (col_map, unescape_map, max_col) = build_projection(columns); let stride = columns.len(); let mut data: Vec>> = Vec::new(); let mut row: Vec> = vec![Cow::Borrowed(b""); stride]; @@ -341,10 +334,10 @@ fn decode_projected_sequential<'a>( if let Some(&proj_idx) = col_map.get(col_idx) { if proj_idx != usize::MAX { let raw = &input[start..pos]; - row[proj_idx] = if skip_map.get(col_idx).copied().unwrap_or(false) { - Cow::Borrowed(raw) - } else { + row[proj_idx] = if unescape_map[col_idx] { unescape_bytes(raw) + } else { + Cow::Borrowed(raw) }; } } @@ -368,10 +361,10 @@ fn decode_projected_sequential<'a>( if let Some(&proj_idx) = col_map.get(col_idx) { if proj_idx != usize::MAX { let raw = &input[start..]; - row[proj_idx] = if skip_map.get(col_idx).copied().unwrap_or(false) { - Cow::Borrowed(raw) - } else { + row[proj_idx] = if unescape_map[col_idx] { unescape_bytes(raw) + } else { + Cow::Borrowed(raw) }; } } @@ -390,14 +383,13 @@ fn decode_projected_sequential<'a>( #[cfg(feature = "parallel")] fn decode_projected_parallel<'a>( input: &'a [u8], - columns: &[usize], - skip_unescape: &[usize], + columns: &[(usize, ColumnKind)], ) -> Vec>> { let num_threads = rayon::current_num_threads(); let chunk_size = input.len() / num_threads; if chunk_size == 0 { - return decode_projected_sequential(input, columns, skip_unescape); + return decode_projected_sequential(input, columns); } let finder = memmem::Finder::new(b"\n\n"); @@ -417,14 +409,14 @@ fn decode_projected_parallel<'a>( splits.dedup(); if splits.len() <= 2 { - return decode_projected_sequential(input, columns, skip_unescape); + return decode_projected_sequential(input, columns); } let chunks: Vec<&[u8]> = splits.windows(2).map(|w| &input[w[0]..w[1]]).collect(); let chunk_results: Vec>>> = chunks .par_iter() - .map(|chunk| decode_projected_sequential(chunk, columns, skip_unescape)) + .map(|chunk| decode_projected_sequential(chunk, columns)) .collect(); let total_rows: usize = chunk_results.iter().map(|r| r.len()).sum(); @@ -1105,10 +1097,15 @@ mod tests { // ── Projected decode tests ── + /// Test helper: project column `c` as a String column (i.e. unescape). + fn s(c: usize) -> (usize, ColumnKind) { (c, ColumnKind::String) } + /// Test helper: project column `c` as Other (i.e. raw, no unescape). + fn o(c: usize) -> (usize, ColumnKind) { (c, ColumnKind::Other) } + #[test] fn test_project_subset() { let nsv = b"c0\nc1\nc2\nc3\n\na\nb\nc\nd\n\ne\nf\ng\nh\n\n"; - let projected = owned(decode_bytes_projected(nsv, &[0, 2])); + let projected = owned(decode_bytes_projected(nsv, &[s(0), s(2)])); assert_eq!(projected.len(), 3); assert_eq!(projected[0], vec![b"c0".to_vec(), b"c2".to_vec()]); assert_eq!(projected[1], vec![b"a".to_vec(), b"c".to_vec()]); @@ -1118,7 +1115,7 @@ mod tests { #[test] fn test_project_single_column() { let nsv = b"name\nage\nsalary\n\nAlice\n30\n50000\n\nBob\n25\n75000\n\n"; - let projected = owned(decode_bytes_projected(nsv, &[1])); + let projected = owned(decode_bytes_projected(nsv, &[s(1)])); assert_eq!(projected.len(), 3); assert_eq!(projected[0], vec![b"age".to_vec()]); assert_eq!(projected[1], vec![b"30".to_vec()]); @@ -1128,7 +1125,7 @@ mod tests { #[test] fn test_project_reorder() { let nsv = b"a\nb\nc\n\n1\n2\n3\n\n"; - let projected = owned(decode_bytes_projected(nsv, &[2, 0])); + let projected = owned(decode_bytes_projected(nsv, &[s(2), s(0)])); assert_eq!(projected[0], vec![b"c".to_vec(), b"a".to_vec()]); assert_eq!(projected[1], vec![b"3".to_vec(), b"1".to_vec()]); } @@ -1136,7 +1133,7 @@ mod tests { #[test] fn test_project_out_of_range() { let nsv = b"a\nb\n\n"; - let projected = owned(decode_bytes_projected(nsv, &[0, 5])); + let projected = owned(decode_bytes_projected(nsv, &[s(0), s(5)])); assert_eq!(projected[0], vec![b"a".to_vec(), b"".to_vec()]); } @@ -1144,7 +1141,7 @@ mod tests { fn test_projected_matches_full() { let nsv = b"c0\nc1\nc2\n\na\nb\nc\n\n"; let full = owned(decode_bytes(nsv)); - let projected = owned(decode_bytes_projected(nsv, &[0, 1, 2])); + let projected = owned(decode_bytes_projected(nsv, &[s(0), s(1), s(2)])); assert_eq!(projected, full); } @@ -1162,7 +1159,7 @@ mod tests { let encoded_bytes = encoded.as_bytes(); assert!(encoded_bytes.len() > PARALLEL_THRESHOLD); - let projected = decode_bytes_projected(encoded_bytes, &[2]); + let projected = decode_bytes_projected(encoded_bytes, &[s(2)]); assert_eq!(projected.len(), data.len()); for (ri, row) in data.iter().enumerate() { assert_eq!( @@ -1172,65 +1169,49 @@ mod tests { } let full = owned(decode_bytes(encoded_bytes)); - let projected_all = owned(decode_bytes_projected(encoded_bytes, &[0, 1, 2])); + let projected_all = owned(decode_bytes_projected(encoded_bytes, &[s(0), s(1), s(2)])); assert_eq!(projected_all, full); } - // ── Skip-unescape tests ── + // ── ColumnKind::Other (skip-unescape) tests ── #[test] - fn test_skip_unescape_returns_raw_bytes() { + fn test_other_returns_raw_bytes() { // Cell contains an escape sequence \\n (encoded as backslash-n). - // Without skipping we get an actual LF; with skip we get the raw bytes. + // Other returns raw bytes; String unescapes. let nsv = b"col\n\nLine 1\\nLine 2\n\n"; - let raw = decode_bytes_projected_skip_unescape(nsv, &[0], &[0]); + let raw = decode_bytes_projected(nsv, &[o(0)]); assert_eq!(raw[1][0].as_ref(), b"Line 1\\nLine 2"); assert!(matches!(raw[1][0], Cow::Borrowed(_))); - let unescaped = decode_bytes_projected_skip_unescape(nsv, &[0], &[]); + let unescaped = decode_bytes_projected(nsv, &[s(0)]); assert_eq!(unescaped[1][0].as_ref(), b"Line 1\nLine 2"); } #[test] - fn test_skip_unescape_uses_original_column_indices() { - // c0 has an escape, c1 doesn't. Project in REVERSE order ([1, 0]) - // and skip c0 (original index 0). The escape in c0 should survive - // regardless of where it lands in the projection order. + fn test_other_kind_independent_of_projection_order() { + // c0 has an escape, c1 doesn't. Project in REVERSE order with + // c0 as Other (raw). The escape should survive regardless of + // where c0 lands in the projection. let nsv = b"c0\nc1\n\nA\\nB\n42\n\n"; - let projected = decode_bytes_projected_skip_unescape(nsv, &[1, 0], &[0]); - assert_eq!(projected[1][0].as_ref(), b"42"); // c1 in slot 0 + let projected = decode_bytes_projected(nsv, &[s(1), o(0)]); + assert_eq!(projected[1][0].as_ref(), b"42"); // c1 in slot 0, unescaped assert_eq!(projected[1][1].as_ref(), b"A\\nB"); // c0 in slot 1, raw } #[test] - fn test_skip_unescape_unknown_index_ignored() { - // c5 isn't in `columns`; listing it in skip_unescape is a no-op. - let nsv = b"c0\n\nA\\nB\n\n"; - let projected = decode_bytes_projected_skip_unescape(nsv, &[0], &[5]); - assert_eq!(projected[1][0].as_ref(), b"A\nB"); // c0 still unescaped - } - - #[test] - fn test_skip_unescape_mixed_columns() { + fn test_mixed_kinds() { // c0 needs unescape (has \\n), c1 is plain numeric, c2 has \\\\. let nsv = b"c0\nc1\nc2\n\nA\\nB\n42\n\\\\\n\n"; - let projected = decode_bytes_projected_skip_unescape(nsv, &[0, 1, 2], &[1, 2]); + let projected = decode_bytes_projected(nsv, &[s(0), o(1), o(2)]); assert_eq!(projected[1][0].as_ref(), b"A\nB"); // unescaped assert_eq!(projected[1][1].as_ref(), b"42"); // raw, no escapes anyway assert_eq!(projected[1][2].as_ref(), b"\\\\"); // raw, escapes preserved } #[test] - fn test_skip_unescape_empty_matches_default() { - let nsv = b"a\nb\n\n1\n2\n\n3\n4\n\n"; - let default = owned(decode_bytes_projected(nsv, &[0, 1])); - let explicit = owned(decode_bytes_projected_skip_unescape(nsv, &[0, 1], &[])); - assert_eq!(default, explicit); - } - - #[test] - fn test_skip_unescape_parallel() { + fn test_other_parallel() { // Force the parallel path with > PARALLEL_THRESHOLD bytes. let mut data = Vec::new(); for i in 0..10_000 { @@ -1250,7 +1231,7 @@ mod tests { } assert!(buf.len() > PARALLEL_THRESHOLD); - let projected = decode_bytes_projected_skip_unescape(&buf, &[0, 1], &[0, 1]); + let projected = decode_bytes_projected(&buf, &[o(0), o(1)]); assert_eq!(projected.len(), data.len()); for (i, row) in data.iter().enumerate() { assert_eq!(projected[i][0].as_ref(), row[0].as_bytes()); From 33519686486015605007ae1c8368fa053d22c1ad Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Apr 2026 15:41:25 +0000 Subject: [PATCH 08/14] rename ColumnKind -> ColumnType per review --- benches/parse.rs | 4 ++-- src/lib.rs | 28 ++++++++++++++-------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/benches/parse.rs b/benches/parse.rs index 63b1706..8635170 100644 --- a/benches/parse.rs +++ b/benches/parse.rs @@ -1,7 +1,7 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion, BenchmarkId}; -use nsv::{encode, decode, decode_bytes, decode_bytes_projected, ColumnKind}; +use nsv::{encode, decode, decode_bytes, decode_bytes_projected, ColumnType}; -fn s(c: usize) -> (usize, ColumnKind) { (c, ColumnKind::String) } +fn s(c: usize) -> (usize, ColumnType) { (c, ColumnType::String) } fn generate_test_data(rows: usize, cells_per_row: usize) -> Vec> { (0..rows) diff --git a/src/lib.rs b/src/lib.rs index 7efdef1..c723625 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -253,8 +253,8 @@ pub fn escape_bytes(s: &[u8]) -> Cow<'_, [u8]> { /// Column kind for projected decoding. /// -/// Used to gate per-column unescape: only [`ColumnKind::String`] cells -/// need to interpret `\n` and `\\`. [`ColumnKind::Other`] is the catch-all +/// Used to gate per-column unescape: only [`ColumnType::String`] cells +/// need to interpret `\n` and `\\`. [`ColumnType::Other`] is the catch-all /// for non-string columns under a schema (numeric, temporal, …) whose /// accepted spellings cannot contain `\n` or `\\` and so are returned /// raw — zero copy. @@ -262,7 +262,7 @@ pub fn escape_bytes(s: &[u8]) -> Cow<'_, [u8]> { /// More variants may be added as the projected-decode API grows; for /// now their only effect is on unescape. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum ColumnKind { +pub enum ColumnType { /// NSV string column — cells go through the unescape pass. String, /// Non-string column — cells are returned raw, escape sequences and all. @@ -270,16 +270,16 @@ pub enum ColumnKind { } /// Per-column projection: `(original_col, kind, unescape)` — built once -/// from a `&[(usize, ColumnKind)]` and indexed by original column. +/// from a `&[(usize, ColumnType)]` and indexed by original column. fn build_projection( - columns: &[(usize, ColumnKind)], + columns: &[(usize, ColumnType)], ) -> (Vec, Vec, usize) { let max_col = columns.iter().map(|&(c, _)| c).max().unwrap_or(0); let mut col_map = vec![usize::MAX; max_col + 1]; let mut unescape_map = vec![false; max_col + 1]; for (proj_idx, &(orig_col, kind)) in columns.iter().enumerate() { col_map[orig_col] = proj_idx; - unescape_map[orig_col] = matches!(kind, ColumnKind::String); + unescape_map[orig_col] = matches!(kind, ColumnType::String); } (col_map, unescape_map, max_col) } @@ -287,8 +287,8 @@ fn build_projection( /// Decode only the specified columns from raw bytes. /// /// Each entry of `columns` pairs an original-column index with its -/// [`ColumnKind`]. Only [`ColumnKind::String`] cells are unescaped; -/// [`ColumnKind::Other`] cells are returned as borrowed slices of `input` +/// [`ColumnType`]. Only [`ColumnType::String`] cells are unescaped; +/// [`ColumnType::Other`] cells are returned as borrowed slices of `input` /// — zero copy, zero allocation. /// /// Single-pass: scans for cell/row boundaries and writes directly into @@ -300,7 +300,7 @@ fn build_projection( /// without escape sequences). pub fn decode_bytes_projected<'a>( input: &'a [u8], - columns: &[(usize, ColumnKind)], + columns: &[(usize, ColumnType)], ) -> Vec>> { if input.is_empty() || columns.is_empty() { return Vec::new(); @@ -317,7 +317,7 @@ pub fn decode_bytes_projected<'a>( /// Sequential single-pass projected decode. fn decode_projected_sequential<'a>( input: &'a [u8], - columns: &[(usize, ColumnKind)], + columns: &[(usize, ColumnType)], ) -> Vec>> { let (col_map, unescape_map, max_col) = build_projection(columns); let stride = columns.len(); @@ -383,7 +383,7 @@ fn decode_projected_sequential<'a>( #[cfg(feature = "parallel")] fn decode_projected_parallel<'a>( input: &'a [u8], - columns: &[(usize, ColumnKind)], + columns: &[(usize, ColumnType)], ) -> Vec>> { let num_threads = rayon::current_num_threads(); let chunk_size = input.len() / num_threads; @@ -1098,9 +1098,9 @@ mod tests { // ── Projected decode tests ── /// Test helper: project column `c` as a String column (i.e. unescape). - fn s(c: usize) -> (usize, ColumnKind) { (c, ColumnKind::String) } + fn s(c: usize) -> (usize, ColumnType) { (c, ColumnType::String) } /// Test helper: project column `c` as Other (i.e. raw, no unescape). - fn o(c: usize) -> (usize, ColumnKind) { (c, ColumnKind::Other) } + fn o(c: usize) -> (usize, ColumnType) { (c, ColumnType::Other) } #[test] fn test_project_subset() { @@ -1173,7 +1173,7 @@ mod tests { assert_eq!(projected_all, full); } - // ── ColumnKind::Other (skip-unescape) tests ── + // ── ColumnType::Other (skip-unescape) tests ── #[test] fn test_other_returns_raw_bytes() { From 4f7b024302900ddf37875e4938e97070bc5e1686 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Apr 2026 15:44:16 +0000 Subject: [PATCH 09/14] benches: replace per-element s() with vec-mapping ss() Per review (#14): bench call sites all wrap whole arrays of indices, so a per-element helper buys nothing. ss(&[...]) -> Vec<(usize, ColumnType)> reads cleaner. Tests keep s/o since they have mixed kinds. --- benches/parse.rs | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/benches/parse.rs b/benches/parse.rs index 8635170..256de85 100644 --- a/benches/parse.rs +++ b/benches/parse.rs @@ -1,7 +1,9 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion, BenchmarkId}; use nsv::{encode, decode, decode_bytes, decode_bytes_projected, ColumnType}; -fn s(c: usize) -> (usize, ColumnType) { (c, ColumnType::String) } +fn ss(cols: &[usize]) -> Vec<(usize, ColumnType)> { + cols.iter().map(|&c| (c, ColumnType::String)).collect() +} fn generate_test_data(rows: usize, cells_per_row: usize) -> Vec> { (0..rows) @@ -86,19 +88,19 @@ fn bench_projection_10k(c: &mut Criterion) { }); group.bench_function("projected_1_of_10", |b| { - b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &[s(0)])) + b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &ss(&[0]))) }); group.bench_function("projected_2_of_10", |b| { - b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &[s(0), s(5)])) + b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &ss(&[0, 5]))) }); group.bench_function("projected_5_of_10", |b| { - b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &[s(0), s(2), s(4), s(6), s(8)])) + b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &ss(&[0, 2, 4, 6, 8]))) }); group.bench_function("projected_all_10", |b| { - b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &[s(0), s(1), s(2), s(3), s(4), s(5), s(6), s(7), s(8), s(9)])) + b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &ss(&[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]))) }); group.finish(); @@ -116,11 +118,11 @@ fn bench_projection_100k(c: &mut Criterion) { }); group.bench_function("projected_1_of_10", |b| { - b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &[s(0)])) + b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &ss(&[0]))) }); group.bench_function("projected_2_of_10", |b| { - b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &[s(0), s(5)])) + b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &ss(&[0, 5]))) }); group.finish(); @@ -139,15 +141,15 @@ fn bench_projection_wide(c: &mut Criterion) { }); group.bench_function("projected_1_of_100", |b| { - b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &[s(50)])) + b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &ss(&[50]))) }); group.bench_function("projected_5_of_100", |b| { - b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &[s(0), s(25), s(50), s(75), s(99)])) + b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &ss(&[0, 25, 50, 75, 99]))) }); group.bench_function("projected_10_of_100", |b| { - b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &[s(0), s(10), s(20), s(30), s(40), s(50), s(60), s(70), s(80), s(90)])) + b.iter(|| decode_bytes_projected(black_box(nsv_bytes), &ss(&[0, 10, 20, 30, 40, 50, 60, 70, 80, 90]))) }); group.finish(); From 18aa0c763d6bb5dc2f2d3d28ee4cc1ec3c42ae85 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Apr 2026 15:47:08 +0000 Subject: [PATCH 10/14] rename ColumnType::Other -> ColumnType::Raw Per review: 'Other' positions relative to String and rules out future non-string types that might need unescaping. 'Raw' describes the actual semantics (cells returned verbatim, no unescape pass). Variant + doc-comments + tests + helpers all renamed; no semantic change. --- src/lib.rs | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index c723625..9baa109 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -251,22 +251,20 @@ pub fn escape_bytes(s: &[u8]) -> Cow<'_, [u8]> { // columns entirely (no allocation, no unescape), and directly produces // the final `Vec>>`. -/// Column kind for projected decoding. +/// Column type for projected decoding. /// -/// Used to gate per-column unescape: only [`ColumnType::String`] cells -/// need to interpret `\n` and `\\`. [`ColumnType::Other`] is the catch-all -/// for non-string columns under a schema (numeric, temporal, …) whose -/// accepted spellings cannot contain `\n` or `\\` and so are returned -/// raw — zero copy. +/// Gates per-column unescape: [`ColumnType::String`] cells go through +/// the unescape pass; [`ColumnType::Raw`] cells are returned verbatim +/// as borrowed slices of the input — zero copy. /// /// More variants may be added as the projected-decode API grows; for /// now their only effect is on unescape. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum ColumnType { - /// NSV string column — cells go through the unescape pass. + /// Cells are unescaped — `\n` and `\\` interpreted as their target bytes. String, - /// Non-string column — cells are returned raw, escape sequences and all. - Other, + /// Cells are returned raw, escape sequences and all. + Raw, } /// Per-column projection: `(original_col, kind, unescape)` — built once @@ -288,7 +286,7 @@ fn build_projection( /// /// Each entry of `columns` pairs an original-column index with its /// [`ColumnType`]. Only [`ColumnType::String`] cells are unescaped; -/// [`ColumnType::Other`] cells are returned as borrowed slices of `input` +/// [`ColumnType::Raw`] cells are returned as borrowed slices of `input` /// — zero copy, zero allocation. /// /// Single-pass: scans for cell/row boundaries and writes directly into @@ -296,7 +294,7 @@ fn build_projection( /// entries (same order as `columns`). /// /// Cells are returned as `Cow<[u8]>` — borrowed when no unescaping was -/// needed (always, for `Other`; opportunistically, for `String` cells +/// needed (always, for `Raw`; opportunistically, for `String` cells /// without escape sequences). pub fn decode_bytes_projected<'a>( input: &'a [u8], @@ -1099,8 +1097,8 @@ mod tests { /// Test helper: project column `c` as a String column (i.e. unescape). fn s(c: usize) -> (usize, ColumnType) { (c, ColumnType::String) } - /// Test helper: project column `c` as Other (i.e. raw, no unescape). - fn o(c: usize) -> (usize, ColumnType) { (c, ColumnType::Other) } + /// Test helper: project column `c` as Raw (no unescape). + fn o(c: usize) -> (usize, ColumnType) { (c, ColumnType::Raw) } #[test] fn test_project_subset() { @@ -1173,12 +1171,12 @@ mod tests { assert_eq!(projected_all, full); } - // ── ColumnType::Other (skip-unescape) tests ── + // ── ColumnType::Raw tests ── #[test] - fn test_other_returns_raw_bytes() { + fn test_raw_returns_raw_bytes() { // Cell contains an escape sequence \\n (encoded as backslash-n). - // Other returns raw bytes; String unescapes. + // Raw returns raw bytes; String unescapes. let nsv = b"col\n\nLine 1\\nLine 2\n\n"; let raw = decode_bytes_projected(nsv, &[o(0)]); @@ -1190,10 +1188,10 @@ mod tests { } #[test] - fn test_other_kind_independent_of_projection_order() { + fn test_raw_independent_of_projection_order() { // c0 has an escape, c1 doesn't. Project in REVERSE order with - // c0 as Other (raw). The escape should survive regardless of - // where c0 lands in the projection. + // c0 as Raw. The escape should survive regardless of where c0 + // lands in the projection. let nsv = b"c0\nc1\n\nA\\nB\n42\n\n"; let projected = decode_bytes_projected(nsv, &[s(1), o(0)]); assert_eq!(projected[1][0].as_ref(), b"42"); // c1 in slot 0, unescaped @@ -1211,7 +1209,7 @@ mod tests { } #[test] - fn test_other_parallel() { + fn test_raw_parallel() { // Force the parallel path with > PARALLEL_THRESHOLD bytes. let mut data = Vec::new(); for i in 0..10_000 { From 7c4090be0fefd9484049a9a079f874d59f6efff5 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Apr 2026 16:10:03 +0000 Subject: [PATCH 11/14] restore prose I shouldn't have touched in 18aa0c7 Reverts the doc-comment rewrites and section-header / test-name edits that went beyond the literal Other -> Raw rename. --- src/lib.rs | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 9baa109..5fdde40 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -251,19 +251,21 @@ pub fn escape_bytes(s: &[u8]) -> Cow<'_, [u8]> { // columns entirely (no allocation, no unescape), and directly produces // the final `Vec>>`. -/// Column type for projected decoding. +/// Column kind for projected decoding. /// -/// Gates per-column unescape: [`ColumnType::String`] cells go through -/// the unescape pass; [`ColumnType::Raw`] cells are returned verbatim -/// as borrowed slices of the input — zero copy. +/// Used to gate per-column unescape: only [`ColumnType::String`] cells +/// need to interpret `\n` and `\\`. [`ColumnType::Raw`] is the catch-all +/// for non-string columns under a schema (numeric, temporal, …) whose +/// accepted spellings cannot contain `\n` or `\\` and so are returned +/// raw — zero copy. /// /// More variants may be added as the projected-decode API grows; for /// now their only effect is on unescape. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum ColumnType { - /// Cells are unescaped — `\n` and `\\` interpreted as their target bytes. + /// NSV string column — cells go through the unescape pass. String, - /// Cells are returned raw, escape sequences and all. + /// Non-string column — cells are returned raw, escape sequences and all. Raw, } @@ -1097,7 +1099,7 @@ mod tests { /// Test helper: project column `c` as a String column (i.e. unescape). fn s(c: usize) -> (usize, ColumnType) { (c, ColumnType::String) } - /// Test helper: project column `c` as Raw (no unescape). + /// Test helper: project column `c` as Raw (i.e. raw, no unescape). fn o(c: usize) -> (usize, ColumnType) { (c, ColumnType::Raw) } #[test] @@ -1171,7 +1173,7 @@ mod tests { assert_eq!(projected_all, full); } - // ── ColumnType::Raw tests ── + // ── ColumnType::Raw (skip-unescape) tests ── #[test] fn test_raw_returns_raw_bytes() { @@ -1188,10 +1190,10 @@ mod tests { } #[test] - fn test_raw_independent_of_projection_order() { - // c0 has an escape, c1 doesn't. Project in REVERSE order with - // c0 as Raw. The escape should survive regardless of where c0 - // lands in the projection. + fn test_raw_kind_independent_of_projection_order() { + // c0 has an escape, c1 doesn't. Project in REVERSE order ([1, 0]) + // and project c0 as Raw (raw). The escape in c0 should survive + // regardless of where it lands in the projection order. let nsv = b"c0\nc1\n\nA\\nB\n42\n\n"; let projected = decode_bytes_projected(nsv, &[s(1), o(0)]); assert_eq!(projected[1][0].as_ref(), b"42"); // c1 in slot 0, unescaped From b7d34f9f40111c030f7f14f3ace525a04bd48dd8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Apr 2026 16:19:32 +0000 Subject: [PATCH 12/14] restore master prose I reworded on decode_bytes_projected and build_col_map --- src/lib.rs | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 5fdde40..2ae6b8e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -269,8 +269,9 @@ pub enum ColumnType { Raw, } -/// Per-column projection: `(original_col, kind, unescape)` — built once -/// from a `&[(usize, ColumnType)]` and indexed by original column. +/// Build a column-map: `col_map[original_col] = projected_index`. +/// Entries for non-projected columns are `usize::MAX`. Also produces +/// `unescape_map[original_col] = true` iff the column is a String column. fn build_projection( columns: &[(usize, ColumnType)], ) -> (Vec, Vec, usize) { @@ -287,17 +288,13 @@ fn build_projection( /// Decode only the specified columns from raw bytes. /// /// Each entry of `columns` pairs an original-column index with its -/// [`ColumnType`]. Only [`ColumnType::String`] cells are unescaped; -/// [`ColumnType::Raw`] cells are returned as borrowed slices of `input` -/// — zero copy, zero allocation. +/// [`ColumnType`]; only [`ColumnType::String`] cells are unescaped. /// -/// Single-pass: scans for cell/row boundaries and writes directly into -/// the projected output. Each inner vec has exactly `columns.len()` -/// entries (same order as `columns`). +/// Single-pass: scans for cell/row boundaries and directly unescapes +/// only the cells in projected columns. No intermediate structural index. +/// Each inner vec has exactly `columns.len()` entries (same order as `columns`). /// -/// Cells are returned as `Cow<[u8]>` — borrowed when no unescaping was -/// needed (always, for `Raw`; opportunistically, for `String` cells -/// without escape sequences). +/// Cells are returned as `Cow<[u8]>` — borrowed when no unescaping was needed. pub fn decode_bytes_projected<'a>( input: &'a [u8], columns: &[(usize, ColumnType)], From 3f1aeeedbd4b68065898a766046f2bbd277f5396 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Apr 2026 16:32:50 +0000 Subject: [PATCH 13/14] rename remaining 'kind' references in projected-decode code to 'type' --- src/lib.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 2ae6b8e..12db4ba 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -251,7 +251,7 @@ pub fn escape_bytes(s: &[u8]) -> Cow<'_, [u8]> { // columns entirely (no allocation, no unescape), and directly produces // the final `Vec>>`. -/// Column kind for projected decoding. +/// Column type for projected decoding. /// /// Used to gate per-column unescape: only [`ColumnType::String`] cells /// need to interpret `\n` and `\\`. [`ColumnType::Raw`] is the catch-all @@ -278,9 +278,9 @@ fn build_projection( let max_col = columns.iter().map(|&(c, _)| c).max().unwrap_or(0); let mut col_map = vec![usize::MAX; max_col + 1]; let mut unescape_map = vec![false; max_col + 1]; - for (proj_idx, &(orig_col, kind)) in columns.iter().enumerate() { + for (proj_idx, &(orig_col, ty)) in columns.iter().enumerate() { col_map[orig_col] = proj_idx; - unescape_map[orig_col] = matches!(kind, ColumnType::String); + unescape_map[orig_col] = matches!(ty, ColumnType::String); } (col_map, unescape_map, max_col) } @@ -1187,7 +1187,7 @@ mod tests { } #[test] - fn test_raw_kind_independent_of_projection_order() { + fn test_raw_independent_of_projection_order() { // c0 has an escape, c1 doesn't. Project in REVERSE order ([1, 0]) // and project c0 as Raw (raw). The escape in c0 should survive // regardless of where it lands in the projection order. @@ -1198,7 +1198,7 @@ mod tests { } #[test] - fn test_mixed_kinds() { + fn test_mixed_types() { // c0 needs unescape (has \\n), c1 is plain numeric, c2 has \\\\. let nsv = b"c0\nc1\nc2\n\nA\\nB\n42\n\\\\\n\n"; let projected = decode_bytes_projected(nsv, &[s(0), o(1), o(2)]); From 9bd06a57a2a18cdbed444f45d118789eae27f9e9 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Apr 2026 16:33:52 +0000 Subject: [PATCH 14/14] trim ColumnType doc to just identify the types --- src/lib.rs | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 12db4ba..3c72c6c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -251,21 +251,10 @@ pub fn escape_bytes(s: &[u8]) -> Cow<'_, [u8]> { // columns entirely (no allocation, no unescape), and directly produces // the final `Vec>>`. -/// Column type for projected decoding. -/// -/// Used to gate per-column unescape: only [`ColumnType::String`] cells -/// need to interpret `\n` and `\\`. [`ColumnType::Raw`] is the catch-all -/// for non-string columns under a schema (numeric, temporal, …) whose -/// accepted spellings cannot contain `\n` or `\\` and so are returned -/// raw — zero copy. -/// -/// More variants may be added as the projected-decode API grows; for -/// now their only effect is on unescape. +/// Column types currently recognized. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum ColumnType { - /// NSV string column — cells go through the unescape pass. String, - /// Non-string column — cells are returned raw, escape sequences and all. Raw, }