From 314b38a30a4d0b81051f7fbcb89e6ab8f6d07b71 Mon Sep 17 00:00:00 2001 From: David Hagen Date: Sun, 29 Mar 2026 17:37:23 -0400 Subject: [PATCH 1/7] Check rust formatting as part of lint run --- noxfile.py | 1 + src/data_frame.rs | 8 +++++++- src/function/interp.rs | 6 +----- src/function/power.rs | 7 +++---- src/function/trapz.rs | 5 +---- 5 files changed, 13 insertions(+), 14 deletions(-) diff --git a/noxfile.py b/noxfile.py index 7a5b322..3b90c3d 100644 --- a/noxfile.py +++ b/noxfile.py @@ -77,6 +77,7 @@ def coverage(s: Session): ["ruff", "check", "."], ["ruff", "format", "--check", "."], ["cargo", "clippy", "--locked", "--", "-D", "warnings"], + ["cargo", "fmt", "--check"], ], ) def lint(s: Session, command: list[str]): diff --git a/src/data_frame.rs b/src/data_frame.rs index 53ea8f5..45e90cd 100644 --- a/src/data_frame.rs +++ b/src/data_frame.rs @@ -1157,7 +1157,13 @@ impl PyDataFrame { .agg([all().as_expr().gather(Expr::Literal(LiteralValue::Series( SpecialEq::new(Series::new("".into(), &indexes)), )))]) - .explode(cols(non_group_columns), ExplodeOptions { empty_as_null: false, keep_nulls: false }) + .explode( + cols(non_group_columns), + ExplodeOptions { + empty_as_null: false, + keep_nulls: false, + }, + ) .sort(["_index"], Default::default()) .drop(cols(["_index"])) .collect(); diff --git a/src/function/interp.rs b/src/function/interp.rs index 9e8b306..cbbb60d 100644 --- a/src/function/interp.rs +++ b/src/function/interp.rs @@ -146,11 +146,7 @@ impl Function for Interp { fn to_polars(&self) -> Expr { apply_multiple( interpolate, - &[ - self.t.to_polars(), - self.ts.to_polars(), - self.ys.to_polars(), - ], + &[self.t.to_polars(), self.ts.to_polars(), self.ys.to_polars()], |_, fields| Ok(fields[0].clone()), true, ) diff --git a/src/function/power.rs b/src/function/power.rs index c2e8d44..c0f8b7e 100644 --- a/src/function/power.rs +++ b/src/function/power.rs @@ -6,8 +6,8 @@ use polars::prelude::*; use crate::expression::Expression; use crate::typed_expression::{ - promote_expression_types, require_numeric, DataFrameType, ExpressionType, LiteralType, - Function, TypedExpression, ValidationError, + promote_expression_types, require_numeric, DataFrameType, ExpressionType, Function, + LiteralType, TypedExpression, ValidationError, }; #[derive(Debug, Clone, PartialEq)] @@ -171,8 +171,7 @@ impl Pow { require_numeric(base_type, "pow", "base")?; require_numeric(exponent_type, "pow", "exponent")?; - let promoted_type = - promote_expression_types(base_type, exponent_type, "pow")?; + let promoted_type = promote_expression_types(base_type, exponent_type, "pow")?; // Check wholeness on the original exponent type, before any casting let exponent_is_whole = match exponent_type { diff --git a/src/function/trapz.rs b/src/function/trapz.rs index d29ecfd..8e6149d 100644 --- a/src/function/trapz.rs +++ b/src/function/trapz.rs @@ -112,10 +112,7 @@ impl Function for Trapz { fn to_polars(&self) -> Expr { apply_multiple( compute_trapz, - &[ - self.t.to_polars(), - self.y.to_polars(), - ], + &[self.t.to_polars(), self.y.to_polars()], |_, fields| Ok(fields[0].clone()), true, ) From ea898f94a876d8edefc87ffaa84860237293e90e Mon Sep 17 00:00:00 2001 From: David Hagen Date: Sun, 29 Mar 2026 21:27:18 -0400 Subject: [PATCH 2/7] Implement proper handling of fully null columns --- src/function/branch.rs | 15 ++- src/function/finite.rs | 28 ++++- src/function/log.rs | 9 +- src/function/logical.rs | 16 ++- src/function/power.rs | 29 ++++- src/function/round.rs | 9 +- src/function/sign.rs | 7 +- src/function/trigonometry.rs | 9 +- src/typed_expression/ast.rs | 116 ++++++++++++++++++-- src/typed_expression/validate.rs | 56 +++++++--- tests/functions/test_branch.py | 28 +++++ tests/functions/test_convert.py | 31 +++++- tests/functions/test_empty_data_frames.py | 128 ---------------------- tests/functions/test_finite.py | 15 +++ tests/operators/test_and.py | 30 +++++ tests/operators/test_or.py | 30 +++++ tests/test_null_columns.py | 104 ++++++++++++++++++ 17 files changed, 485 insertions(+), 175 deletions(-) delete mode 100644 tests/functions/test_empty_data_frames.py create mode 100644 tests/test_null_columns.py diff --git a/src/function/branch.rs b/src/function/branch.rs index 2777bbd..80c6001 100644 --- a/src/function/branch.rs +++ b/src/function/branch.rs @@ -70,11 +70,16 @@ impl IfElse { impl Function for IfElse { fn to_polars(&self) -> Expr { - ternary_expr( - self.condition.to_polars(), - self.then_branch.to_polars(), - self.else_branch.to_polars(), - ) + if self.condition.expression_type().data_type() == crate::data_type::DataType::Nothing { + // WORKAROUND: Polars ternary_expr crashes when condition has DataType::Null + lit(NULL) + } else { + ternary_expr( + self.condition.to_polars(), + self.then_branch.to_polars(), + self.else_branch.to_polars(), + ) + } } fn substitute(&self, substitutions: &HashMap<&str, TypedExpression>) -> Arc { diff --git a/src/function/finite.rs b/src/function/finite.rs index dd014cc..94767b3 100644 --- a/src/function/finite.rs +++ b/src/function/finite.rs @@ -96,16 +96,26 @@ impl IsNan { require_numeric(arg_type, "is_nan", "argument")?; + let expression_type = if arg_type.data_type() == DataType::Nothing { + arg_type + } else { + arg_type.with_data_type(DataType::Boolean) + }; Ok(Arc::new(IsNan { argument: Arc::new(typed_arg), - expression_type: arg_type.with_data_type(DataType::Boolean), + expression_type, }) as Arc) } } impl Function for IsNan { fn to_polars(&self) -> Expr { - self.argument.to_polars().is_nan() + if self.argument.expression_type().data_type() == DataType::Nothing { + // WORKAROUND: Polars is_nan() crashes on DataType::Null (Nothing) columns + lit(NULL) + } else { + self.argument.to_polars().is_nan() + } } fn substitute(&self, substitutions: &HashMap<&str, TypedExpression>) -> Arc { @@ -160,16 +170,26 @@ impl IsFinite { require_numeric(arg_type, "is_finite", "argument")?; + let expression_type = if arg_type.data_type() == DataType::Nothing { + arg_type + } else { + arg_type.with_data_type(DataType::Boolean) + }; Ok(Arc::new(IsFinite { argument: Arc::new(typed_arg), - expression_type: arg_type.with_data_type(DataType::Boolean), + expression_type, }) as Arc) } } impl Function for IsFinite { fn to_polars(&self) -> Expr { - self.argument.to_polars().is_finite() + if self.argument.expression_type().data_type() == DataType::Nothing { + // WORKAROUND: Polars is_finite() crashes on DataType::Null (Nothing) columns + lit(NULL) + } else { + self.argument.to_polars().is_finite() + } } fn substitute(&self, substitutions: &HashMap<&str, TypedExpression>) -> Arc { diff --git a/src/function/log.rs b/src/function/log.rs index 27a2dc5..8a09ab4 100644 --- a/src/function/log.rs +++ b/src/function/log.rs @@ -54,7 +54,14 @@ macro_rules! impl_log_function { impl Function for $name { fn to_polars(&self) -> Expr { - self.argument.to_polars().log(Expr::from($base)) + if self.argument.expression_type().data_type() + == crate::data_type::DataType::Nothing + { + // WORKAROUND: Polars returns Unknown(Float) for log on empty DataType::Null columns + lit(NULL) + } else { + self.argument.to_polars().log(Expr::from($base)) + } } fn substitute( diff --git a/src/function/logical.rs b/src/function/logical.rs index ee22afd..8a30c0c 100644 --- a/src/function/logical.rs +++ b/src/function/logical.rs @@ -42,7 +42,13 @@ impl Any { impl Function for Any { fn to_polars(&self) -> Expr { - self.argument.to_polars().any(false) + if self.argument.expression_type().data_type() == crate::data_type::DataType::Nothing { + // WORKAROUND: Polars any() crashes on DataType::Null (Nothing) columns; + // any(nothing) = nothing per three-value logic + lit(NULL) + } else { + self.argument.to_polars().any(false) + } } fn substitute(&self, substitutions: &HashMap<&str, TypedExpression>) -> Arc { @@ -107,7 +113,13 @@ impl All { impl Function for All { fn to_polars(&self) -> Expr { - self.argument.to_polars().all(false) + if self.argument.expression_type().data_type() == crate::data_type::DataType::Nothing { + // WORKAROUND: Polars all() crashes on DataType::Null (Nothing) columns; + // all(nothing) = nothing per three-value logic + lit(NULL) + } else { + self.argument.to_polars().all(false) + } } fn substitute(&self, substitutions: &HashMap<&str, TypedExpression>) -> Arc { diff --git a/src/function/power.rs b/src/function/power.rs index c0f8b7e..0579712 100644 --- a/src/function/power.rs +++ b/src/function/power.rs @@ -45,7 +45,12 @@ impl Sqrt { impl Function for Sqrt { fn to_polars(&self) -> Expr { - self.argument.to_polars().sqrt() + if self.argument.expression_type().data_type() == crate::data_type::DataType::Nothing { + // WORKAROUND: Polars returns Unknown(Float) for sqrt on empty DataType::Null columns + lit(NULL) + } else { + self.argument.to_polars().sqrt() + } } fn substitute(&self, substitutions: &HashMap<&str, TypedExpression>) -> Arc { @@ -111,7 +116,12 @@ impl Exp { impl Function for Exp { fn to_polars(&self) -> Expr { - self.argument.to_polars().exp() + if self.argument.expression_type().data_type() == crate::data_type::DataType::Nothing { + // WORKAROUND: Polars returns Float64 for exp() on DataType::Null (Nothing) columns + lit(NULL) + } else { + self.argument.to_polars().exp() + } } fn substitute(&self, substitutions: &HashMap<&str, TypedExpression>) -> Arc { @@ -191,8 +201,12 @@ impl Pow { } else { // anything ** int/float → float operation let result_dt = promoted_type.data_type(); - let float_dt = result_dt.promote_to_float(result_dt); - promoted_type.with_data_type(float_dt) + if result_dt == crate::data_type::DataType::Nothing { + promoted_type + } else { + let float_dt = result_dt.promote_to_float(result_dt); + promoted_type.with_data_type(float_dt) + } }; let result_dt = expression_type.data_type(); @@ -206,7 +220,12 @@ impl Pow { impl Function for Pow { fn to_polars(&self) -> Expr { - self.base.to_polars().pow(self.exponent.to_polars()) + if self.expression_type.data_type() == crate::data_type::DataType::Nothing { + // WORKAROUND: Polars pow crashes on DataType::Null (Nothing) columns + lit(NULL) + } else { + self.base.to_polars().pow(self.exponent.to_polars()) + } } fn substitute(&self, substitutions: &HashMap<&str, TypedExpression>) -> Arc { diff --git a/src/function/round.rs b/src/function/round.rs index b5ba4ff..21bd901 100644 --- a/src/function/round.rs +++ b/src/function/round.rs @@ -43,7 +43,14 @@ macro_rules! impl_round_function { impl Function for $name { fn to_polars(&self) -> Expr { - self.argument.to_polars().$polars_method() + if self.argument.expression_type().data_type() + == crate::data_type::DataType::Nothing + { + // WORKAROUND: Polars crashes on DataType::Null columns for rounding functions + lit(NULL) + } else { + self.argument.to_polars().$polars_method() + } } fn substitute( diff --git a/src/function/sign.rs b/src/function/sign.rs index 3429138..7706d41 100644 --- a/src/function/sign.rs +++ b/src/function/sign.rs @@ -41,7 +41,12 @@ impl Abs { impl Function for Abs { fn to_polars(&self) -> Expr { - self.argument.to_polars().abs() + if self.argument.expression_type().data_type() == crate::data_type::DataType::Nothing { + // WORKAROUND: Polars abs crashes on DataType::Null columns + lit(NULL) + } else { + self.argument.to_polars().abs() + } } fn substitute(&self, substitutions: &HashMap<&str, TypedExpression>) -> Arc { diff --git a/src/function/trigonometry.rs b/src/function/trigonometry.rs index bfedcae..7b26004 100644 --- a/src/function/trigonometry.rs +++ b/src/function/trigonometry.rs @@ -47,7 +47,14 @@ macro_rules! impl_trig_function { impl Function for $name { fn to_polars(&self) -> Expr { - self.argument.to_polars().$polars_method() + if self.argument.expression_type().data_type() + == crate::data_type::DataType::Nothing + { + // WORKAROUND: Polars crashes on DataType::Null columns for trigonometry functions + lit(NULL) + } else { + self.argument.to_polars().$polars_method() + } } fn substitute( diff --git a/src/typed_expression/ast.rs b/src/typed_expression/ast.rs index 9f258de..7dd66da 100644 --- a/src/typed_expression/ast.rs +++ b/src/typed_expression/ast.rs @@ -119,6 +119,15 @@ pub enum TypedExpression { } impl TypedExpression { + /// Returns true if this expression ultimately originates from a Nothing (all-null) column, + /// even if it has been wrapped in a Cast node during comparison type harmonization. + fn is_nothing_origin(&self) -> bool { + match self { + TypedExpression::Cast { content, .. } => content.is_nothing_origin(), + _ => self.expression_type().data_type() == DataType::Nothing, + } + } + pub fn cast_if_needed(self, target: DataType) -> TypedExpression { let current = self.expression_type(); // Always cast literals because their Polars type may not match the target. @@ -220,24 +229,84 @@ impl TypedExpression { TypedExpression::StringLiteral { value } => lit(value.clone()), TypedExpression::Variable { name, .. } => col(name), TypedExpression::Positive { content, .. } => content.to_polars(), - TypedExpression::Negative { content, .. } => -content.to_polars(), + TypedExpression::Negative { content, .. } => { + if content.expression_type().data_type() == crate::data_type::DataType::Nothing { + // WORKAROUND: Polars neg operation not supported for DataType::Null columns + lit(NULL) + } else { + -content.to_polars() + } + } TypedExpression::Add { left, right, .. } => left.to_polars() + right.to_polars(), TypedExpression::Subtract { left, right, .. } => left.to_polars() - right.to_polars(), TypedExpression::Multiply { left, right, .. } => left.to_polars() * right.to_polars(), - TypedExpression::TrueDivide { left, right, .. } => { - binary_expr(left.to_polars(), Operator::TrueDivide, right.to_polars()) + TypedExpression::TrueDivide { + left, + right, + expression_type, + } => { + if expression_type.data_type() == crate::data_type::DataType::Nothing { + // WORKAROUND: Polars true_divide crashes on DataType::Null (Nothing) columns + lit(NULL) + } else { + binary_expr(left.to_polars(), Operator::TrueDivide, right.to_polars()) + } } TypedExpression::FloorDivide { left, right, .. } => { - left.to_polars().floor_div(right.to_polars()) + if left.expression_type().data_type() == crate::data_type::DataType::Nothing + || right.expression_type().data_type() == crate::data_type::DataType::Nothing + { + // WORKAROUND: Polars floor_div crashes on DataType::Null columns + lit(NULL) + } else { + left.to_polars().floor_div(right.to_polars()) + } } TypedExpression::Mod { left, right, .. } => left.to_polars() % right.to_polars(), - TypedExpression::Power { left, right, .. } => left.to_polars().pow(right.to_polars()), + TypedExpression::Power { + left, + right, + expression_type, + } => { + if expression_type.data_type() == crate::data_type::DataType::Nothing { + // WORKAROUND: Polars pow crashes on DataType::Null (Nothing) columns + lit(NULL) + } else { + left.to_polars().pow(right.to_polars()) + } + } TypedExpression::Call { call } => call.to_polars(), TypedExpression::Equal { left, right, .. } => { - left.to_polars().eq_missing(right.to_polars()) + // WORKAROUND: eq_missing on DataType::Null (Nothing) columns returns null + // instead of respecting eq_missing semantics. Implement manually: + // nothing eq_missing x = x.is_null(); nothing eq_missing nothing = true + let left_is_nothing = left.is_nothing_origin(); + let right_is_nothing = right.is_nothing_origin(); + if left_is_nothing && right_is_nothing { + lit(true) + } else if left_is_nothing { + right.to_polars().is_null() + } else if right_is_nothing { + left.to_polars().is_null() + } else { + left.to_polars().eq_missing(right.to_polars()) + } } TypedExpression::NotEqual { left, right, .. } => { - left.to_polars().neq_missing(right.to_polars()) + // WORKAROUND: neq_missing on DataType::Null (Nothing) columns returns null + // instead of respecting the neq_missing semantics. Implement manually: + // nothing neq_missing x = x.is_not_null(); nothing neq_missing nothing = false + let left_is_nothing = left.is_nothing_origin(); + let right_is_nothing = right.is_nothing_origin(); + if left_is_nothing && right_is_nothing { + lit(false) + } else if left_is_nothing { + right.to_polars().is_not_null() + } else if right_is_nothing { + left.to_polars().is_not_null() + } else { + left.to_polars().neq_missing(right.to_polars()) + } } TypedExpression::GreaterThanOrEqual { left, right, .. } => { left.to_polars().gt_eq(right.to_polars()) @@ -249,9 +318,36 @@ impl TypedExpression { left.to_polars().gt(right.to_polars()) } TypedExpression::LessThan { left, right, .. } => left.to_polars().lt(right.to_polars()), - TypedExpression::Not { content, .. } => content.to_polars().not(), - TypedExpression::And { left, right, .. } => left.to_polars().and(right.to_polars()), - TypedExpression::Or { left, right, .. } => left.to_polars().or(right.to_polars()), + TypedExpression::Not { content, .. } => { + if content.expression_type().data_type() == crate::data_type::DataType::Nothing { + // WORKAROUND: Polars not() crashes on DataType::Null columns + lit(NULL) + } else { + content.to_polars().not() + } + } + TypedExpression::And { left, right, .. } => { + if left.expression_type().data_type() == crate::data_type::DataType::Nothing + && right.expression_type().data_type() == crate::data_type::DataType::Nothing + { + // WORKAROUND: Polars does not support bitand on DataType::Null columns; + // Nothing AND Nothing = Nothing per three-value logic + lit(NULL) + } else { + left.to_polars().and(right.to_polars()) + } + } + TypedExpression::Or { left, right, .. } => { + if left.expression_type().data_type() == crate::data_type::DataType::Nothing + && right.expression_type().data_type() == crate::data_type::DataType::Nothing + { + // WORKAROUND: Polars does not support bitor on DataType::Null columns; + // Nothing OR Nothing = Nothing per three-value logic + lit(NULL) + } else { + left.to_polars().or(right.to_polars()) + } + } TypedExpression::Cast { content, expression_type, diff --git a/src/typed_expression/validate.rs b/src/typed_expression/validate.rs index 1786460..d92e66c 100644 --- a/src/typed_expression/validate.rs +++ b/src/typed_expression/validate.rs @@ -147,14 +147,22 @@ impl Expression { Expression::TrueDivide { left, right } => { validate_binary_arithmetic(left, right, df_type, "division", |l, r, et| { - let float_dt = l - .expression_type() - .data_type() - .promote_to_float(r.expression_type().data_type()); - TypedExpression::TrueDivide { - left: Arc::new(l.cast_if_needed(float_dt)), - right: Arc::new(r.cast_if_needed(float_dt)), - expression_type: et.with_data_type(float_dt), + if et.data_type() == DataType::Nothing { + TypedExpression::TrueDivide { + left: Arc::new(l), + right: Arc::new(r), + expression_type: et, + } + } else { + let float_dt = l + .expression_type() + .data_type() + .promote_to_float(r.expression_type().data_type()); + TypedExpression::TrueDivide { + left: Arc::new(l.cast_if_needed(float_dt)), + right: Arc::new(r.cast_if_needed(float_dt)), + expression_type: et.with_data_type(float_dt), + } } }) } @@ -226,12 +234,20 @@ impl Expression { } else { // anything ** int/float → float operation let result_dt = result_type.data_type(); - let float_dt = result_dt.promote_to_float(result_dt); - Ok(TypedExpression::Power { - left: Arc::new(typed_left.cast_if_needed(float_dt)), - right: Arc::new(typed_right.cast_if_needed(float_dt)), - expression_type: result_type.with_data_type(float_dt), - }) + if result_dt == DataType::Nothing { + Ok(TypedExpression::Power { + left: Arc::new(typed_left), + right: Arc::new(typed_right), + expression_type: result_type, + }) + } else { + let float_dt = result_dt.promote_to_float(result_dt); + Ok(TypedExpression::Power { + left: Arc::new(typed_left.cast_if_needed(float_dt)), + right: Arc::new(typed_right.cast_if_needed(float_dt)), + expression_type: result_type.with_data_type(float_dt), + }) + } } } @@ -437,9 +453,17 @@ where }); } - // Harmonize to get correct shape, then override data type to Boolean + // Harmonize to get correct shape, then override data type to Boolean. + // Exception: if both operands are Nothing, result is Nothing (three-value logic: Nothing + // input → Nothing output, same as any/all). let harmonized = harmonize_expression_types(left_type, right_type, operation)?; - let result_type = harmonized.with_data_type(DataType::Boolean); + let result_type = if left_type.data_type() == DataType::Nothing + && right_type.data_type() == DataType::Nothing + { + harmonized + } else { + harmonized.with_data_type(DataType::Boolean) + }; Ok(constructor(typed_left, typed_right, result_type)) } diff --git a/tests/functions/test_branch.py b/tests/functions/test_branch.py index 9e930b3..5e7a641 100644 --- a/tests/functions/test_branch.py +++ b/tests/functions/test_branch.py @@ -135,3 +135,31 @@ def test_if_else_broadcasts_on_array_condition(): with pytest.raises(SummarizeTypeError): df.group_by("x").summarize(z="if_else(y > 15, 1, 0)") + + +def test_if_else_null_condition(): + df = DataFrame(c=[None, None], a=[5.0, 6.0], b=[7.0, 8.0]) + actual = df.mutate(x="if_else(c, a, b)") + expected = df.mutate(x="c") + assert_data_frames_equal(actual, expected) + + +def test_if_else_null_if_true(): + df = DataFrame(c=[True, False], a=[None, None], b=[5.0, 6.0]) + actual = df.mutate(x="if_else(c, a, b)") + expected = DataFrame(c=[True, False], a=[None, None], b=[5.0, 6.0], x=[None, 6.0]) + assert_data_frames_equal(actual, expected) + + +def test_if_else_null_if_false(): + df = DataFrame(c=[True, False], a=[5.0, 6.0], b=[None, None]) + actual = df.mutate(x="if_else(c, a, b)") + expected = DataFrame(c=[True, False], a=[5.0, 6.0], b=[None, None], x=[5.0, None]) + assert_data_frames_equal(actual, expected) + + +def test_if_else_null_condition_empty(): + df = DataFrame(c=[], a=[], b=[]) + actual = df.mutate(x="if_else(c, a, b)") + expected = df.mutate(x="c") + assert_data_frames_equal(actual, expected) diff --git a/tests/functions/test_convert.py b/tests/functions/test_convert.py index 240226d..45fa171 100644 --- a/tests/functions/test_convert.py +++ b/tests/functions/test_convert.py @@ -1,6 +1,7 @@ import pytest -from tabeline import DataFrame +from tabeline import Array, DataFrame, DataType +from tabeline.testing import assert_data_frames_equal @pytest.mark.parametrize( @@ -124,3 +125,31 @@ def test_cast_string_literal(expression, expected): actual = df.mutate(result=expression) expected = DataFrame(result=[expected]) assert actual == expected + + +@pytest.mark.parametrize("nulls", [[], [None, None]]) +def test_cast_boolean_on_null_column(nulls): + actual = DataFrame(a=nulls).transmute(x="to_boolean(a)") + expected = DataFrame(x=Array[DataType.Boolean](*nulls)) + assert_data_frames_equal(actual, expected) + + +@pytest.mark.parametrize("nulls", [[], [None, None]]) +def test_cast_integer_on_null_column(nulls): + actual = DataFrame(a=nulls).transmute(x="to_integer(a)") + expected = DataFrame(x=Array[DataType.Integer64](*nulls)) + assert_data_frames_equal(actual, expected) + + +@pytest.mark.parametrize("nulls", [[], [None, None]]) +def test_cast_float_on_null_column(nulls): + actual = DataFrame(a=nulls).transmute(x="to_float(a)") + expected = DataFrame(x=Array[DataType.Float64](*nulls)) + assert_data_frames_equal(actual, expected) + + +@pytest.mark.parametrize("nulls", [[], [None, None]]) +def test_cast_string_on_null_column(nulls): + actual = DataFrame(a=nulls).transmute(x="to_string(a)") + expected = DataFrame(x=Array[DataType.String](*nulls)) + assert_data_frames_equal(actual, expected) diff --git a/tests/functions/test_empty_data_frames.py b/tests/functions/test_empty_data_frames.py deleted file mode 100644 index c8ee2b8..0000000 --- a/tests/functions/test_empty_data_frames.py +++ /dev/null @@ -1,128 +0,0 @@ -import pytest - -from tabeline import DataFrame - -zero_argument_functions = [ - "n", - "row_index0", - "row_index1", -] - -one_argument_elementwise_functions = [ - "abs", - "sqrt", - "log", - "log2", - "log10", - "exp", - "sin", - "cos", - "tan", - "arcsin", - "arccos", - "arctan", - "floor", - "ceil", - "is_nan", - "is_finite", - "to_boolean", - "to_integer", - "to_float", - "to_string", -] - -one_argument_math_reduction_functions = [ - "std", - "var", - "max", - "min", - "sum", - "mean", - "median", -] - -one_argument_functions = [ - *one_argument_elementwise_functions, - "is_null", - "first", - "last", - "any", - "all", - "same", - *one_argument_math_reduction_functions, -] - - -@pytest.mark.parametrize("name", zero_argument_functions) -@pytest.mark.parametrize( - "df", - [ - DataFrame(), - DataFrame().group_by(), - DataFrame().group_by().group_by(), - DataFrame(a=[]).group_by("a"), - DataFrame(a=[], b=[]).group_by("a", "b"), - DataFrame(a=[], b=[]).group_by("a").group_by("b"), - ], -) -def test_zero_argument_functions_on_rowless_data_frame_with_mutate(name, df): - actual = df.mutate(x=f"{name}()") - expected = df.mutate(x="1") - assert actual == expected - - -# # https://github.com/pola-rs/polars/issues/15257 -@pytest.mark.skip(reason="Polars operators do not work on all null columns") -@pytest.mark.parametrize("name", zero_argument_functions) -@pytest.mark.parametrize( - "df", - [ - DataFrame(), - DataFrame().group_by(), - DataFrame(a=[]), - DataFrame(a=[]).group_by("a"), - DataFrame(a=[], b=[]).group_by("a", "b"), - DataFrame(a=[], b=[]).group_by("a").group_by("b"), - ], -) -def test_zero_argument_functions_on_rowless_data_frame_with_summarize(name, df): - actual = df.group_by().summarize(x=f"{name}()") - expected = df.mutate(x="1") - assert actual == expected - - -@pytest.mark.parametrize("name", one_argument_functions) -@pytest.mark.parametrize( - "df", - [ - DataFrame(a=[]), - DataFrame(a=[]).group_by(), - DataFrame(a=[]).group_by().group_by(), - DataFrame(a=[]).group_by("a"), - DataFrame(a=[], b=[], c=[]).group_by("a", "b"), - DataFrame(a=[], b=[], c=[]).group_by("a").group_by("b"), - ], -) -def test_one_argument_functions_on_rowless_data_frame_with_mutate(name, df): - actual = df.mutate(x=f"{name}(a)") - expected = df.mutate(x="a") - assert actual == expected - - -# https://github.com/pola-rs/polars/issues/15257 -@pytest.mark.skip(reason="Polars operators do not work on all null columns") -@pytest.mark.parametrize("name", one_argument_functions) -@pytest.mark.parametrize( - "df", - [ - DataFrame(a=[]), - DataFrame(a=[]).group_by(), - DataFrame(a=[]).group_by("a"), - DataFrame(a=[], b=[], c=[]).group_by("a", "b"), - DataFrame(a=[], b=[], c=[]).group_by("a").group_by("b"), - ], -) -def test_one_argument_functions_on_rowless_data_frame_with_summarize(name, df): - actual = df.group_by().summarize(x=f"{name}(a)") - expected = df.mutate(x="a") - assert actual == expected diff --git a/tests/functions/test_finite.py b/tests/functions/test_finite.py index 487f020..db7adb7 100644 --- a/tests/functions/test_finite.py +++ b/tests/functions/test_finite.py @@ -3,6 +3,7 @@ import pytest from tabeline import Array, DataFrame, DataType +from tabeline.testing import assert_data_frames_equal from .._types import numeric_types @@ -88,3 +89,17 @@ def test_is_finite_literal(expression, expected): actual = df.mutate(result=expression) expected = DataFrame(result=[expected]) assert actual == expected + + +def test_is_null_on_nonempty_null_column(): + df = DataFrame(x=[None, None]) + actual = df.mutate(x="is_null(x)") + expected = DataFrame(x=[True, True]) + assert_data_frames_equal(actual, expected) + + +def test_is_null_on_empty_null_column(): + df = DataFrame(x=[]) + actual = df.mutate(x="is_null(x)") + expected = DataFrame(x=Array[DataType.Boolean]()) + assert_data_frames_equal(actual, expected) diff --git a/tests/operators/test_and.py b/tests/operators/test_and.py index 1715ea9..fe15047 100644 --- a/tests/operators/test_and.py +++ b/tests/operators/test_and.py @@ -2,6 +2,36 @@ from tabeline import DataFrame, DataType from tabeline.exceptions import TypeMismatchError +from tabeline.testing import assert_data_frames_equal + + +def test_and(): + df = DataFrame(a=[True, True, False, False], b=[True, False, True, False]) + actual = df.transmute(c="a & b") + expected = DataFrame(c=[True, False, False, False]) + assert_data_frames_equal(actual, expected) + + +def test_and_null_left(): + df = DataFrame(a=[None, None], b=[True, False]) + actual = df.transmute(c="a & b") + expected = DataFrame(c=[None, False]) + assert_data_frames_equal(actual, expected) + + +def test_and_null_right(): + df = DataFrame(a=[True, False], b=[None, None]) + actual = df.transmute(c="a & b") + expected = DataFrame(c=[None, False]) + assert_data_frames_equal(actual, expected) + + +@pytest.mark.parametrize("nulls", [[], [None, None]]) +def test_and_both_null(nulls): + df = DataFrame(a=nulls, b=nulls) + actual = df.transmute(c="a & b") + expected = DataFrame(c=nulls) + assert_data_frames_equal(actual, expected) @pytest.mark.parametrize( diff --git a/tests/operators/test_or.py b/tests/operators/test_or.py index 60e4efd..da42129 100644 --- a/tests/operators/test_or.py +++ b/tests/operators/test_or.py @@ -2,6 +2,36 @@ from tabeline import DataFrame, DataType from tabeline.exceptions import TypeMismatchError +from tabeline.testing import assert_data_frames_equal + + +def test_or(): + df = DataFrame(a=[True, True, False, False], b=[True, False, True, False]) + actual = df.transmute(c="a | b") + expected = DataFrame(c=[True, True, True, False]) + assert_data_frames_equal(actual, expected) + + +def test_or_null_left(): + df = DataFrame(a=[None, None], b=[True, False]) + actual = df.transmute(c="a | b") + expected = DataFrame(c=[True, None]) + assert_data_frames_equal(actual, expected) + + +def test_or_null_right(): + df = DataFrame(a=[True, False], b=[None, None]) + actual = df.transmute(c="a | b") + expected = DataFrame(c=[True, None]) + assert_data_frames_equal(actual, expected) + + +@pytest.mark.parametrize("nulls", [[], [None, None]]) +def test_or_both_null(nulls): + df = DataFrame(a=nulls, b=nulls) + actual = df.transmute(c="a | b") + expected = DataFrame(c=nulls) + assert_data_frames_equal(actual, expected) @pytest.mark.parametrize( diff --git a/tests/test_null_columns.py b/tests/test_null_columns.py new file mode 100644 index 0000000..1611504 --- /dev/null +++ b/tests/test_null_columns.py @@ -0,0 +1,104 @@ +import pytest + +from tabeline import DataFrame +from tabeline.testing import assert_data_frames_equal + + +@pytest.mark.parametrize( + "expression", + [ + "-n", + "+n", + "~n", + "abs(n)", + "sqrt(n)", + "exp(n)", + "log(n)", + "log2(n)", + "log10(n)", + "floor(n)", + "ceil(n)", + "sin(n)", + "cos(n)", + "tan(n)", + "arcsin(n)", + "arccos(n)", + "arctan(n)", + "is_nan(n)", + "is_finite(n)", + "n + a", + "n - a", + "n * a", + "n / a", + "n // a", + "n % a", + "n ** a", + "a + n", + "a - n", + "a * n", + "a / n", + "a // n", + "a % n", + "a ** n", + "n + n", + "n - n", + "n * n", + "n / n", + "n // n", + "n % n", + "n ** n", + "first(n)", + "last(n)", + "same(n)", + "std(n)", + "var(n)", + "max(n)", + "min(n)", + "sum(n)", + "mean(n)", + "median(n)", + "any(n)", + "all(n)", + "pmax(n)", + "pmax(n, a)", + "pmax(a, n)", + "pmax(n, n)", + "pmin(n)", + "pmin(n, a)", + "pmin(a, n)", + "pmin(n, n)", + "pow(n, a)", + "pow(a, n)", + "pow(n, n)", + ], +) +@pytest.mark.parametrize("nulls", [[], [None, None]]) +def test_nothing_is_preserved(expression, nulls): + df = DataFrame(n=nulls, a=list(range(len(nulls)))) + actual = df.transmute(x=expression) + expected = DataFrame(x=nulls) + assert_data_frames_equal(actual, expected) + + +@pytest.mark.parametrize( + "name", + [ + "first", + "last", + "same", + "std", + "var", + "max", + "min", + "sum", + "mean", + "median", + "any", + "all", + ], +) +def test_nothing_preserving_reduction_with_summarize(name): + df = DataFrame(n=[None, None]) + actual = df.group_by().summarize(x=f"{name}(n)") + expected = DataFrame(x=[None]) + assert_data_frames_equal(actual, expected) From ef3a357df2a14965d24d8f53b21019013ec47731 Mon Sep 17 00:00:00 2001 From: David Hagen Date: Tue, 31 Mar 2026 21:37:01 -0400 Subject: [PATCH 3/7] Simplify promote_to_float to preserve Nothing --- src/data_type.rs | 4 +- src/function/elementwise_extrema.rs | 10 +++++ src/function/power.rs | 8 +--- src/typed_expression/ast.rs | 58 ++++++++++++++++++++++++++--- src/typed_expression/validate.rs | 45 ++++++++-------------- 5 files changed, 83 insertions(+), 42 deletions(-) diff --git a/src/data_type.rs b/src/data_type.rs index 12e5602..8899df2 100644 --- a/src/data_type.rs +++ b/src/data_type.rs @@ -150,7 +150,9 @@ impl DataType { /// Returns Float32 if either operand is Float32 and neither is Float64. /// Returns Float64 otherwise (including when both are integers). pub fn promote_to_float(self, other: DataType) -> DataType { - if self == DataType::Float64 || other == DataType::Float64 { + if self == DataType::Nothing || other == DataType::Nothing { + DataType::Nothing + } else if self == DataType::Float64 || other == DataType::Float64 { DataType::Float64 } else if self == DataType::Float32 || other == DataType::Float32 { DataType::Float32 diff --git a/src/function/elementwise_extrema.rs b/src/function/elementwise_extrema.rs index 0044820..45bcbeb 100644 --- a/src/function/elementwise_extrema.rs +++ b/src/function/elementwise_extrema.rs @@ -65,6 +65,11 @@ impl PMax { impl Function for PMax { fn to_polars(&self) -> Expr { + if self.expression_type.data_type() == crate::data_type::DataType::Nothing { + // WORKAROUND: Polars max_horizontal on DataType::Null columns may not preserve Null dtype + return lit(NULL); + } + if self.arguments.len() == 1 { return self.arguments[0].to_polars(); } @@ -168,6 +173,11 @@ impl PMin { impl Function for PMin { fn to_polars(&self) -> Expr { + if self.expression_type.data_type() == crate::data_type::DataType::Nothing { + // WORKAROUND: Polars min_horizontal on DataType::Null columns may not preserve Null dtype + return lit(NULL); + } + if self.arguments.len() == 1 { return self.arguments[0].to_polars(); } diff --git a/src/function/power.rs b/src/function/power.rs index 0579712..2ab2962 100644 --- a/src/function/power.rs +++ b/src/function/power.rs @@ -201,12 +201,8 @@ impl Pow { } else { // anything ** int/float → float operation let result_dt = promoted_type.data_type(); - if result_dt == crate::data_type::DataType::Nothing { - promoted_type - } else { - let float_dt = result_dt.promote_to_float(result_dt); - promoted_type.with_data_type(float_dt) - } + let float_dt = result_dt.promote_to_float(result_dt); + promoted_type.with_data_type(float_dt) }; let result_dt = expression_type.data_type(); diff --git a/src/typed_expression/ast.rs b/src/typed_expression/ast.rs index 7dd66da..5a7a5d3 100644 --- a/src/typed_expression/ast.rs +++ b/src/typed_expression/ast.rs @@ -130,9 +130,13 @@ impl TypedExpression { pub fn cast_if_needed(self, target: DataType) -> TypedExpression { let current = self.expression_type(); + // Casting to Nothing is a no-op: the parent node's Nothing expression_type + // drives the lit(NULL) workaround in to_polars, so the operand keeps its type. + if target == DataType::Nothing { + self // Always cast literals because their Polars type may not match the target. // For concrete types, only cast if the type differs. - if !current.is_literal() && current.data_type() == target { + } else if !current.is_literal() && current.data_type() == target { self } else { TypedExpression::Cast { @@ -237,9 +241,42 @@ impl TypedExpression { -content.to_polars() } } - TypedExpression::Add { left, right, .. } => left.to_polars() + right.to_polars(), - TypedExpression::Subtract { left, right, .. } => left.to_polars() - right.to_polars(), - TypedExpression::Multiply { left, right, .. } => left.to_polars() * right.to_polars(), + TypedExpression::Add { + left, + right, + expression_type, + } => { + if expression_type.data_type() == crate::data_type::DataType::Nothing { + // WORKAROUND: Polars add on DataType::Null columns may not preserve Null dtype + lit(NULL) + } else { + left.to_polars() + right.to_polars() + } + } + TypedExpression::Subtract { + left, + right, + expression_type, + } => { + if expression_type.data_type() == crate::data_type::DataType::Nothing { + // WORKAROUND: Polars subtract on DataType::Null columns may not preserve Null dtype + lit(NULL) + } else { + left.to_polars() - right.to_polars() + } + } + TypedExpression::Multiply { + left, + right, + expression_type, + } => { + if expression_type.data_type() == crate::data_type::DataType::Nothing { + // WORKAROUND: Polars multiply on DataType::Null columns may not preserve Null dtype + lit(NULL) + } else { + left.to_polars() * right.to_polars() + } + } TypedExpression::TrueDivide { left, right, @@ -262,7 +299,18 @@ impl TypedExpression { left.to_polars().floor_div(right.to_polars()) } } - TypedExpression::Mod { left, right, .. } => left.to_polars() % right.to_polars(), + TypedExpression::Mod { + left, + right, + expression_type, + } => { + if expression_type.data_type() == crate::data_type::DataType::Nothing { + // WORKAROUND: Polars modulo on DataType::Null columns may not preserve Null dtype + lit(NULL) + } else { + left.to_polars() % right.to_polars() + } + } TypedExpression::Power { left, right, diff --git a/src/typed_expression/validate.rs b/src/typed_expression/validate.rs index d92e66c..c8ecb8b 100644 --- a/src/typed_expression/validate.rs +++ b/src/typed_expression/validate.rs @@ -147,22 +147,14 @@ impl Expression { Expression::TrueDivide { left, right } => { validate_binary_arithmetic(left, right, df_type, "division", |l, r, et| { - if et.data_type() == DataType::Nothing { - TypedExpression::TrueDivide { - left: Arc::new(l), - right: Arc::new(r), - expression_type: et, - } - } else { - let float_dt = l - .expression_type() - .data_type() - .promote_to_float(r.expression_type().data_type()); - TypedExpression::TrueDivide { - left: Arc::new(l.cast_if_needed(float_dt)), - right: Arc::new(r.cast_if_needed(float_dt)), - expression_type: et.with_data_type(float_dt), - } + let float_dt = l + .expression_type() + .data_type() + .promote_to_float(r.expression_type().data_type()); + TypedExpression::TrueDivide { + left: Arc::new(l.cast_if_needed(float_dt)), + right: Arc::new(r.cast_if_needed(float_dt)), + expression_type: et.with_data_type(float_dt), } }) } @@ -232,22 +224,15 @@ impl Expression { expression_type: result_type.with_data_type(base_dt), }) } else { + // anything ** int/float → float operation // anything ** int/float → float operation let result_dt = result_type.data_type(); - if result_dt == DataType::Nothing { - Ok(TypedExpression::Power { - left: Arc::new(typed_left), - right: Arc::new(typed_right), - expression_type: result_type, - }) - } else { - let float_dt = result_dt.promote_to_float(result_dt); - Ok(TypedExpression::Power { - left: Arc::new(typed_left.cast_if_needed(float_dt)), - right: Arc::new(typed_right.cast_if_needed(float_dt)), - expression_type: result_type.with_data_type(float_dt), - }) - } + let float_dt = result_dt.promote_to_float(result_dt); + Ok(TypedExpression::Power { + left: Arc::new(typed_left.cast_if_needed(float_dt)), + right: Arc::new(typed_right.cast_if_needed(float_dt)), + expression_type: result_type.with_data_type(float_dt), + }) } } From 3ff83ba0c01402fb875a37eaeabbaa7efaf445c0 Mon Sep 17 00:00:00 2001 From: David Hagen Date: Thu, 2 Apr 2026 21:23:50 -0400 Subject: [PATCH 4/7] Handle nulls on inequalities --- src/typed_expression/ast.rs | 58 +++++++++++- src/typed_expression/validate.rs | 47 +++++++--- tests/operators/test_equals.py | 81 +++++++++++------ tests/operators/test_inequalities.py | 128 +++++++++++++++++++++++++++ 4 files changed, 268 insertions(+), 46 deletions(-) create mode 100644 tests/operators/test_inequalities.py diff --git a/src/typed_expression/ast.rs b/src/typed_expression/ast.rs index 5a7a5d3..981b0aa 100644 --- a/src/typed_expression/ast.rs +++ b/src/typed_expression/ast.rs @@ -357,15 +357,65 @@ impl TypedExpression { } } TypedExpression::GreaterThanOrEqual { left, right, .. } => { - left.to_polars().gt_eq(right.to_polars()) + // WORKAROUND: gt_eq on DataType::Null columns returns null instead of + // respecting == semantics. Implement manually: + // nothing >= nothing = true; nothing >= x = x.is_null() + let left_is_nothing = left.is_nothing_origin(); + let right_is_nothing = right.is_nothing_origin(); + if left_is_nothing && right_is_nothing { + lit(true) + } else if left_is_nothing { + right.to_polars().is_null() + } else if right_is_nothing { + left.to_polars().is_null() + } else { + let left_polars = left.to_polars(); + let right_polars = right.to_polars(); + left_polars + .clone() + .gt(right_polars.clone()) + .or(left_polars.eq_missing(right_polars)) + } } TypedExpression::LessThanOrEqual { left, right, .. } => { - left.to_polars().lt_eq(right.to_polars()) + // WORKAROUND: lt_eq on DataType::Null columns returns null instead of + // respecting == semantics. Implement manually: + // nothing <= nothing = true; nothing <= x = x.is_null() + let left_is_nothing = left.is_nothing_origin(); + let right_is_nothing = right.is_nothing_origin(); + if left_is_nothing && right_is_nothing { + lit(true) + } else if left_is_nothing { + right.to_polars().is_null() + } else if right_is_nothing { + left.to_polars().is_null() + } else { + let left_polars = left.to_polars(); + let right_polars = right.to_polars(); + left_polars + .clone() + .lt(right_polars.clone()) + .or(left_polars.eq_missing(right_polars)) + } } TypedExpression::GreaterThan { left, right, .. } => { - left.to_polars().gt(right.to_polars()) + // WORKAROUND: gt on DataType::Null columns returns a Boolean null column + // instead of preserving DataType::Null; propagate Nothing + if left.is_nothing_origin() || right.is_nothing_origin() { + lit(NULL) + } else { + left.to_polars().gt(right.to_polars()) + } + } + TypedExpression::LessThan { left, right, .. } => { + // WORKAROUND: lt on DataType::Null columns returns a Boolean null column + // instead of preserving DataType::Null; propagate Nothing + if left.is_nothing_origin() || right.is_nothing_origin() { + lit(NULL) + } else { + left.to_polars().lt(right.to_polars()) + } } - TypedExpression::LessThan { left, right, .. } => left.to_polars().lt(right.to_polars()), TypedExpression::Not { content, .. } => { if content.expression_type().data_type() == crate::data_type::DataType::Nothing { // WORKAROUND: Polars not() crashes on DataType::Null columns diff --git a/src/typed_expression/validate.rs b/src/typed_expression/validate.rs index c8ecb8b..87e8e11 100644 --- a/src/typed_expression/validate.rs +++ b/src/typed_expression/validate.rs @@ -238,7 +238,7 @@ impl Expression { // Comparison operators Expression::Equal { left, right } => { - validate_comparison(left, right, df_type, "equality", |l, r, et| { + validate_comparison(left, right, df_type, "equality", true, |l, r, et| { TypedExpression::Equal { left: Arc::new(l), right: Arc::new(r), @@ -248,7 +248,7 @@ impl Expression { } Expression::NotEqual { left, right } => { - validate_comparison(left, right, df_type, "inequality", |l, r, et| { + validate_comparison(left, right, df_type, "inequality", true, |l, r, et| { TypedExpression::NotEqual { left: Arc::new(l), right: Arc::new(r), @@ -258,7 +258,7 @@ impl Expression { } Expression::GreaterThan { left, right } => { - validate_comparison(left, right, df_type, "greater than", |l, r, et| { + validate_comparison(left, right, df_type, "greater than", false, |l, r, et| { TypedExpression::GreaterThan { left: Arc::new(l), right: Arc::new(r), @@ -268,17 +268,22 @@ impl Expression { } Expression::GreaterThanOrEqual { left, right } => { - validate_comparison(left, right, df_type, "greater than or equal", |l, r, et| { - TypedExpression::GreaterThanOrEqual { + validate_comparison( + left, + right, + df_type, + "greater than or equal", + true, + |l, r, et| TypedExpression::GreaterThanOrEqual { left: Arc::new(l), right: Arc::new(r), expression_type: et, - } - }) + }, + ) } Expression::LessThan { left, right } => { - validate_comparison(left, right, df_type, "less than", |l, r, et| { + validate_comparison(left, right, df_type, "less than", false, |l, r, et| { TypedExpression::LessThan { left: Arc::new(l), right: Arc::new(r), @@ -288,13 +293,18 @@ impl Expression { } Expression::LessThanOrEqual { left, right } => { - validate_comparison(left, right, df_type, "less than or equal", |l, r, et| { - TypedExpression::LessThanOrEqual { + validate_comparison( + left, + right, + df_type, + "less than or equal", + true, + |l, r, et| TypedExpression::LessThanOrEqual { left: Arc::new(l), right: Arc::new(r), expression_type: et, - } - }) + }, + ) } // Logical operators @@ -373,6 +383,7 @@ fn validate_comparison( right: &Expression, df_type: &DataFrameType, operation: &str, + nulls_equal: bool, constructor: F, ) -> Result where @@ -399,8 +410,16 @@ where let typed_left = typed_left.cast_if_needed(harmonized_dt); let typed_right = typed_right.cast_if_needed(harmonized_dt); - // Result is Boolean with the harmonized shape - let result_type = harmonized.with_data_type(DataType::Boolean); + // Strict ordering of Nothing is undefined; propagate Nothing. Non-strict operators + // treat Nothing as equal to itself and to null (same semantics as ==). + let result_type = if !nulls_equal + && (left_type.data_type() == DataType::Nothing + || right_type.data_type() == DataType::Nothing) + { + harmonized.with_data_type(DataType::Nothing) + } else { + harmonized.with_data_type(DataType::Boolean) + }; Ok(constructor(typed_left, typed_right, result_type)) } diff --git a/tests/operators/test_equals.py b/tests/operators/test_equals.py index 912fd99..79d1d2a 100644 --- a/tests/operators/test_equals.py +++ b/tests/operators/test_equals.py @@ -5,13 +5,12 @@ from tabeline import Array, DataFrame, DataType from tabeline.testing import assert_data_frames_equal -from .._types import float_data_types, integer_data_types, whole_data_types +from .._types import float_data_types, integer_data_types, numeric_types, whole_data_types -@pytest.mark.parametrize("dtype_left", whole_data_types + integer_data_types + float_data_types) -@pytest.mark.parametrize("dtype_right", whole_data_types + integer_data_types + float_data_types) -def test_numbers_equal(dtype_left, dtype_right): - df = DataFrame(a=Array[dtype_left](2), b=Array[dtype_right](2)) +@pytest.mark.parametrize("value", [True, False, "hello", ""]) +def test_equal_basic(value): + df = DataFrame(a=Array(value), b=Array(value)) actual = df.transmute(c="a == b") expected = DataFrame(c=Array(True)) @@ -22,10 +21,10 @@ def test_numbers_equal(dtype_left, dtype_right): assert_data_frames_equal(actual, expected) -@pytest.mark.parametrize("number", [2.25, math.nan, math.inf, -math.inf]) -@pytest.mark.parametrize("dtype", float_data_types) -def test_floats_equal(number, dtype): - df = DataFrame(a=Array[dtype](number), b=Array[dtype](number)) +@pytest.mark.parametrize("dtype_left", numeric_types) +@pytest.mark.parametrize("dtype_right", numeric_types) +def test_equal_numeric(dtype_left, dtype_right): + df = DataFrame(a=Array[dtype_left](2), b=Array[dtype_right](2)) actual = df.transmute(c="a == b") expected = DataFrame(c=Array(True)) @@ -36,9 +35,10 @@ def test_floats_equal(number, dtype): assert_data_frames_equal(actual, expected) -@pytest.mark.parametrize("value", [True, False, "hello", ""]) -def test_values_equal(value): - df = DataFrame(a=Array(value), b=Array(value)) +@pytest.mark.parametrize("number", [2.25, math.nan, math.inf, -math.inf]) +@pytest.mark.parametrize("dtype", float_data_types) +def test_equal_float(number, dtype): + df = DataFrame(a=Array[dtype](number), b=Array[dtype](number)) actual = df.transmute(c="a == b") expected = DataFrame(c=Array(True)) @@ -49,12 +49,9 @@ def test_values_equal(value): assert_data_frames_equal(actual, expected) -@pytest.mark.parametrize( - "dtype", - [DataType.Boolean, DataType.String], -) +@pytest.mark.parametrize("dtype", [DataType.Boolean, DataType.String]) def test_null_equal_null_basic(dtype): - df = DataFrame(a=Array[dtype](None), b=Array[dtype](None), c=Array[DataType.Nothing](None)) + df = DataFrame(a=Array[dtype](None), b=Array[dtype](None)) actual = df.transmute(c="a == b") expected = DataFrame(c=Array(True)) @@ -64,23 +61,22 @@ def test_null_equal_null_basic(dtype): expected = DataFrame(c=Array(False)) assert_data_frames_equal(actual, expected) - actual = df.transmute(c="a == c") + +@pytest.mark.parametrize("dtype", [DataType.Boolean, DataType.String]) +def test_null_equal_nothing_basic(dtype): + df = DataFrame(a=Array[dtype](None), b=Array[DataType.Nothing](None)) + + actual = df.transmute(c="a == b") expected = DataFrame(c=Array(True)) assert_data_frames_equal(actual, expected) - actual = df.transmute(c="a != c") + actual = df.transmute(c="a != b") expected = DataFrame(c=Array(False)) assert_data_frames_equal(actual, expected) -@pytest.mark.parametrize( - "dtype_left", - whole_data_types + integer_data_types + float_data_types + [DataType.Nothing], -) -@pytest.mark.parametrize( - "dtype_right", - whole_data_types + integer_data_types + float_data_types + [DataType.Nothing], -) +@pytest.mark.parametrize("dtype_left", [*numeric_types, DataType.Nothing]) +@pytest.mark.parametrize("dtype_right", [*numeric_types, DataType.Nothing]) def test_null_equal_null_numeric(dtype_left, dtype_right): df = DataFrame(a=Array[dtype_left](None), b=Array[dtype_right](None)) @@ -109,7 +105,7 @@ def test_integers_not_equal_null(dtype): @pytest.mark.parametrize("number", [2.25, math.nan, math.inf, -math.inf]) @pytest.mark.parametrize("dtype", float_data_types) def test_floats_not_equal_null(number, dtype): - df = DataFrame(a=Array[DataType.Float32](math.nan), b=Array[DataType.Float32](None)) + df = DataFrame(a=Array[dtype](number), b=Array[dtype](None)) actual = df.transmute(c="a == b") expected = DataFrame(c=Array(False)) @@ -118,3 +114,32 @@ def test_floats_not_equal_null(number, dtype): actual = df.transmute(c="a != b") expected = DataFrame(c=Array(True)) assert_data_frames_equal(actual, expected) + + +@pytest.mark.parametrize( + "dtype", + [*numeric_types, DataType.Boolean, DataType.String, DataType.Nothing], +) +def test_nothing_equal_null(dtype): + df = DataFrame(a=[None, None], b=Array[dtype](None, None)) + + actual = df.transmute(c="a == b") + expected = DataFrame(c=[True, True]) + assert_data_frames_equal(actual, expected) + + actual = df.transmute(c="a != b") + expected = DataFrame(c=[False, False]) + assert_data_frames_equal(actual, expected) + + +@pytest.mark.parametrize("dtype", numeric_types) +def test_nothing_not_equal_non_null(dtype): + df = DataFrame(a=[None, None], b=Array[dtype](2, 2)) + + actual = df.transmute(c="a == b") + expected = DataFrame(c=[False, False]) + assert_data_frames_equal(actual, expected) + + actual = df.transmute(c="a != b") + expected = DataFrame(c=[True, True]) + assert_data_frames_equal(actual, expected) diff --git a/tests/operators/test_inequalities.py b/tests/operators/test_inequalities.py new file mode 100644 index 0000000..417e119 --- /dev/null +++ b/tests/operators/test_inequalities.py @@ -0,0 +1,128 @@ +import pytest + +from tabeline import Array, DataFrame, DataType +from tabeline.exceptions import IncomparableTypesError +from tabeline.testing import assert_data_frames_equal + +from .._types import numeric_types + + +@pytest.mark.parametrize("dtype_left", numeric_types) +@pytest.mark.parametrize("dtype_right", numeric_types) +@pytest.mark.parametrize( + ("op", "results"), + [ + ("<", [True, False, None, None, None]), + (">", [False, False, None, None, None]), + ("<=", [True, True, None, None, True]), + (">=", [False, True, None, None, True]), + ], +) +def test_numeric_inequalities(dtype_left, dtype_right, op, results): + df = DataFrame( + a=Array[dtype_left](2, 4, None, 3, None), + b=Array[dtype_right](3, 4, 2, None, None), + ) + actual = df.transmute(c=f"a {op} b") + expected = DataFrame(c=results) + assert_data_frames_equal(actual, expected) + + +@pytest.mark.parametrize("dtype", numeric_types) +@pytest.mark.parametrize("op", [">=", "<="]) +def test_null_ge_le_nothing(dtype, op): + df = DataFrame(a=Array[dtype](None, None), b=[None, None]) + expected = DataFrame(c=[True, True]) + + actual = df.transmute(c=f"a {op} b") + assert_data_frames_equal(actual, expected) + + actual = df.transmute(c=f"b {op} a") + assert_data_frames_equal(actual, expected) + + +@pytest.mark.parametrize("dtype", numeric_types) +@pytest.mark.parametrize("op", [">", "<"]) +def test_null_gt_lt_nothing(dtype, op): + df = DataFrame(a=Array[dtype](None, None), b=[None, None]) + expected = DataFrame(c=[None, None]) + + actual = df.transmute(c=f"a {op} b") + assert_data_frames_equal(actual, expected) + + actual = df.transmute(c=f"b {op} a") + assert_data_frames_equal(actual, expected) + + +@pytest.mark.parametrize("op", [">=", "<="]) +@pytest.mark.parametrize("nulls", [[], [None, None]]) +def test_nothing_ge_le_numeric(op, nulls): + df = DataFrame(n=nulls, a=list(range(len(nulls)))) + expected = df.transmute(c="False") + + actual = df.transmute(c=f"n {op} a") + assert_data_frames_equal(actual, expected) + + actual = df.transmute(c=f"a {op} n") + assert_data_frames_equal(actual, expected) + + +@pytest.mark.parametrize("op", [">", "<"]) +@pytest.mark.parametrize("nulls", [[], [None, None]]) +def test_nothing_gt_lt_numeric(op, nulls): + df = DataFrame(n=nulls, a=list(range(len(nulls)))) + expected = DataFrame(c=nulls) + + actual = df.transmute(c=f"n {op} a") + assert_data_frames_equal(actual, expected) + + actual = df.transmute(c=f"a {op} n") + assert_data_frames_equal(actual, expected) + + +@pytest.mark.parametrize("op", [">=", "<="]) +@pytest.mark.parametrize("nulls", [[], [None, None]]) +def test_nothing_ge_le_nothing(op, nulls): + df = DataFrame(a=nulls, b=nulls) + expected = df.transmute(c="True") + + actual = df.transmute(c=f"a {op} b") + assert_data_frames_equal(actual, expected) + + +@pytest.mark.parametrize("op", [">", "<"]) +@pytest.mark.parametrize("nulls", [[], [None, None]]) +def test_nothing_gt_lt_nothing(op, nulls): + df = DataFrame(a=nulls, b=nulls) + expected = DataFrame(c=nulls) + + actual = df.transmute(c=f"a {op} b") + assert_data_frames_equal(actual, expected) + + +@pytest.mark.parametrize( + ("expression", "operation"), + [ + ("x > y", "greater than"), + ("x >= y", "greater than or equal"), + ("x < y", "less than"), + ("x <= y", "less than or equal"), + ], +) +@pytest.mark.parametrize( + ("left_values", "right_values", "left_type", "right_type"), + [ + ([1, 2, 3], ["a", "b", "c"], DataType.Integer64, DataType.String), + ([1, 2, 3], [True, False, True], DataType.Integer64, DataType.Boolean), + (["a", "b", "c"], [True, False, True], DataType.String, DataType.Boolean), + ], +) +def test_inequality_rejects_incomparable_types( + expression, operation, left_values, right_values, left_type, right_type +): + df = DataFrame(x=left_values, y=right_values) + + with pytest.raises(IncomparableTypesError) as exc_info: + df.filter(expression) + + assert exc_info.value == IncomparableTypesError(operation, left_type, right_type) From 9f5a1fdea41bb3eea37e001562d413c8f9c1d504 Mon Sep 17 00:00:00 2001 From: David Hagen Date: Fri, 3 Apr 2026 08:10:51 -0400 Subject: [PATCH 5/7] Test interp and trapz --- src/function/interp.rs | 26 +++++++++++----- src/function/trapz.rs | 24 ++++++++++----- src/typed_expression/validate.rs | 53 +++++++++++++++----------------- tests/functions/test_branch.py | 14 --------- tests/test_null_columns.py | 37 +++++++++++++--------- 5 files changed, 82 insertions(+), 72 deletions(-) diff --git a/src/function/interp.rs b/src/function/interp.rs index cbbb60d..a88387f 100644 --- a/src/function/interp.rs +++ b/src/function/interp.rs @@ -130,8 +130,16 @@ impl Interp { require_numeric(ts_type, "interp", "ts")?; require_numeric(ys_type, "interp", "ys")?; - let result_type = t_type.with_data_type(crate::data_type::DataType::Float64); let float64 = crate::data_type::DataType::Float64; + let nothing = crate::data_type::DataType::Nothing; + let result_type = if t_type.data_type() == nothing + || ts_type.data_type() == nothing + || ys_type.data_type() == nothing + { + t_type.with_data_type(nothing) + } else { + t_type.with_data_type(float64) + }; Ok(Arc::new(Interp { t: Arc::new(t.cast_if_needed(float64)), @@ -144,12 +152,16 @@ impl Interp { impl Function for Interp { fn to_polars(&self) -> Expr { - apply_multiple( - interpolate, - &[self.t.to_polars(), self.ts.to_polars(), self.ys.to_polars()], - |_, fields| Ok(fields[0].clone()), - true, - ) + if self.expression_type.data_type() == crate::data_type::DataType::Nothing { + lit(NULL) + } else { + apply_multiple( + interpolate, + &[self.t.to_polars(), self.ts.to_polars(), self.ys.to_polars()], + |_, fields| Ok(fields[0].clone()), + true, + ) + } } fn substitute(&self, substitutions: &HashMap<&str, TypedExpression>) -> Arc { diff --git a/src/function/trapz.rs b/src/function/trapz.rs index 8e6149d..1f144bb 100644 --- a/src/function/trapz.rs +++ b/src/function/trapz.rs @@ -99,23 +99,33 @@ impl Trapz { require_array(y_type, "trapz", "y")?; let float64 = crate::data_type::DataType::Float64; + let nothing = crate::data_type::DataType::Nothing; + let expression_type = if t_type.data_type() == nothing || y_type.data_type() == nothing { + ExpressionType::Scalar(nothing) + } else { + ExpressionType::Scalar(float64) + }; Ok(Arc::new(Trapz { t: Arc::new(t.cast_if_needed(float64)), y: Arc::new(y.cast_if_needed(float64)), - expression_type: ExpressionType::Scalar(float64), + expression_type, }) as Arc) } } impl Function for Trapz { fn to_polars(&self) -> Expr { - apply_multiple( - compute_trapz, - &[self.t.to_polars(), self.y.to_polars()], - |_, fields| Ok(fields[0].clone()), - true, - ) + if self.expression_type.data_type() == crate::data_type::DataType::Nothing { + lit(NULL) + } else { + apply_multiple( + compute_trapz, + &[self.t.to_polars(), self.y.to_polars()], + |_, fields| Ok(fields[0].clone()), + true, + ) + } } fn substitute(&self, substitutions: &HashMap<&str, TypedExpression>) -> Arc { diff --git a/src/typed_expression/validate.rs b/src/typed_expression/validate.rs index 87e8e11..7c8cd42 100644 --- a/src/typed_expression/validate.rs +++ b/src/typed_expression/validate.rs @@ -224,7 +224,6 @@ impl Expression { expression_type: result_type.with_data_type(base_dt), }) } else { - // anything ** int/float → float operation // anything ** int/float → float operation let result_dt = result_type.data_type(); let float_dt = result_dt.promote_to_float(result_dt); @@ -267,20 +266,18 @@ impl Expression { }) } - Expression::GreaterThanOrEqual { left, right } => { - validate_comparison( - left, - right, - df_type, - "greater than or equal", - true, - |l, r, et| TypedExpression::GreaterThanOrEqual { - left: Arc::new(l), - right: Arc::new(r), - expression_type: et, - }, - ) - } + Expression::GreaterThanOrEqual { left, right } => validate_comparison( + left, + right, + df_type, + "greater than or equal", + true, + |l, r, et| TypedExpression::GreaterThanOrEqual { + left: Arc::new(l), + right: Arc::new(r), + expression_type: et, + }, + ), Expression::LessThan { left, right } => { validate_comparison(left, right, df_type, "less than", false, |l, r, et| { @@ -292,20 +289,18 @@ impl Expression { }) } - Expression::LessThanOrEqual { left, right } => { - validate_comparison( - left, - right, - df_type, - "less than or equal", - true, - |l, r, et| TypedExpression::LessThanOrEqual { - left: Arc::new(l), - right: Arc::new(r), - expression_type: et, - }, - ) - } + Expression::LessThanOrEqual { left, right } => validate_comparison( + left, + right, + df_type, + "less than or equal", + true, + |l, r, et| TypedExpression::LessThanOrEqual { + left: Arc::new(l), + right: Arc::new(r), + expression_type: et, + }, + ), // Logical operators Expression::And { left, right } => { diff --git a/tests/functions/test_branch.py b/tests/functions/test_branch.py index 5e7a641..a9b791b 100644 --- a/tests/functions/test_branch.py +++ b/tests/functions/test_branch.py @@ -137,13 +137,6 @@ def test_if_else_broadcasts_on_array_condition(): df.group_by("x").summarize(z="if_else(y > 15, 1, 0)") -def test_if_else_null_condition(): - df = DataFrame(c=[None, None], a=[5.0, 6.0], b=[7.0, 8.0]) - actual = df.mutate(x="if_else(c, a, b)") - expected = df.mutate(x="c") - assert_data_frames_equal(actual, expected) - - def test_if_else_null_if_true(): df = DataFrame(c=[True, False], a=[None, None], b=[5.0, 6.0]) actual = df.mutate(x="if_else(c, a, b)") @@ -156,10 +149,3 @@ def test_if_else_null_if_false(): actual = df.mutate(x="if_else(c, a, b)") expected = DataFrame(c=[True, False], a=[5.0, 6.0], b=[None, None], x=[5.0, None]) assert_data_frames_equal(actual, expected) - - -def test_if_else_null_condition_empty(): - df = DataFrame(c=[], a=[], b=[]) - actual = df.mutate(x="if_else(c, a, b)") - expected = df.mutate(x="c") - assert_data_frames_equal(actual, expected) diff --git a/tests/test_null_columns.py b/tests/test_null_columns.py index 1611504..8c74406 100644 --- a/tests/test_null_columns.py +++ b/tests/test_null_columns.py @@ -70,6 +70,11 @@ "pow(n, a)", "pow(a, n)", "pow(n, n)", + "if_else(n, a, a)", + "interp(n, a, a)", + "interp(a, n, a)", + "interp(a, a, n)", + "interp(n, n, n)", ], ) @pytest.mark.parametrize("nulls", [[], [None, None]]) @@ -81,24 +86,26 @@ def test_nothing_is_preserved(expression, nulls): @pytest.mark.parametrize( - "name", + "expression", [ - "first", - "last", - "same", - "std", - "var", - "max", - "min", - "sum", - "mean", - "median", - "any", - "all", + "first(n)", + "last(n)", + "same(n)", + "std(n)", + "var(n)", + "max(n)", + "min(n)", + "sum(n)", + "mean(n)", + "median(n)", + "any(n)", + "all(n)", + "quantile(n, 0.5)", + "trapz(n, n)", ], ) -def test_nothing_preserving_reduction_with_summarize(name): +def test_nothing_preserving_reduction_with_summarize(expression): df = DataFrame(n=[None, None]) - actual = df.group_by().summarize(x=f"{name}(n)") + actual = df.group_by().summarize(x=expression) expected = DataFrame(x=[None]) assert_data_frames_equal(actual, expected) From b87955f3b94dd03a82710da56fd406ed86bfbe5a Mon Sep 17 00:00:00 2001 From: David Hagen Date: Fri, 3 Apr 2026 20:33:21 -0400 Subject: [PATCH 6/7] Change ge and le to have nulls not equal --- src/typed_expression/ast.rs | 59 +++++++++------------------- src/typed_expression/validate.rs | 4 +- tests/operators/test_inequalities.py | 52 ++++-------------------- 3 files changed, 28 insertions(+), 87 deletions(-) diff --git a/src/typed_expression/ast.rs b/src/typed_expression/ast.rs index 981b0aa..ea1d532 100644 --- a/src/typed_expression/ast.rs +++ b/src/typed_expression/ast.rs @@ -128,15 +128,16 @@ impl TypedExpression { } } + /// Cast this expression to the target type if needed. + /// + /// Rules: + /// 1. Do not cast if the target is Nothing because there is nothing to cast + /// 2. Otherwise, if this is a literal, cast it because literals do not have + /// a well defined type + /// 3. Otherwise, if the type already matches, do not bother casting pub fn cast_if_needed(self, target: DataType) -> TypedExpression { let current = self.expression_type(); - // Casting to Nothing is a no-op: the parent node's Nothing expression_type - // drives the lit(NULL) workaround in to_polars, so the operand keeps its type. - if target == DataType::Nothing { - self - // Always cast literals because their Polars type may not match the target. - // For concrete types, only cast if the type differs. - } else if !current.is_literal() && current.data_type() == target { + if target == DataType::Nothing || !current.is_literal() && current.data_type() == target { self } else { TypedExpression::Cast { @@ -357,45 +358,21 @@ impl TypedExpression { } } TypedExpression::GreaterThanOrEqual { left, right, .. } => { - // WORKAROUND: gt_eq on DataType::Null columns returns null instead of - // respecting == semantics. Implement manually: - // nothing >= nothing = true; nothing >= x = x.is_null() - let left_is_nothing = left.is_nothing_origin(); - let right_is_nothing = right.is_nothing_origin(); - if left_is_nothing && right_is_nothing { - lit(true) - } else if left_is_nothing { - right.to_polars().is_null() - } else if right_is_nothing { - left.to_polars().is_null() + // WORKAROUND: gt_eq on DataType::Null columns returns a Boolean null column + // instead of preserving DataType::Null; propagate Nothing + if left.is_nothing_origin() || right.is_nothing_origin() { + lit(NULL) } else { - let left_polars = left.to_polars(); - let right_polars = right.to_polars(); - left_polars - .clone() - .gt(right_polars.clone()) - .or(left_polars.eq_missing(right_polars)) + left.to_polars().gt_eq(right.to_polars()) } } TypedExpression::LessThanOrEqual { left, right, .. } => { - // WORKAROUND: lt_eq on DataType::Null columns returns null instead of - // respecting == semantics. Implement manually: - // nothing <= nothing = true; nothing <= x = x.is_null() - let left_is_nothing = left.is_nothing_origin(); - let right_is_nothing = right.is_nothing_origin(); - if left_is_nothing && right_is_nothing { - lit(true) - } else if left_is_nothing { - right.to_polars().is_null() - } else if right_is_nothing { - left.to_polars().is_null() + // WORKAROUND: lt_eq on DataType::Null columns returns a Boolean null column + // instead of preserving DataType::Null; propagate Nothing + if left.is_nothing_origin() || right.is_nothing_origin() { + lit(NULL) } else { - let left_polars = left.to_polars(); - let right_polars = right.to_polars(); - left_polars - .clone() - .lt(right_polars.clone()) - .or(left_polars.eq_missing(right_polars)) + left.to_polars().lt_eq(right.to_polars()) } } TypedExpression::GreaterThan { left, right, .. } => { diff --git a/src/typed_expression/validate.rs b/src/typed_expression/validate.rs index 7c8cd42..ee245ea 100644 --- a/src/typed_expression/validate.rs +++ b/src/typed_expression/validate.rs @@ -271,7 +271,7 @@ impl Expression { right, df_type, "greater than or equal", - true, + false, |l, r, et| TypedExpression::GreaterThanOrEqual { left: Arc::new(l), right: Arc::new(r), @@ -294,7 +294,7 @@ impl Expression { right, df_type, "less than or equal", - true, + false, |l, r, et| TypedExpression::LessThanOrEqual { left: Arc::new(l), right: Arc::new(r), diff --git a/tests/operators/test_inequalities.py b/tests/operators/test_inequalities.py index 417e119..97c4404 100644 --- a/tests/operators/test_inequalities.py +++ b/tests/operators/test_inequalities.py @@ -14,8 +14,8 @@ [ ("<", [True, False, None, None, None]), (">", [False, False, None, None, None]), - ("<=", [True, True, None, None, True]), - (">=", [False, True, None, None, True]), + ("<=", [True, True, None, None, None]), + (">=", [False, True, None, None, None]), ], ) def test_numeric_inequalities(dtype_left, dtype_right, op, results): @@ -29,21 +29,8 @@ def test_numeric_inequalities(dtype_left, dtype_right, op, results): @pytest.mark.parametrize("dtype", numeric_types) -@pytest.mark.parametrize("op", [">=", "<="]) -def test_null_ge_le_nothing(dtype, op): - df = DataFrame(a=Array[dtype](None, None), b=[None, None]) - expected = DataFrame(c=[True, True]) - - actual = df.transmute(c=f"a {op} b") - assert_data_frames_equal(actual, expected) - - actual = df.transmute(c=f"b {op} a") - assert_data_frames_equal(actual, expected) - - -@pytest.mark.parametrize("dtype", numeric_types) -@pytest.mark.parametrize("op", [">", "<"]) -def test_null_gt_lt_nothing(dtype, op): +@pytest.mark.parametrize("op", [">", "<", ">=", "<="]) +def test_null_inequality_nothing(dtype, op): df = DataFrame(a=Array[dtype](None, None), b=[None, None]) expected = DataFrame(c=[None, None]) @@ -54,22 +41,9 @@ def test_null_gt_lt_nothing(dtype, op): assert_data_frames_equal(actual, expected) -@pytest.mark.parametrize("op", [">=", "<="]) +@pytest.mark.parametrize("op", [">", "<", ">=", "<="]) @pytest.mark.parametrize("nulls", [[], [None, None]]) -def test_nothing_ge_le_numeric(op, nulls): - df = DataFrame(n=nulls, a=list(range(len(nulls)))) - expected = df.transmute(c="False") - - actual = df.transmute(c=f"n {op} a") - assert_data_frames_equal(actual, expected) - - actual = df.transmute(c=f"a {op} n") - assert_data_frames_equal(actual, expected) - - -@pytest.mark.parametrize("op", [">", "<"]) -@pytest.mark.parametrize("nulls", [[], [None, None]]) -def test_nothing_gt_lt_numeric(op, nulls): +def test_nothing_inequality_numeric(op, nulls): df = DataFrame(n=nulls, a=list(range(len(nulls)))) expected = DataFrame(c=nulls) @@ -80,19 +54,9 @@ def test_nothing_gt_lt_numeric(op, nulls): assert_data_frames_equal(actual, expected) -@pytest.mark.parametrize("op", [">=", "<="]) -@pytest.mark.parametrize("nulls", [[], [None, None]]) -def test_nothing_ge_le_nothing(op, nulls): - df = DataFrame(a=nulls, b=nulls) - expected = df.transmute(c="True") - - actual = df.transmute(c=f"a {op} b") - assert_data_frames_equal(actual, expected) - - -@pytest.mark.parametrize("op", [">", "<"]) +@pytest.mark.parametrize("op", [">", "<", ">=", "<="]) @pytest.mark.parametrize("nulls", [[], [None, None]]) -def test_nothing_gt_lt_nothing(op, nulls): +def test_nothing_inequality_nothing(op, nulls): df = DataFrame(a=nulls, b=nulls) expected = DataFrame(c=nulls) From bfd3b8b69c491099f104ce0b4c9bb12f911c7644 Mon Sep 17 00:00:00 2001 From: David Hagen Date: Fri, 3 Apr 2026 20:45:40 -0400 Subject: [PATCH 7/7] Code review fixes --- src/typed_expression/ast.rs | 10 ++++++---- src/typed_expression/validate.rs | 5 +++-- tests/operators/test_inequalities.py | 23 ----------------------- tests/test_null_columns.py | 12 ++++++++++++ 4 files changed, 21 insertions(+), 29 deletions(-) diff --git a/src/typed_expression/ast.rs b/src/typed_expression/ast.rs index ea1d532..b4017df 100644 --- a/src/typed_expression/ast.rs +++ b/src/typed_expression/ast.rs @@ -290,10 +290,12 @@ impl TypedExpression { binary_expr(left.to_polars(), Operator::TrueDivide, right.to_polars()) } } - TypedExpression::FloorDivide { left, right, .. } => { - if left.expression_type().data_type() == crate::data_type::DataType::Nothing - || right.expression_type().data_type() == crate::data_type::DataType::Nothing - { + TypedExpression::FloorDivide { + left, + right, + expression_type, + } => { + if expression_type.data_type() == crate::data_type::DataType::Nothing { // WORKAROUND: Polars floor_div crashes on DataType::Null columns lit(NULL) } else { diff --git a/src/typed_expression/validate.rs b/src/typed_expression/validate.rs index ee245ea..36ae73a 100644 --- a/src/typed_expression/validate.rs +++ b/src/typed_expression/validate.rs @@ -405,8 +405,9 @@ where let typed_left = typed_left.cast_if_needed(harmonized_dt); let typed_right = typed_right.cast_if_needed(harmonized_dt); - // Strict ordering of Nothing is undefined; propagate Nothing. Non-strict operators - // treat Nothing as equal to itself and to null (same semantics as ==). + // If nulls are considered equal, then Nothing inputs should produce true + // (not Nothing) results. Otherwise, if either input is Nothing, result is + // Nothing. let result_type = if !nulls_equal && (left_type.data_type() == DataType::Nothing || right_type.data_type() == DataType::Nothing) diff --git a/tests/operators/test_inequalities.py b/tests/operators/test_inequalities.py index 97c4404..2d76760 100644 --- a/tests/operators/test_inequalities.py +++ b/tests/operators/test_inequalities.py @@ -41,29 +41,6 @@ def test_null_inequality_nothing(dtype, op): assert_data_frames_equal(actual, expected) -@pytest.mark.parametrize("op", [">", "<", ">=", "<="]) -@pytest.mark.parametrize("nulls", [[], [None, None]]) -def test_nothing_inequality_numeric(op, nulls): - df = DataFrame(n=nulls, a=list(range(len(nulls)))) - expected = DataFrame(c=nulls) - - actual = df.transmute(c=f"n {op} a") - assert_data_frames_equal(actual, expected) - - actual = df.transmute(c=f"a {op} n") - assert_data_frames_equal(actual, expected) - - -@pytest.mark.parametrize("op", [">", "<", ">=", "<="]) -@pytest.mark.parametrize("nulls", [[], [None, None]]) -def test_nothing_inequality_nothing(op, nulls): - df = DataFrame(a=nulls, b=nulls) - expected = DataFrame(c=nulls) - - actual = df.transmute(c=f"a {op} b") - assert_data_frames_equal(actual, expected) - - @pytest.mark.parametrize( ("expression", "operation"), [ diff --git a/tests/test_null_columns.py b/tests/test_null_columns.py index 8c74406..2abb346 100644 --- a/tests/test_null_columns.py +++ b/tests/test_null_columns.py @@ -75,6 +75,18 @@ "interp(a, n, a)", "interp(a, a, n)", "interp(n, n, n)", + "n > a", + "n < a", + "n >= a", + "n <= a", + "a > n", + "a < n", + "a >= n", + "a <= n", + "n > n", + "n < n", + "n >= n", + "n <= n", ], ) @pytest.mark.parametrize("nulls", [[], [None, None]])